{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 10.285362243652344,
      "kl": 55.3999137878418,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -38240443.33643411,
      "logits/rejected": -38302824.818897635,
      "logps/chosen": -489.996996124031,
      "logps/rejected": -354.4371062992126,
      "loss": 0.4594,
      "loss/base_kto": 3.675299882888794,
      "metrics/advantage_var": 640.5584716796875,
      "rewards/chosen": 1.0600410875423933,
      "rewards/margins": 0.3018039366442634,
      "rewards/rejected": 0.7582371508981299,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 11.506349563598633,
      "kl": 35.54319381713867,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37366320.5382263,
      "logits/rejected": -38880592.971246004,
      "logps/chosen": -479.0196387614679,
      "logps/rejected": -371.96016373801916,
      "loss": 0.4534,
      "loss/base_kto": 3.6272010803222656,
      "metrics/advantage_var": 710.2990112304688,
      "rewards/chosen": 0.7506291100738245,
      "rewards/margins": 0.343996421594142,
      "rewards/rejected": 0.4066326884796825,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 10.1532564163208,
      "kl": 31.856287002563477,
      "learning_rate": 5.9e-07,
      "logits/chosen": -38610258.201834865,
      "logits/rejected": -38919020.77955271,
      "logps/chosen": -463.1452599388379,
      "logps/rejected": -351.8382587859425,
      "loss": 0.4361,
      "loss/base_kto": 3.488476514816284,
      "metrics/advantage_var": 788.6171875,
      "rewards/chosen": 0.7459707930911936,
      "rewards/margins": 0.5374012816532443,
      "rewards/rejected": 0.20856951143794927,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 10.552718162536621,
      "kl": 19.224849700927734,
      "learning_rate": 7.9e-07,
      "logits/chosen": -38283614.11195159,
      "logits/rejected": -39037983.43134087,
      "logps/chosen": -497.19265317700456,
      "logps/rejected": -383.6700323101777,
      "loss": 0.4227,
      "loss/base_kto": 3.381673574447632,
      "metrics/advantage_var": 1146.7421875,
      "rewards/chosen": 0.671803946213715,
      "rewards/margins": 0.732286359608871,
      "rewards/rejected": -0.060482413395156,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 9.475604057312012,
      "kl": 11.81312084197998,
      "learning_rate": 9.9e-07,
      "logits/chosen": -37433490.28571428,
      "logits/rejected": -39362005.46461538,
      "logps/chosen": -490.66299603174605,
      "logps/rejected": -365.20064903846156,
      "loss": 0.4078,
      "loss/base_kto": 3.2620036602020264,
      "metrics/advantage_var": 2193.870849609375,
      "rewards/chosen": 0.6134866381448413,
      "rewards/margins": 1.0385611949357068,
      "rewards/rejected": -0.42507455679086537,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 8.439735412597656,
      "kl": 32.87488555908203,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -38353533.08649469,
      "logits/rejected": -39914458.074074075,
      "logps/chosen": -471.7189871016692,
      "logps/rejected": -354.6473429951691,
      "loss": 0.3855,
      "loss/base_kto": 3.0840797424316406,
      "metrics/advantage_var": 3620.40234375,
      "rewards/chosen": 1.2449965990729324,
      "rewards/margins": 1.3967639550732092,
      "rewards/rejected": -0.15176735600027677,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 8.383479118347168,
      "kl": 38.29127502441406,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -38660961.716088325,
      "logits/rejected": -40106234.45201238,
      "logps/chosen": -478.3365536277602,
      "logps/rejected": -386.499661377709,
      "loss": 0.3895,
      "loss/base_kto": 3.115814208984375,
      "metrics/advantage_var": 3755.92041015625,
      "rewards/chosen": 1.317629456144026,
      "rewards/margins": 1.2437299360730967,
      "rewards/rejected": 0.07389952007092927,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 10.654783248901367,
      "kl": 5.33766508102417,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -38697368.25041186,
      "logits/rejected": -39911624.84398217,
      "logps/chosen": -488.94491350906094,
      "logps/rejected": -395.10261887072807,
      "loss": 0.3803,
      "loss/base_kto": 3.0426743030548096,
      "metrics/advantage_var": 4218.814453125,
      "rewards/chosen": 0.3239868013234272,
      "rewards/margins": 1.5602282255665552,
      "rewards/rejected": -1.2362414242431279,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 8.015681266784668,
      "kl": 8.5147066116333,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -42394393.1936508,
      "logits/rejected": -42333899.22461539,
      "logps/chosen": -500.61165674603177,
      "logps/rejected": -386.1232451923077,
      "loss": 0.3698,
      "loss/base_kto": 2.958132266998291,
      "metrics/advantage_var": 4260.21044921875,
      "rewards/chosen": 0.66495361328125,
      "rewards/margins": 1.7300279822716345,
      "rewards/rejected": -1.0650743689903845,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 9.380935668945312,
      "kl": 22.67376136779785,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -43375171.368421055,
      "logits/rejected": -43511302.0952381,
      "logps/chosen": -475.55556126644734,
      "logps/rejected": -396.3759300595238,
      "loss": 0.3757,
      "loss/base_kto": 3.005749225616455,
      "metrics/advantage_var": 4565.32275390625,
      "rewards/chosen": 1.1464886916311163,
      "rewards/margins": 1.739021394485818,
      "rewards/rejected": -0.5925327028547015,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 7.990581035614014,
      "kl": 12.28122329711914,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -42476874.531645566,
      "logits/rejected": -42830000.98765432,
      "logps/chosen": -489.24673655063293,
      "logps/rejected": -400.7535927854938,
      "loss": 0.3564,
      "loss/base_kto": 2.850947856903076,
      "metrics/advantage_var": 5003.28662109375,
      "rewards/chosen": 0.750718225406695,
      "rewards/margins": 2.134706499875011,
      "rewards/rejected": -1.383988274468316,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 8.215432167053223,
      "kl": 21.210721969604492,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -43546450.33918129,
      "logits/rejected": -43897532.99328859,
      "logps/chosen": -491.60517178362574,
      "logps/rejected": -358.4549863674497,
      "loss": 0.356,
      "loss/base_kto": 2.847736358642578,
      "metrics/advantage_var": 4862.79296875,
      "rewards/chosen": 1.5007320649442617,
      "rewards/margins": 1.9320024850955306,
      "rewards/rejected": -0.4312704201512689,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 9.131887435913086,
      "kl": 28.84025001525879,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -42420382.02469136,
      "logits/rejected": -42786744.70886076,
      "logps/chosen": -473.4367283950617,
      "logps/rejected": -355.517948971519,
      "loss": 0.3616,
      "loss/base_kto": 2.892714738845825,
      "metrics/advantage_var": 5250.83642578125,
      "rewards/chosen": 1.442228434998312,
      "rewards/margins": 1.906059696741785,
      "rewards/rejected": -0.4638312617434731,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 7.937675476074219,
      "kl": 13.750167846679688,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -43773237.037520394,
      "logits/rejected": -44890299.298350826,
      "logps/chosen": -481.702691680261,
      "logps/rejected": -380.5896973388306,
      "loss": 0.3431,
      "loss/base_kto": 2.744920015335083,
      "metrics/advantage_var": 6587.87646484375,
      "rewards/chosen": 0.8538975676743474,
      "rewards/margins": 2.445918817791476,
      "rewards/rejected": -1.592021250117129,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 8.017827033996582,
      "kl": 20.4591121673584,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -42772393.6,
      "logits/rejected": -42926860.8,
      "logps/chosen": -481.9798828125,
      "logps/rejected": -387.5751220703125,
      "loss": 0.3476,
      "loss/base_kto": 2.780568838119507,
      "metrics/advantage_var": 6826.62744140625,
      "rewards/chosen": 1.5604660034179687,
      "rewards/margins": 2.356216478347778,
      "rewards/rejected": -0.7957504749298095,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 8.48287296295166,
      "kl": 21.597509384155273,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -43267334.603945374,
      "logits/rejected": -42670350.42834138,
      "logps/chosen": -445.9710735963581,
      "logps/rejected": -382.9682467793881,
      "loss": 0.361,
      "loss/base_kto": 2.8881566524505615,
      "metrics/advantage_var": 6283.49365234375,
      "rewards/chosen": 1.3021855216829477,
      "rewards/margins": 2.0412563955884426,
      "rewards/rejected": -0.7390708739054952,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 8.588059425354004,
      "kl": 57.5013313293457,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -43813313.794392526,
      "logits/rejected": -43601355.03448276,
      "logps/chosen": -454.7376362928349,
      "logps/rejected": -382.6814753134796,
      "loss": 0.3646,
      "loss/base_kto": 2.9167778491973877,
      "metrics/advantage_var": 6674.80029296875,
      "rewards/chosen": 2.1397389444606696,
      "rewards/margins": 1.9807793733637051,
      "rewards/rejected": 0.1589595710969644,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 7.111739635467529,
      "kl": 25.617679595947266,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -45081461.44945568,
      "logits/rejected": -44390255.321821034,
      "logps/chosen": -477.66524105754274,
      "logps/rejected": -389.9218995290424,
      "loss": 0.3426,
      "loss/base_kto": 2.7409420013427734,
      "metrics/advantage_var": 7744.74853515625,
      "rewards/chosen": 1.7191458267520412,
      "rewards/margins": 2.679072078653042,
      "rewards/rejected": -0.9599262519010008,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 7.782184600830078,
      "kl": 30.644330978393555,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -45203308.8,
      "logits/rejected": -45315228.8,
      "logps/chosen": -485.4841796875,
      "logps/rejected": -381.0125,
      "loss": 0.3284,
      "loss/base_kto": 2.627208709716797,
      "metrics/advantage_var": 8336.6318359375,
      "rewards/chosen": 1.670359230041504,
      "rewards/margins": 2.859906005859375,
      "rewards/rejected": -1.189546775817871,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 8.94594669342041,
      "kl": 24.091541290283203,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -45122213.187800966,
      "logits/rejected": -45430444.22526636,
      "logps/chosen": -477.6580557784912,
      "logps/rejected": -384.92660768645356,
      "loss": 0.3297,
      "loss/base_kto": 2.63732647895813,
      "metrics/advantage_var": 9143.4013671875,
      "rewards/chosen": 1.8824368839661918,
      "rewards/margins": 2.9564070341516944,
      "rewards/rejected": -1.0739701501855023,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 8.263615608215332,
      "kl": 21.048532485961914,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -44807026.93030794,
      "logits/rejected": -44261839.34841629,
      "logps/chosen": -470.25040518638576,
      "logps/rejected": -388.39844928355956,
      "loss": 0.3381,
      "loss/base_kto": 2.7047669887542725,
      "metrics/advantage_var": 9148.7412109375,
      "rewards/chosen": 1.416009096117808,
      "rewards/margins": 2.946486720611115,
      "rewards/rejected": -1.530477624493307,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 8.365622520446777,
      "kl": 15.443939208984375,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -45616454.258675076,
      "logits/rejected": -45114244.359133124,
      "logps/chosen": -482.53677050473186,
      "logps/rejected": -377.2045762383901,
      "loss": 0.3365,
      "loss/base_kto": 2.6917216777801514,
      "metrics/advantage_var": 10364.0166015625,
      "rewards/chosen": 1.4138483956033123,
      "rewards/margins": 3.064291174482956,
      "rewards/rejected": -1.6504427788796439,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 6.976726055145264,
      "kl": 23.841480255126953,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -46631704.66445183,
      "logits/rejected": -46827486.772861354,
      "logps/chosen": -462.5598006644518,
      "logps/rejected": -386.4936854719764,
      "loss": 0.3282,
      "loss/base_kto": 2.625518798828125,
      "metrics/advantage_var": 10974.43359375,
      "rewards/chosen": 1.5608189985205565,
      "rewards/margins": 3.1252206307338675,
      "rewards/rejected": -1.5644016322133112,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 10.27958869934082,
      "kl": 17.574188232421875,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -45412389.52061069,
      "logits/rejected": -45046031.9744,
      "logps/chosen": -442.57266221374044,
      "logps/rejected": -382.1891,
      "loss": 0.3453,
      "loss/base_kto": 2.76213002204895,
      "metrics/advantage_var": 10283.8876953125,
      "rewards/chosen": 1.488247331226145,
      "rewards/margins": 2.860261296069895,
      "rewards/rejected": -1.37201396484375,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 8.24674129486084,
      "kl": 53.127315521240234,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -45054310.79689922,
      "logits/rejected": -43142458.45669291,
      "logps/chosen": -451.6226744186047,
      "logps/rejected": -376.9277066929134,
      "loss": 0.3293,
      "loss/base_kto": 2.6341941356658936,
      "metrics/advantage_var": 9086.931640625,
      "rewards/chosen": 2.6122694858284885,
      "rewards/margins": 2.905609444920516,
      "rewards/rejected": -0.29333995909202754,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.033597946166992,
      "kl": 27.368396759033203,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -43520045.438985735,
      "logits/rejected": -42915855.77812018,
      "logps/chosen": -431.1351030110935,
      "logps/rejected": -346.852561633282,
      "loss": 0.3128,
      "loss/base_kto": 2.5020527839660645,
      "metrics/advantage_var": 10318.7060546875,
      "rewards/chosen": 2.0742268751238115,
      "rewards/margins": 3.4516921687847324,
      "rewards/rejected": -1.3774652936609206,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 10.496723175048828,
      "kl": 10.868855476379395,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -44145195.279106855,
      "logits/rejected": -44725798.419601835,
      "logps/chosen": -456.7164074960128,
      "logps/rejected": -405.54881316998467,
      "loss": 0.3472,
      "loss/base_kto": 2.777850866317749,
      "metrics/advantage_var": 12616.919921875,
      "rewards/chosen": 1.2103571306195176,
      "rewards/margins": 3.257504685773614,
      "rewards/rejected": -2.0471475551540963,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 8.59062385559082,
      "kl": 29.14273452758789,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -45075778.79939668,
      "logits/rejected": -45115972.875202596,
      "logps/chosen": -472.31447963800906,
      "logps/rejected": -381.77162682333875,
      "loss": 0.3205,
      "loss/base_kto": 2.5638864040374756,
      "metrics/advantage_var": 11787.837890625,
      "rewards/chosen": 2.3244678618141967,
      "rewards/margins": 3.2861555897315387,
      "rewards/rejected": -0.961687727917342,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 9.160472869873047,
      "kl": 26.70770263671875,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -47760614.12971342,
      "logits/rejected": -46645672.86871961,
      "logps/chosen": -488.71747737556564,
      "logps/rejected": -395.1577694489465,
      "loss": 0.3214,
      "loss/base_kto": 2.5714452266693115,
      "metrics/advantage_var": 11503.3125,
      "rewards/chosen": 2.2000402850443064,
      "rewards/margins": 3.423707126869671,
      "rewards/rejected": -1.2236668418253647,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 7.438222408294678,
      "kl": 29.855695724487305,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -45878961.10691824,
      "logits/rejected": -46041781.26708075,
      "logps/chosen": -475.91204795597486,
      "logps/rejected": -368.5074970885093,
      "loss": 0.3212,
      "loss/base_kto": 2.5699808597564697,
      "metrics/advantage_var": 12049.52734375,
      "rewards/chosen": 2.0778695352422365,
      "rewards/margins": 3.3896634608415184,
      "rewards/rejected": -1.3117939255992819,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 7.2168684005737305,
      "kl": 26.392969131469727,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -44772896.820512824,
      "logits/rejected": -45504362.146341465,
      "logps/chosen": -445.9444611378205,
      "logps/rejected": -397.6354087271341,
      "loss": 0.3171,
      "loss/base_kto": 2.536456823348999,
      "metrics/advantage_var": 13829.486328125,
      "rewards/chosen": 2.100123870067107,
      "rewards/margins": 3.6253483231325605,
      "rewards/rejected": -1.5252244530654535,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 7.897971153259277,
      "kl": 23.46337127685547,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -46756372.157480314,
      "logits/rejected": -46982729.823255815,
      "logps/chosen": -465.72391732283467,
      "logps/rejected": -399.53880813953486,
      "loss": 0.318,
      "loss/base_kto": 2.543635606765747,
      "metrics/advantage_var": 12641.2958984375,
      "rewards/chosen": 1.9364203986220472,
      "rewards/margins": 3.441243973900632,
      "rewards/rejected": -1.5048235752785852,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 9.333341598510742,
      "kl": 34.5018424987793,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -48553082.384266265,
      "logits/rejected": -47805041.31825525,
      "logps/chosen": -457.4876134644478,
      "logps/rejected": -392.6214155896607,
      "loss": 0.3238,
      "loss/base_kto": 2.5903139114379883,
      "metrics/advantage_var": 14256.0498046875,
      "rewards/chosen": 2.465614584318268,
      "rewards/margins": 3.4557863465166223,
      "rewards/rejected": -0.9901717621983542,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 8.291895866394043,
      "kl": 21.646602630615234,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -48295671.960330576,
      "logits/rejected": -47792886.518518515,
      "logps/chosen": -446.1569214876033,
      "logps/rejected": -388.03953703703706,
      "loss": 0.3251,
      "loss/base_kto": 2.6008059978485107,
      "metrics/advantage_var": 13585.8935546875,
      "rewards/chosen": 1.8920416209323347,
      "rewards/margins": 3.5260764516615017,
      "rewards/rejected": -1.6340348307291668,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 8.178174018859863,
      "kl": 15.580222129821777,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -48345981.443932414,
      "logits/rejected": -48517605.13831478,
      "logps/chosen": -470.8646313364055,
      "logps/rejected": -403.8453895071542,
      "loss": 0.3235,
      "loss/base_kto": 2.5876247882843018,
      "metrics/advantage_var": 14429.3095703125,
      "rewards/chosen": 1.8095410606278801,
      "rewards/margins": 3.8868899729018267,
      "rewards/rejected": -2.077348912273947,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 10.443662643432617,
      "kl": 35.15016555786133,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -48543886.222222224,
      "logits/rejected": -48038218.531645566,
      "logps/chosen": -485.367862654321,
      "logps/rejected": -395.15822784810126,
      "loss": 0.3011,
      "loss/base_kto": 2.4088985919952393,
      "metrics/advantage_var": 14492.658203125,
      "rewards/chosen": 2.147234692985629,
      "rewards/margins": 3.97073647304892,
      "rewards/rejected": -1.8235017800632911,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 8.647013664245605,
      "kl": 25.72066307067871,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -46683429.50318471,
      "logits/rejected": -48587132.073619634,
      "logps/chosen": -455.54488455414014,
      "logps/rejected": -378.162768404908,
      "loss": 0.3145,
      "loss/base_kto": 2.516106128692627,
      "metrics/advantage_var": 15056.3798828125,
      "rewards/chosen": 1.9133522373855494,
      "rewards/margins": 3.77387735690961,
      "rewards/rejected": -1.8605251195240606,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 7.549548149108887,
      "kl": 24.89582061767578,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -47521797.97084548,
      "logits/rejected": -47149724.875420876,
      "logps/chosen": -462.1880466472303,
      "logps/rejected": -416.2795664983165,
      "loss": 0.3123,
      "loss/base_kto": 2.4984090328216553,
      "metrics/advantage_var": 13964.5234375,
      "rewards/chosen": 2.1415812767629374,
      "rewards/margins": 3.6034990383633163,
      "rewards/rejected": -1.461917761600379,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 7.165398120880127,
      "kl": 36.23041915893555,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -47457207.53230769,
      "logits/rejected": -47688159.38853503,
      "logps/chosen": -456.6162019230769,
      "logps/rejected": -383.27644804936307,
      "loss": 0.2998,
      "loss/base_kto": 2.398082733154297,
      "metrics/advantage_var": 14451.869140625,
      "rewards/chosen": 2.6437039888822116,
      "rewards/margins": 4.025765989715711,
      "rewards/rejected": -1.3820620008334992,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 6.993330001831055,
      "kl": 6.0000762939453125,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -47126860.63694268,
      "logits/rejected": -47067569.47239264,
      "logps/chosen": -451.8484773089172,
      "logps/rejected": -389.6282112730061,
      "loss": 0.2586,
      "loss/base_kto": 2.0687718391418457,
      "metrics/advantage_var": 17359.888671875,
      "rewards/chosen": 1.9123997779408837,
      "rewards/margins": 5.285741344155416,
      "rewards/rejected": -3.373341566214532,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 5.417101860046387,
      "kl": 31.433074951171875,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -48107728.05079365,
      "logits/rejected": -49156763.96307693,
      "logps/chosen": -457.90739087301586,
      "logps/rejected": -417.5091346153846,
      "loss": 0.2757,
      "loss/base_kto": 2.2058160305023193,
      "metrics/advantage_var": 15574.1142578125,
      "rewards/chosen": 2.6718709309895834,
      "rewards/margins": 4.651981733273237,
      "rewards/rejected": -1.9801108022836538,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 8.132774353027344,
      "kl": 20.201974868774414,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -47990535.95031056,
      "logits/rejected": -48448924.17610063,
      "logps/chosen": -463.0153823757764,
      "logps/rejected": -422.75388168238993,
      "loss": 0.2484,
      "loss/base_kto": 1.9872734546661377,
      "metrics/advantage_var": 17828.435546875,
      "rewards/chosen": 2.6961753323951863,
      "rewards/margins": 5.303709680985005,
      "rewards/rejected": -2.607534348589819,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 8.2401123046875,
      "kl": 15.7005033493042,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -49548721.29318542,
      "logits/rejected": -48194883.45146379,
      "logps/chosen": -487.02595087163235,
      "logps/rejected": -386.0352224576271,
      "loss": 0.2666,
      "loss/base_kto": 2.1330721378326416,
      "metrics/advantage_var": 15226.6318359375,
      "rewards/chosen": 2.371564426815075,
      "rewards/margins": 4.769108688118523,
      "rewards/rejected": -2.3975442613034477,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 8.672164916992188,
      "kl": 20.882816314697266,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -47332732.252696455,
      "logits/rejected": -48399553.11568938,
      "logps/chosen": -466.80975539291217,
      "logps/rejected": -404.24197702060223,
      "loss": 0.2544,
      "loss/base_kto": 2.0349247455596924,
      "metrics/advantage_var": 17013.724609375,
      "rewards/chosen": 2.4880128457843798,
      "rewards/margins": 5.045798145965144,
      "rewards/rejected": -2.5577853001807647,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 8.386229515075684,
      "kl": 22.4251766204834,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -49451777.97226502,
      "logits/rejected": -49354158.047543585,
      "logps/chosen": -458.80017334360554,
      "logps/rejected": -399.8250297147385,
      "loss": 0.2621,
      "loss/base_kto": 2.097113609313965,
      "metrics/advantage_var": 14544.2685546875,
      "rewards/chosen": 2.6462496388674883,
      "rewards/margins": 4.811208610861348,
      "rewards/rejected": -2.164958971993859,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 8.636312484741211,
      "kl": 12.601666450500488,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -47768676.32970451,
      "logits/rejected": -48685194.24803767,
      "logps/chosen": -451.256949844479,
      "logps/rejected": -393.63358516483515,
      "loss": 0.2516,
      "loss/base_kto": 2.012883424758911,
      "metrics/advantage_var": 14977.6279296875,
      "rewards/chosen": 2.213305626093507,
      "rewards/margins": 4.917228586503633,
      "rewards/rejected": -2.7039229604101256,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 5.238379001617432,
      "kl": 29.21479606628418,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -49693457.75038521,
      "logits/rejected": -50187937.47068146,
      "logps/chosen": -493.2758089368259,
      "logps/rejected": -401.07621830427894,
      "loss": 0.247,
      "loss/base_kto": 1.9758790731430054,
      "metrics/advantage_var": 16148.375,
      "rewards/chosen": 2.7205349726743067,
      "rewards/margins": 5.184224568182429,
      "rewards/rejected": -2.463689595508122,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 6.7903971672058105,
      "kl": 33.20451736450195,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -48231641.972434916,
      "logits/rejected": -48072583.14513557,
      "logps/chosen": -481.6626148545176,
      "logps/rejected": -389.2785087719298,
      "loss": 0.2603,
      "loss/base_kto": 2.0826306343078613,
      "metrics/advantage_var": 14499.822265625,
      "rewards/chosen": 3.0388022827335375,
      "rewards/margins": 4.795185956539857,
      "rewards/rejected": -1.7563836738063199,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 7.804295539855957,
      "kl": 19.160192489624023,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -48358886.4,
      "logits/rejected": -49291187.2,
      "logps/chosen": -440.519384765625,
      "logps/rejected": -394.4533203125,
      "loss": 0.2548,
      "loss/base_kto": 2.0385074615478516,
      "metrics/advantage_var": 15006.9560546875,
      "rewards/chosen": 2.5941253662109376,
      "rewards/margins": 4.971536636352539,
      "rewards/rejected": -2.3774112701416015,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 8.665066719055176,
      "kl": 12.234046936035156,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -49109730.077922076,
      "logits/rejected": -49753285.39759036,
      "logps/chosen": -456.95200892857144,
      "logps/rejected": -397.8301016566265,
      "loss": 0.2594,
      "loss/base_kto": 2.075338363647461,
      "metrics/advantage_var": 16453.171875,
      "rewards/chosen": 1.7339770824878247,
      "rewards/margins": 4.999917185674007,
      "rewards/rejected": -3.2659401031861823,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 6.741236209869385,
      "kl": 29.949003219604492,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -47405793.54340836,
      "logits/rejected": -48602127.56231003,
      "logps/chosen": -454.48939911575565,
      "logps/rejected": -386.7006791413374,
      "loss": 0.2577,
      "loss/base_kto": 2.061556100845337,
      "metrics/advantage_var": 15960.3935546875,
      "rewards/chosen": 2.663867423005426,
      "rewards/margins": 4.971707973204894,
      "rewards/rejected": -2.307840550199468,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 7.356171607971191,
      "kl": 18.42108726501465,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -50573719.68895801,
      "logits/rejected": -50023354.87598116,
      "logps/chosen": -475.37942262830484,
      "logps/rejected": -419.8053375196232,
      "loss": 0.2653,
      "loss/base_kto": 2.122389554977417,
      "metrics/advantage_var": 14337.359375,
      "rewards/chosen": 2.1847764841319983,
      "rewards/margins": 4.641081444517595,
      "rewards/rejected": -2.4563049603855966,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 9.710753440856934,
      "kl": 20.63035011291504,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -48821714.030959755,
      "logits/rejected": -49922997.70347003,
      "logps/chosen": -465.2204914860681,
      "logps/rejected": -399.9880224763407,
      "loss": 0.2516,
      "loss/base_kto": 2.0126187801361084,
      "metrics/advantage_var": 15542.337890625,
      "rewards/chosen": 2.6994681815982973,
      "rewards/margins": 5.189337885980198,
      "rewards/rejected": -2.4898697043819005,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 7.381426811218262,
      "kl": 11.87850570678711,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -49439426.09672387,
      "logits/rejected": -48244963.55555555,
      "logps/chosen": -457.3438475039002,
      "logps/rejected": -399.84497261345854,
      "loss": 0.2693,
      "loss/base_kto": 2.154747247695923,
      "metrics/advantage_var": 14240.484375,
      "rewards/chosen": 2.155618700333951,
      "rewards/margins": 4.671614878138627,
      "rewards/rejected": -2.5159961778046753,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 6.18265962600708,
      "kl": 10.194323539733887,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -47570332.42266463,
      "logits/rejected": -49176585.79904306,
      "logps/chosen": -438.3728943338438,
      "logps/rejected": -400.97306120414675,
      "loss": 0.2297,
      "loss/base_kto": 1.8375297784805298,
      "metrics/advantage_var": 15763.3896484375,
      "rewards/chosen": 2.536791954799962,
      "rewards/margins": 5.334581216662004,
      "rewards/rejected": -2.7977892618620417,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 8.46866226196289,
      "kl": 24.690845489501953,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -49655544.68571428,
      "logits/rejected": -49987552.49230769,
      "logps/chosen": -452.3748015873016,
      "logps/rejected": -388.4841586538462,
      "loss": 0.2663,
      "loss/base_kto": 2.130300998687744,
      "metrics/advantage_var": 14631.166015625,
      "rewards/chosen": 2.4113482762896825,
      "rewards/margins": 4.7256937916142014,
      "rewards/rejected": -2.3143455153245194,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 6.674213886260986,
      "kl": 17.886266708374023,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -47965006.79938744,
      "logits/rejected": -47721463.83413078,
      "logps/chosen": -452.15141653905056,
      "logps/rejected": -397.80699262360446,
      "loss": 0.2471,
      "loss/base_kto": 1.9766285419464111,
      "metrics/advantage_var": 15228.6953125,
      "rewards/chosen": 2.2878199251651035,
      "rewards/margins": 4.977289404082567,
      "rewards/rejected": -2.6894694789174642,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 8.288200378417969,
      "kl": 24.316864013671875,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -50055032.75471698,
      "logits/rejected": -49824163.77639752,
      "logps/chosen": -465.1016607704403,
      "logps/rejected": -398.7861752717391,
      "loss": 0.2521,
      "loss/base_kto": 2.0171926021575928,
      "metrics/advantage_var": 15025.9482421875,
      "rewards/chosen": 2.60492154487274,
      "rewards/margins": 5.016617700491917,
      "rewards/rejected": -2.411696155619177,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 9.282503128051758,
      "kl": 13.939154624938965,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -50474677.67741936,
      "logits/rejected": -50641575.56363636,
      "logps/chosen": -481.20604838709676,
      "logps/rejected": -408.43645833333335,
      "loss": 0.25,
      "loss/base_kto": 2.0002853870391846,
      "metrics/advantage_var": 15123.5830078125,
      "rewards/chosen": 2.093593277469758,
      "rewards/margins": 4.947470859441823,
      "rewards/rejected": -2.8538775819720645,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 9.465474128723145,
      "kl": 16.497163772583008,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -48454600.434108526,
      "logits/rejected": -49707682.06614173,
      "logps/chosen": -461.271511627907,
      "logps/rejected": -380.18929625984254,
      "loss": 0.2406,
      "loss/base_kto": 1.9244734048843384,
      "metrics/advantage_var": 15232.501953125,
      "rewards/chosen": 2.4654355544452518,
      "rewards/margins": 5.215662970476256,
      "rewards/rejected": -2.750227416031004,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 9.055628776550293,
      "kl": 23.978981018066406,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -49989815.30864198,
      "logits/rejected": -50560006.48101266,
      "logps/chosen": -478.30314429012344,
      "logps/rejected": -392.64883306962025,
      "loss": 0.2452,
      "loss/base_kto": 1.9618843793869019,
      "metrics/advantage_var": 15591.2568359375,
      "rewards/chosen": 2.683606183087384,
      "rewards/margins": 5.173647795157202,
      "rewards/rejected": -2.490041612069818,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 6.475921154022217,
      "kl": 13.34227180480957,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -48551734.04006163,
      "logits/rejected": -48786677.045958795,
      "logps/chosen": -440.99017719568565,
      "logps/rejected": -377.6189332408875,
      "loss": 0.2512,
      "loss/base_kto": 2.009838581085205,
      "metrics/advantage_var": 15655.4658203125,
      "rewards/chosen": 2.48008617524557,
      "rewards/margins": 4.9992728813049006,
      "rewards/rejected": -2.5191867060593305,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 7.807542324066162,
      "kl": 19.20454216003418,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -51067883.78947368,
      "logits/rejected": -51769593.9047619,
      "logps/chosen": -488.40409128289474,
      "logps/rejected": -407.9049479166667,
      "loss": 0.2542,
      "loss/base_kto": 2.033954620361328,
      "metrics/advantage_var": 16857.92578125,
      "rewards/chosen": 1.9239341334292763,
      "rewards/margins": 5.132258384150073,
      "rewards/rejected": -3.2083242507207963,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 6.524475574493408,
      "kl": 23.61891746520996,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -48980962.02157165,
      "logits/rejected": -49016809.28050713,
      "logps/chosen": -468.9685092449923,
      "logps/rejected": -386.245418977813,
      "loss": 0.2513,
      "loss/base_kto": 2.0107860565185547,
      "metrics/advantage_var": 15216.9609375,
      "rewards/chosen": 2.5923028366958785,
      "rewards/margins": 4.926098205839599,
      "rewards/rejected": -2.3337953691437203,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 5.242307186126709,
      "kl": 22.227195739746094,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -49624793.61978362,
      "logits/rejected": -49335724.68878357,
      "logps/chosen": -451.85896445131374,
      "logps/rejected": -386.2475562796209,
      "loss": 0.2486,
      "loss/base_kto": 1.9891036748886108,
      "metrics/advantage_var": 16150.3232421875,
      "rewards/chosen": 2.3942293759659967,
      "rewards/margins": 5.14253813324779,
      "rewards/rejected": -2.748308757281793,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 9.66505241394043,
      "kl": 22.66203498840332,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -50443175.5431918,
      "logits/rejected": -49279524.87772194,
      "logps/chosen": -451.52379209370423,
      "logps/rejected": -381.40991415410383,
      "loss": 0.264,
      "loss/base_kto": 2.1119725704193115,
      "metrics/advantage_var": 14104.1640625,
      "rewards/chosen": 2.5510915194912154,
      "rewards/margins": 4.746612377361295,
      "rewards/rejected": -2.1955208578700796,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 6.532532215118408,
      "kl": 16.468366622924805,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -51127444.06697108,
      "logits/rejected": -52672495.56340289,
      "logps/chosen": -449.2677891933029,
      "logps/rejected": -387.2645465489567,
      "loss": 0.2703,
      "loss/base_kto": 2.162468433380127,
      "metrics/advantage_var": 14352.1005859375,
      "rewards/chosen": 2.2623242707738775,
      "rewards/margins": 4.768650373678171,
      "rewards/rejected": -2.5063261029042936,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 9.158368110656738,
      "kl": 26.074420928955078,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -50660605.70746268,
      "logits/rejected": -50919967.89508197,
      "logps/chosen": -441.9116604477612,
      "logps/rejected": -411.3995901639344,
      "loss": 0.2705,
      "loss/base_kto": 2.164259672164917,
      "metrics/advantage_var": 14123.228515625,
      "rewards/chosen": 2.9296955165578358,
      "rewards/margins": 4.729554635410295,
      "rewards/rejected": -1.799859118852459,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 9.339201927185059,
      "kl": 27.37833023071289,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -50341196.63694268,
      "logits/rejected": -50491489.37423313,
      "logps/chosen": -432.54578025477707,
      "logps/rejected": -408.8282208588957,
      "loss": 0.2559,
      "loss/base_kto": 2.046827793121338,
      "metrics/advantage_var": 13350.6435546875,
      "rewards/chosen": 2.771653290766819,
      "rewards/margins": 4.713148285135109,
      "rewards/rejected": -1.94149499436829,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.989336967468262,
      "kl": 17.84269905090332,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -49385063.646879755,
      "logits/rejected": -49007443.415730335,
      "logps/chosen": -481.3707191780822,
      "logps/rejected": -401.7123294542536,
      "loss": 0.2502,
      "loss/base_kto": 2.0016610622406006,
      "metrics/advantage_var": 14103.3125,
      "rewards/chosen": 2.322153773663432,
      "rewards/margins": 5.013345273964395,
      "rewards/rejected": -2.691191500300963,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 8.873215675354004,
      "kl": 25.032546997070312,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -48653897.3784219,
      "logits/rejected": -48986276.710166916,
      "logps/chosen": -460.5298913043478,
      "logps/rejected": -387.5011380880121,
      "loss": 0.2557,
      "loss/base_kto": 2.0458295345306396,
      "metrics/advantage_var": 13571.0654296875,
      "rewards/chosen": 2.3807917547302737,
      "rewards/margins": 4.739925207404781,
      "rewards/rejected": -2.3591334526745067,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 7.878697872161865,
      "kl": 22.722684860229492,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -50874733.147286825,
      "logits/rejected": -52412786.89763779,
      "logps/chosen": -482.9199612403101,
      "logps/rejected": -405.6564960629921,
      "loss": 0.2612,
      "loss/base_kto": 2.0892159938812256,
      "metrics/advantage_var": 13676.0380859375,
      "rewards/chosen": 2.6010393955910853,
      "rewards/margins": 4.692092891223467,
      "rewards/rejected": -2.091053495632382,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 8.507803916931152,
      "kl": 23.1352596282959,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -48004160.726094,
      "logits/rejected": -49818035.54751131,
      "logps/chosen": -450.5213735818476,
      "logps/rejected": -391.52747926093514,
      "loss": 0.235,
      "loss/base_kto": 1.880078911781311,
      "metrics/advantage_var": 14898.5185546875,
      "rewards/chosen": 2.3386291800673624,
      "rewards/margins": 5.1856568508111405,
      "rewards/rejected": -2.8470276707437785,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 7.470147609710693,
      "kl": 15.821776390075684,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -49612119.48346457,
      "logits/rejected": -50831232.99224806,
      "logps/chosen": -438.76028543307086,
      "logps/rejected": -388.28682170542635,
      "loss": 0.2585,
      "loss/base_kto": 2.067831516265869,
      "metrics/advantage_var": 14473.900390625,
      "rewards/chosen": 2.060779096948819,
      "rewards/margins": 4.792124103610641,
      "rewards/rejected": -2.731345006661822,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 6.032121658325195,
      "kl": 17.546789169311523,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -52378246.056782335,
      "logits/rejected": -52407920.54489164,
      "logps/chosen": -488.7177641955836,
      "logps/rejected": -397.67942143962847,
      "loss": 0.2422,
      "loss/base_kto": 1.9377278089523315,
      "metrics/advantage_var": 15465.8251953125,
      "rewards/chosen": 2.5849694092567033,
      "rewards/margins": 5.338190289251382,
      "rewards/rejected": -2.7532208799946787,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 7.5287394523620605,
      "kl": 25.733800888061523,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -51796815.335463256,
      "logits/rejected": -51933249.76146789,
      "logps/chosen": -471.7799520766773,
      "logps/rejected": -385.5217173165138,
      "loss": 0.2477,
      "loss/base_kto": 1.9814552068710327,
      "metrics/advantage_var": 15103.8818359375,
      "rewards/chosen": 2.7011432099266175,
      "rewards/margins": 5.033878884023521,
      "rewards/rejected": -2.3327356740969036,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 9.235956192016602,
      "kl": 29.76057243347168,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -52228933.06623586,
      "logits/rejected": -51011735.81845688,
      "logps/chosen": -465.15690630048465,
      "logps/rejected": -424.50902987897126,
      "loss": 0.2564,
      "loss/base_kto": 2.0513575077056885,
      "metrics/advantage_var": 15029.6220703125,
      "rewards/chosen": 2.412079794148829,
      "rewards/margins": 4.9222398853931555,
      "rewards/rejected": -2.5101600912443267,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 9.331817626953125,
      "kl": 25.071556091308594,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -51467012.24736048,
      "logits/rejected": -50623096.71544716,
      "logps/chosen": -449.86156674208144,
      "logps/rejected": -428.4603658536585,
      "loss": 0.2127,
      "loss/base_kto": 1.7012656927108765,
      "metrics/advantage_var": 15838.7451171875,
      "rewards/chosen": 2.864487223675528,
      "rewards/margins": 5.68939889637624,
      "rewards/rejected": -2.8249116727007113,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 7.488211154937744,
      "kl": 15.822104454040527,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -50769451.14149444,
      "logits/rejected": -50692316.21505377,
      "logps/chosen": -452.8102146263911,
      "logps/rejected": -400.2412634408602,
      "loss": 0.2097,
      "loss/base_kto": 1.6774075031280518,
      "metrics/advantage_var": 17759.546875,
      "rewards/chosen": 2.733765036578398,
      "rewards/margins": 6.010643381792492,
      "rewards/rejected": -3.2768783452140937,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 7.268414497375488,
      "kl": 9.772688865661621,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -49696237.77346278,
      "logits/rejected": -50958045.196374625,
      "logps/chosen": -463.1455299352751,
      "logps/rejected": -399.58780211480365,
      "loss": 0.218,
      "loss/base_kto": 1.7441755533218384,
      "metrics/advantage_var": 17008.37109375,
      "rewards/chosen": 2.337260323434972,
      "rewards/margins": 6.028836955562992,
      "rewards/rejected": -3.691576632128021,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 9.88840103149414,
      "kl": 22.12735939025879,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -50274113.64102564,
      "logits/rejected": -51329461.07317073,
      "logps/chosen": -448.53205128205127,
      "logps/rejected": -396.9389291158537,
      "loss": 0.2082,
      "loss/base_kto": 1.6654434204101562,
      "metrics/advantage_var": 18491.703125,
      "rewards/chosen": 2.5738243689903846,
      "rewards/margins": 5.829478308586421,
      "rewards/rejected": -3.2556539395960367,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 6.961165428161621,
      "kl": 18.75223731994629,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -50454448.49689441,
      "logits/rejected": -50288182.74213836,
      "logps/chosen": -487.41605201863354,
      "logps/rejected": -402.01046580188677,
      "loss": 0.2131,
      "loss/base_kto": 1.7046397924423218,
      "metrics/advantage_var": 15868.8974609375,
      "rewards/chosen": 2.810227672505823,
      "rewards/margins": 5.734512793439883,
      "rewards/rejected": -2.9242851209340603,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 6.136966228485107,
      "kl": 18.14278221130371,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -49533700.24736048,
      "logits/rejected": -50127515.17666127,
      "logps/chosen": -445.97134238310707,
      "logps/rejected": -397.2188006482982,
      "loss": 0.2009,
      "loss/base_kto": 1.6072006225585938,
      "metrics/advantage_var": 16655.65625,
      "rewards/chosen": 2.8560848092241704,
      "rewards/margins": 5.881715420105957,
      "rewards/rejected": -3.025630610881787,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 8.093600273132324,
      "kl": 17.36097526550293,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -50249372.24413145,
      "logits/rejected": -51871648.14976599,
      "logps/chosen": -471.8325508607199,
      "logps/rejected": -434.346236349454,
      "loss": 0.2028,
      "loss/base_kto": 1.6226786375045776,
      "metrics/advantage_var": 17302.814453125,
      "rewards/chosen": 2.768492792693662,
      "rewards/margins": 5.937769085229738,
      "rewards/rejected": -3.1692762925360762,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 8.161694526672363,
      "kl": 14.578579902648926,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51127046.74728682,
      "logits/rejected": -51333645.707086615,
      "logps/chosen": -435.92863372093024,
      "logps/rejected": -429.8619586614173,
      "loss": 0.2108,
      "loss/base_kto": 1.6867679357528687,
      "metrics/advantage_var": 15839.5849609375,
      "rewards/chosen": 2.5460812590843025,
      "rewards/margins": 5.6935579599947355,
      "rewards/rejected": -3.147476700910433,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 7.36198091506958,
      "kl": 19.158613204956055,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -50113851.57232705,
      "logits/rejected": -51104122.43478261,
      "logps/chosen": -465.19536163522014,
      "logps/rejected": -397.4027319487578,
      "loss": 0.2135,
      "loss/base_kto": 1.7078889608383179,
      "metrics/advantage_var": 16287.8193359375,
      "rewards/chosen": 2.5575148864362225,
      "rewards/margins": 5.749604654366152,
      "rewards/rejected": -3.19208976792993,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 9.570615768432617,
      "kl": 17.231595993041992,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -50655309.40472441,
      "logits/rejected": -49515077.06046512,
      "logps/chosen": -457.63287401574803,
      "logps/rejected": -404.8307897286822,
      "loss": 0.2221,
      "loss/base_kto": 1.7766475677490234,
      "metrics/advantage_var": 15297.1845703125,
      "rewards/chosen": 2.469990311269685,
      "rewards/margins": 5.49773869123577,
      "rewards/rejected": -3.027748379966085,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 9.824966430664062,
      "kl": 19.26784324645996,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -52412467.28012519,
      "logits/rejected": -50879607.81279251,
      "logps/chosen": -460.15307120500785,
      "logps/rejected": -395.80703978159124,
      "loss": 0.23,
      "loss/base_kto": 1.8403421640396118,
      "metrics/advantage_var": 14083.1220703125,
      "rewards/chosen": 2.4090841708235526,
      "rewards/margins": 5.085994333167546,
      "rewards/rejected": -2.6769101623439937,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 6.393725395202637,
      "kl": 15.844639778137207,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -50632716.43095599,
      "logits/rejected": -51273401.507246375,
      "logps/chosen": -446.82312215478,
      "logps/rejected": -389.93422906602257,
      "loss": 0.2031,
      "loss/base_kto": 1.6246970891952515,
      "metrics/advantage_var": 16746.81640625,
      "rewards/chosen": 2.7439853975009485,
      "rewards/margins": 5.871646962768661,
      "rewards/rejected": -3.1276615652677133,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 7.77438497543335,
      "kl": 28.328439712524414,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -52250043.568438,
      "logits/rejected": -53629083.38694993,
      "logps/chosen": -470.0669283413849,
      "logps/rejected": -411.1176498482549,
      "loss": 0.1992,
      "loss/base_kto": 1.5935217142105103,
      "metrics/advantage_var": 15910.0224609375,
      "rewards/chosen": 3.0569956314160627,
      "rewards/margins": 5.844844467246867,
      "rewards/rejected": -2.7878488358308045,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 6.337928771972656,
      "kl": 19.673072814941406,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -51213865.86163522,
      "logits/rejected": -52022189.31677019,
      "logps/chosen": -450.1392492138365,
      "logps/rejected": -394.90855493012424,
      "loss": 0.2168,
      "loss/base_kto": 1.7344608306884766,
      "metrics/advantage_var": 16663.849609375,
      "rewards/chosen": 2.961632302722091,
      "rewards/margins": 5.595088894215686,
      "rewards/rejected": -2.633456591493595,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 8.079915046691895,
      "kl": 16.748624801635742,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -53084767.64835165,
      "logits/rejected": -52579020.640746504,
      "logps/chosen": -467.8896683673469,
      "logps/rejected": -415.1285964230171,
      "loss": 0.221,
      "loss/base_kto": 1.768080711364746,
      "metrics/advantage_var": 16247.9970703125,
      "rewards/chosen": 2.7143727157329276,
      "rewards/margins": 5.659321852469903,
      "rewards/rejected": -2.9449491367369753,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 5.926120281219482,
      "kl": 17.201818466186523,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -51085710.222222224,
      "logits/rejected": -52309610.93670886,
      "logps/chosen": -451.0401234567901,
      "logps/rejected": -372.73615506329116,
      "loss": 0.2095,
      "loss/base_kto": 1.6759108304977417,
      "metrics/advantage_var": 14099.5068359375,
      "rewards/chosen": 2.6364719720534335,
      "rewards/margins": 5.612774879490143,
      "rewards/rejected": -2.976302907436709,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 7.158931255340576,
      "kl": 15.858980178833008,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -50755445.20481928,
      "logits/rejected": -51616528.62337662,
      "logps/chosen": -461.50009412650604,
      "logps/rejected": -415.9528206168831,
      "loss": 0.2032,
      "loss/base_kto": 1.625422477722168,
      "metrics/advantage_var": 15971.3046875,
      "rewards/chosen": 2.809034967996988,
      "rewards/margins": 5.888142871811922,
      "rewards/rejected": -3.079107903814935,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 7.33803653717041,
      "kl": 19.720277786254883,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -50146619.81308411,
      "logits/rejected": -50380286.39498433,
      "logps/chosen": -454.36365848909657,
      "logps/rejected": -386.84357856583074,
      "loss": 0.2157,
      "loss/base_kto": 1.7253481149673462,
      "metrics/advantage_var": 15806.0859375,
      "rewards/chosen": 2.822266195422021,
      "rewards/margins": 5.662788602786343,
      "rewards/rejected": -2.8405224073643223,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 6.7824177742004395,
      "kl": 18.44866371154785,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -51437188.12903226,
      "logits/rejected": -52641382.4,
      "logps/chosen": -472.04944556451613,
      "logps/rejected": -402.6609848484849,
      "loss": 0.2058,
      "loss/base_kto": 1.6467617750167847,
      "metrics/advantage_var": 17072.724609375,
      "rewards/chosen": 2.6735591765372986,
      "rewards/margins": 5.771285709444495,
      "rewards/rejected": -3.097726532907197,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 7.855797290802002,
      "kl": 16.122150421142578,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -51113090.556067586,
      "logits/rejected": -51917897.2591415,
      "logps/chosen": -470.721678187404,
      "logps/rejected": -417.7419515103339,
      "loss": 0.2039,
      "loss/base_kto": 1.6308526992797852,
      "metrics/advantage_var": 17782.169921875,
      "rewards/chosen": 2.5794075820852536,
      "rewards/margins": 6.061200668830087,
      "rewards/rejected": -3.4817930867448332,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 10.747197151184082,
      "kl": 14.113003730773926,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -50281886.3627907,
      "logits/rejected": -52991419.46456693,
      "logps/chosen": -472.15610465116276,
      "logps/rejected": -378.7300196850394,
      "loss": 0.2021,
      "loss/base_kto": 1.6164880990982056,
      "metrics/advantage_var": 14748.0537109375,
      "rewards/chosen": 2.6983644470687986,
      "rewards/margins": 5.782121406037795,
      "rewards/rejected": -3.083756958968996,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 6.209463596343994,
      "kl": 15.16028118133545,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -49942296.4006462,
      "logits/rejected": -51573172.86535552,
      "logps/chosen": -440.99828352180936,
      "logps/rejected": -403.07531202723146,
      "loss": 0.2099,
      "loss/base_kto": 1.679508090019226,
      "metrics/advantage_var": 16923.134765625,
      "rewards/chosen": 2.595790289908118,
      "rewards/margins": 6.006971856663035,
      "rewards/rejected": -3.411181566754917,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 7.013144016265869,
      "kl": 14.801623344421387,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -50693691.96396396,
      "logits/rejected": -51218171.830618896,
      "logps/chosen": -453.25079767267266,
      "logps/rejected": -415.81479030944627,
      "loss": 0.2103,
      "loss/base_kto": 1.6823161840438843,
      "metrics/advantage_var": 15928.6845703125,
      "rewards/chosen": 2.736321893182245,
      "rewards/margins": 5.762679140103051,
      "rewards/rejected": -3.026357246920806,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 7.630949020385742,
      "kl": 18.435169219970703,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -52484132.870703764,
      "logits/rejected": -51555151.9760837,
      "logps/chosen": -465.0326309328969,
      "logps/rejected": -394.2265508221226,
      "loss": 0.199,
      "loss/base_kto": 1.5920383930206299,
      "metrics/advantage_var": 14698.828125,
      "rewards/chosen": 2.884322096204992,
      "rewards/margins": 5.721907636657159,
      "rewards/rejected": -2.8375855404521673,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 10.548248291015625,
      "kl": 19.13457489013672,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -52158762.342943855,
      "logits/rejected": -51546728.70853462,
      "logps/chosen": -462.69850151745067,
      "logps/rejected": -411.6543377616747,
      "loss": 0.2165,
      "loss/base_kto": 1.7320812940597534,
      "metrics/advantage_var": 14811.025390625,
      "rewards/chosen": 2.6523830199639606,
      "rewards/margins": 5.49644517399979,
      "rewards/rejected": -2.8440621540358295,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 6.244058609008789,
      "kl": 26.8873233795166,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -50528268.70077519,
      "logits/rejected": -51241775.97480315,
      "logps/chosen": -460.00116279069766,
      "logps/rejected": -399.81500984251966,
      "loss": 0.2054,
      "loss/base_kto": 1.643135666847229,
      "metrics/advantage_var": 14407.8173828125,
      "rewards/chosen": 2.8557206501332364,
      "rewards/margins": 5.599035041742488,
      "rewards/rejected": -2.743314391609252,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 8.370528221130371,
      "kl": 25.490325927734375,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -53414775.359497644,
      "logits/rejected": -53532280.23639191,
      "logps/chosen": -463.2737441130298,
      "logps/rejected": -394.83257192846037,
      "loss": 0.2222,
      "loss/base_kto": 1.7775534391403198,
      "metrics/advantage_var": 14866.4375,
      "rewards/chosen": 2.80948139472135,
      "rewards/margins": 5.5009601654979825,
      "rewards/rejected": -2.691478770776633,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.29795503616333,
      "kl": 20.725143432617188,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -50126091.96261682,
      "logits/rejected": -52081834.131661445,
      "logps/chosen": -438.7496105919003,
      "logps/rejected": -399.84926038401255,
      "loss": 0.2124,
      "loss/base_kto": 1.6995877027511597,
      "metrics/advantage_var": 14963.5888671875,
      "rewards/chosen": 2.7964680989583335,
      "rewards/margins": 5.438754632804336,
      "rewards/rejected": -2.6422865338460033,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 8.448776245117188,
      "kl": 15.388836860656738,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -50943257.52265861,
      "logits/rejected": -50960695.508090615,
      "logps/chosen": -452.53663141993957,
      "logps/rejected": -375.4554004854369,
      "loss": 0.2081,
      "loss/base_kto": 1.6651920080184937,
      "metrics/advantage_var": 14255.4609375,
      "rewards/chosen": 3.0594740576614425,
      "rewards/margins": 5.675124222318097,
      "rewards/rejected": -2.6156501646566546,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 7.798056602478027,
      "kl": 20.17386245727539,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -51199567.4624,
      "logits/rejected": -51550445.63053435,
      "logps/chosen": -455.71965,
      "logps/rejected": -413.4286259541985,
      "loss": 0.2098,
      "loss/base_kto": 1.678598403930664,
      "metrics/advantage_var": 16558.5,
      "rewards/chosen": 2.8292068359375,
      "rewards/margins": 5.923838537690839,
      "rewards/rejected": -3.0946317017533396,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 7.845084190368652,
      "kl": 20.583127975463867,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -53823648.5632,
      "logits/rejected": -52538316.40916031,
      "logps/chosen": -484.1464,
      "logps/rejected": -412.46030534351144,
      "loss": 0.2159,
      "loss/base_kto": 1.7270206212997437,
      "metrics/advantage_var": 16702.03515625,
      "rewards/chosen": 2.6395771484375,
      "rewards/margins": 5.676251915851623,
      "rewards/rejected": -3.036674767414122,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 6.940150737762451,
      "kl": 16.465957641601562,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -51824299.2180937,
      "logits/rejected": -51926562.85627837,
      "logps/chosen": -458.44476978998387,
      "logps/rejected": -406.61365355521934,
      "loss": 0.2052,
      "loss/base_kto": 1.641516089439392,
      "metrics/advantage_var": 14746.0771484375,
      "rewards/chosen": 2.707992036046042,
      "rewards/margins": 5.695761181694207,
      "rewards/rejected": -2.9877691456481656,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 8.538128852844238,
      "kl": 15.000971794128418,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -52334231.32618683,
      "logits/rejected": -51295815.0430622,
      "logps/chosen": -469.02340160796325,
      "logps/rejected": -422.6503688197767,
      "loss": 0.211,
      "loss/base_kto": 1.6879322528839111,
      "metrics/advantage_var": 16872.240234375,
      "rewards/chosen": 2.591564698387251,
      "rewards/margins": 5.641363934581429,
      "rewards/rejected": -3.0497992361941786,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 8.950865745544434,
      "kl": 21.08583641052246,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -51087206.48165869,
      "logits/rejected": -50116241.05359878,
      "logps/chosen": -446.7995414673046,
      "logps/rejected": -396.5116050918836,
      "loss": 0.2158,
      "loss/base_kto": 1.72600257396698,
      "metrics/advantage_var": 17154.9921875,
      "rewards/chosen": 2.645399565141547,
      "rewards/margins": 5.832831893697252,
      "rewards/rejected": -3.1874323285557042,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 8.326827049255371,
      "kl": 17.32566261291504,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -52515542.10909091,
      "logits/rejected": -51846609.75483871,
      "logps/chosen": -461.6983428030303,
      "logps/rejected": -394.51542338709675,
      "loss": 0.2242,
      "loss/base_kto": 1.7936210632324219,
      "metrics/advantage_var": 14815.369140625,
      "rewards/chosen": 2.720080196496212,
      "rewards/margins": 5.3184590239911715,
      "rewards/rejected": -2.5983788274949595,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.218096733093262,
      "kl": 15.86826229095459,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -51368914.91823899,
      "logits/rejected": -50538655.00621118,
      "logps/chosen": -457.04255110062894,
      "logps/rejected": -402.4126067546584,
      "loss": 0.2094,
      "loss/base_kto": 1.6755253076553345,
      "metrics/advantage_var": 14514.9814453125,
      "rewards/chosen": 2.684348244337166,
      "rewards/margins": 5.517094667146269,
      "rewards/rejected": -2.832746422809103,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.36819314956665,
      "kl": 17.891759872436523,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -51435028.86520376,
      "logits/rejected": -50859017.57009346,
      "logps/chosen": -462.7827194357367,
      "logps/rejected": -381.15272098909657,
      "loss": 0.1961,
      "loss/base_kto": 1.568899154663086,
      "metrics/advantage_var": 14655.2255859375,
      "rewards/chosen": 2.8108076615767046,
      "rewards/margins": 5.7752992710490565,
      "rewards/rejected": -2.964491609472352,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 7.518880844116211,
      "kl": 13.201791763305664,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -51772033.81417323,
      "logits/rejected": -52071339.85736434,
      "logps/chosen": -462.1588090551181,
      "logps/rejected": -407.6047480620155,
      "loss": 0.2082,
      "loss/base_kto": 1.6658432483673096,
      "metrics/advantage_var": 15404.8857421875,
      "rewards/chosen": 2.8972886934055118,
      "rewards/margins": 5.823128991370628,
      "rewards/rejected": -2.925840297965116,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.977114557203087e+17,
      "train_loss": 0.27261167929374286,
      "train_runtime": 11062.6242,
      "train_samples_per_second": 13.398,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.977114557203087e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}