{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 10.767630577087402,
      "kl": 18.16756248474121,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -39039743.18730158,
      "logits/rejected": -38866717.14461538,
      "logps/chosen": -492.0530257936508,
      "logps/rejected": -370.46858173076924,
      "loss": 0.475,
      "loss/base_kto": 3.799793004989624,
      "metrics/advantage_var": 574.7128295898438,
      "rewards/chosen": 0.17904757545107886,
      "rewards/margins": 0.1815789432665367,
      "rewards/rejected": -0.0025313678154578576,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 9.717031478881836,
      "kl": 13.010165214538574,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -38004318.75583204,
      "logits/rejected": -38601925.72684459,
      "logps/chosen": -497.3858864696734,
      "logps/rejected": -377.9446624803768,
      "loss": 0.4455,
      "loss/base_kto": 3.5642311573028564,
      "metrics/advantage_var": 685.6561889648438,
      "rewards/chosen": 0.36572222909882873,
      "rewards/margins": 0.4868113319950954,
      "rewards/rejected": -0.12108910289626668,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 9.341075897216797,
      "kl": 41.835079193115234,
      "learning_rate": 5.9e-07,
      "logits/chosen": -37667752.318042815,
      "logits/rejected": -38090961.38019169,
      "logps/chosen": -464.83524464831805,
      "logps/rejected": -390.9385982428115,
      "loss": 0.4285,
      "loss/base_kto": 3.4281976222991943,
      "metrics/advantage_var": 1246.1971435546875,
      "rewards/chosen": 1.098412003356747,
      "rewards/margins": 0.5388954622444018,
      "rewards/rejected": 0.5595165411123453,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 11.590300559997559,
      "kl": 10.14930248260498,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37663127.658767775,
      "logits/rejected": -39938677.910355486,
      "logps/chosen": -475.84547788309635,
      "logps/rejected": -390.74145092735705,
      "loss": 0.4368,
      "loss/base_kto": 3.494100332260132,
      "metrics/advantage_var": 1539.3009033203125,
      "rewards/chosen": 0.45085223591158174,
      "rewards/margins": 0.5807951755873388,
      "rewards/rejected": -0.1299429396757571,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 10.024121284484863,
      "kl": 7.55079984664917,
      "learning_rate": 9.9e-07,
      "logits/chosen": -38242689.01105845,
      "logits/rejected": -37975244.16692427,
      "logps/chosen": -469.24180489731435,
      "logps/rejected": -385.064142194745,
      "loss": 0.4168,
      "loss/base_kto": 3.334552764892578,
      "metrics/advantage_var": 2498.937744140625,
      "rewards/chosen": 0.3644587078365669,
      "rewards/margins": 0.9627499309999652,
      "rewards/rejected": -0.5982912231633983,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 7.915276050567627,
      "kl": 25.576862335205078,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -39889112.55589124,
      "logits/rejected": -39712290.79611651,
      "logps/chosen": -466.6666351963746,
      "logps/rejected": -365.86524069579286,
      "loss": 0.4039,
      "loss/base_kto": 3.2308411598205566,
      "metrics/advantage_var": 2719.4296875,
      "rewards/chosen": 1.257896584686556,
      "rewards/margins": 1.1497725955196048,
      "rewards/rejected": 0.1081239891669511,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 9.480732917785645,
      "kl": 21.655780792236328,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -39355852.8,
      "logits/rejected": -40773708.8,
      "logps/chosen": -486.68837890625,
      "logps/rejected": -367.1841796875,
      "loss": 0.3849,
      "loss/base_kto": 3.0790488719940186,
      "metrics/advantage_var": 3540.862548828125,
      "rewards/chosen": 1.2245597839355469,
      "rewards/margins": 1.3650813817977905,
      "rewards/rejected": -0.14052159786224366,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 7.545348644256592,
      "kl": 17.1834659576416,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -39774104.34355828,
      "logits/rejected": -40451626.39490446,
      "logps/chosen": -471.5014378834356,
      "logps/rejected": -368.3911723726115,
      "loss": 0.3973,
      "loss/base_kto": 3.1785781383514404,
      "metrics/advantage_var": 3657.906982421875,
      "rewards/chosen": 0.9276840116348735,
      "rewards/margins": 1.2241404154279167,
      "rewards/rejected": -0.2964564037930434,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 7.958818435668945,
      "kl": 12.04544734954834,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -40987777.24271844,
      "logits/rejected": -40042390.81570997,
      "logps/chosen": -464.29571197411,
      "logps/rejected": -378.16432212990935,
      "loss": 0.3868,
      "loss/base_kto": 3.0944411754608154,
      "metrics/advantage_var": 4641.96533203125,
      "rewards/chosen": 0.6893991019733515,
      "rewards/margins": 1.5439098212957174,
      "rewards/rejected": -0.8545107193223659,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 9.993614196777344,
      "kl": 28.704259872436523,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -40937725.49837133,
      "logits/rejected": -41801798.726726726,
      "logps/chosen": -460.3880293159609,
      "logps/rejected": -372.76360735735733,
      "loss": 0.3701,
      "loss/base_kto": 2.960904598236084,
      "metrics/advantage_var": 5026.33154296875,
      "rewards/chosen": 1.4231080294432004,
      "rewards/margins": 1.8823935478717853,
      "rewards/rejected": -0.45928551842858484,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 7.57697868347168,
      "kl": 22.163732528686523,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -42421246.491899855,
      "logits/rejected": -42000642.98169717,
      "logps/chosen": -480.43611929307804,
      "logps/rejected": -385.0795029118136,
      "loss": 0.3502,
      "loss/base_kto": 2.8012216091156006,
      "metrics/advantage_var": 5297.86181640625,
      "rewards/chosen": 1.5864491525911266,
      "rewards/margins": 2.1334137249834937,
      "rewards/rejected": -0.5469645723923668,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 10.573487281799316,
      "kl": 36.15892028808594,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -42419916.151898734,
      "logits/rejected": -42658585.28395062,
      "logps/chosen": -481.88399920886076,
      "logps/rejected": -379.80114293981484,
      "loss": 0.3628,
      "loss/base_kto": 2.9025771617889404,
      "metrics/advantage_var": 5566.79541015625,
      "rewards/chosen": 1.7851292091079904,
      "rewards/margins": 2.0506995788755447,
      "rewards/rejected": -0.265570369767554,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 9.307571411132812,
      "kl": 11.663390159606934,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -43635637.975903615,
      "logits/rejected": -42869414.233766235,
      "logps/chosen": -473.2483998493976,
      "logps/rejected": -375.81688818993507,
      "loss": 0.3579,
      "loss/base_kto": 2.8630669116973877,
      "metrics/advantage_var": 5602.22607421875,
      "rewards/chosen": 0.9390560288027109,
      "rewards/margins": 2.1901733883108787,
      "rewards/rejected": -1.2511173595081677,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 7.039910793304443,
      "kl": 21.85430908203125,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -43542039.48623853,
      "logits/rejected": -43862886.23642173,
      "logps/chosen": -515.8261181192661,
      "logps/rejected": -375.6230780750799,
      "loss": 0.3502,
      "loss/base_kto": 2.801739454269409,
      "metrics/advantage_var": 5745.34326171875,
      "rewards/chosen": 1.5303742295011469,
      "rewards/margins": 2.2513882340469986,
      "rewards/rejected": -0.7210140045458516,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 10.740477561950684,
      "kl": 25.465660095214844,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -43092120.64596273,
      "logits/rejected": -43654388.72955975,
      "logps/chosen": -467.90828804347825,
      "logps/rejected": -364.07173742138366,
      "loss": 0.3515,
      "loss/base_kto": 2.8117666244506836,
      "metrics/advantage_var": 5965.7041015625,
      "rewards/chosen": 1.3311123107531055,
      "rewards/margins": 2.160643422560299,
      "rewards/rejected": -0.8295311118071934,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 9.744688987731934,
      "kl": 17.645050048828125,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -44299203.57256778,
      "logits/rejected": -44484341.41500766,
      "logps/chosen": -453.71630781499204,
      "logps/rejected": -392.85255551301685,
      "loss": 0.3614,
      "loss/base_kto": 2.8914036750793457,
      "metrics/advantage_var": 7013.50634765625,
      "rewards/chosen": 1.324709075489683,
      "rewards/margins": 2.303055521878944,
      "rewards/rejected": -0.978346446389261,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 7.595713138580322,
      "kl": 15.515207290649414,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -43934022.80851064,
      "logits/rejected": -44995774.971061096,
      "logps/chosen": -458.45117781155017,
      "logps/rejected": -380.3065213022508,
      "loss": 0.3471,
      "loss/base_kto": 2.7768313884735107,
      "metrics/advantage_var": 7399.63134765625,
      "rewards/chosen": 1.3333393317225495,
      "rewards/margins": 2.5059771730484135,
      "rewards/rejected": -1.1726378413258642,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 9.919568061828613,
      "kl": 21.106929779052734,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -43839122.40661157,
      "logits/rejected": -44855490.180740744,
      "logps/chosen": -468.6537190082645,
      "logps/rejected": -384.3515972222222,
      "loss": 0.3418,
      "loss/base_kto": 2.7347142696380615,
      "metrics/advantage_var": 8413.7919921875,
      "rewards/chosen": 1.445809255552686,
      "rewards/margins": 2.578799978208936,
      "rewards/rejected": -1.13299072265625,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 9.684901237487793,
      "kl": 29.528310775756836,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -44314246.056782335,
      "logits/rejected": -45580085.10216718,
      "logps/chosen": -474.65348974763407,
      "logps/rejected": -365.3409442724458,
      "loss": 0.3338,
      "loss/base_kto": 2.670184373855591,
      "metrics/advantage_var": 8667.9609375,
      "rewards/chosen": 2.0107843537435923,
      "rewards/margins": 2.7288761472172522,
      "rewards/rejected": -0.71809179347366,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 9.16874885559082,
      "kl": 15.83648681640625,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -43639409.24610592,
      "logits/rejected": -43527915.93730407,
      "logps/chosen": -496.9823792834891,
      "logps/rejected": -396.6072443181818,
      "loss": 0.3452,
      "loss/base_kto": 2.7614870071411133,
      "metrics/advantage_var": 9076.080078125,
      "rewards/chosen": 1.3471149195020444,
      "rewards/margins": 2.7789011487462645,
      "rewards/rejected": -1.4317862292442203,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 7.9212493896484375,
      "kl": 20.031747817993164,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -44447096.73905997,
      "logits/rejected": -45098418.00301659,
      "logps/chosen": -451.9254457050243,
      "logps/rejected": -391.1688584087481,
      "loss": 0.334,
      "loss/base_kto": 2.671661615371704,
      "metrics/advantage_var": 9060.0986328125,
      "rewards/chosen": 1.5117804776134522,
      "rewards/margins": 2.875957186082297,
      "rewards/rejected": -1.3641767084688443,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 10.213101387023926,
      "kl": 25.621938705444336,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -45655890.23030303,
      "logits/rejected": -44756073.70322581,
      "logps/chosen": -461.4982481060606,
      "logps/rejected": -408.54506048387094,
      "loss": 0.3187,
      "loss/base_kto": 2.5492563247680664,
      "metrics/advantage_var": 8461.5302734375,
      "rewards/chosen": 2.110018273555871,
      "rewards/margins": 3.026513418307635,
      "rewards/rejected": -0.9164951447517641,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 7.825594425201416,
      "kl": 38.75930404663086,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -46050952.25123967,
      "logits/rejected": -45506256.59259259,
      "logps/chosen": -464.1485020661157,
      "logps/rejected": -389.2406481481481,
      "loss": 0.3459,
      "loss/base_kto": 2.76682186126709,
      "metrics/advantage_var": 9444.908203125,
      "rewards/chosen": 1.8575812725981404,
      "rewards/margins": 2.7491066089696683,
      "rewards/rejected": -0.8915253363715278,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 7.18243932723999,
      "kl": 30.665624618530273,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -46040105.6,
      "logits/rejected": -45821132.8,
      "logps/chosen": -493.368310546875,
      "logps/rejected": -386.420751953125,
      "loss": 0.3471,
      "loss/base_kto": 2.7768948078155518,
      "metrics/advantage_var": 10165.1181640625,
      "rewards/chosen": 1.8238739013671874,
      "rewards/margins": 2.9732950210571287,
      "rewards/rejected": -1.1494211196899413,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 7.917231559753418,
      "kl": 41.2276496887207,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -47402153.579617836,
      "logits/rejected": -46038845.25153374,
      "logps/chosen": -491.7269108280255,
      "logps/rejected": -406.0648006134969,
      "loss": 0.31,
      "loss/base_kto": 2.479668140411377,
      "metrics/advantage_var": 11272.2255859375,
      "rewards/chosen": 2.552598698124005,
      "rewards/margins": 3.6254450932304083,
      "rewards/rejected": -1.0728463951064033,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 7.153599739074707,
      "kl": 17.412412643432617,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -46631119.31288344,
      "logits/rejected": -47223462.318471335,
      "logps/chosen": -476.2435774539877,
      "logps/rejected": -383.86176353503186,
      "loss": 0.3234,
      "loss/base_kto": 2.587484836578369,
      "metrics/advantage_var": 10399.548828125,
      "rewards/chosen": 1.5981022185343174,
      "rewards/margins": 3.244075375505854,
      "rewards/rejected": -1.6459731569715366,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 5.476096153259277,
      "kl": 27.94318199157715,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -44313476.15339233,
      "logits/rejected": -45888906.63122924,
      "logps/chosen": -489.6357853982301,
      "logps/rejected": -359.63382475083057,
      "loss": 0.3029,
      "loss/base_kto": 2.423074245452881,
      "metrics/advantage_var": 9790.6162109375,
      "rewards/chosen": 2.436796025188975,
      "rewards/margins": 3.40802311976954,
      "rewards/rejected": -0.9712270945805648,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 7.021985054016113,
      "kl": 13.960652351379395,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -44421100.587677725,
      "logits/rejected": -46085900.26584235,
      "logps/chosen": -475.8004541864139,
      "logps/rejected": -367.6319551777434,
      "loss": 0.3452,
      "loss/base_kto": 2.7619121074676514,
      "metrics/advantage_var": 10958.1064453125,
      "rewards/chosen": 1.160645495470478,
      "rewards/margins": 3.2594266035896817,
      "rewards/rejected": -2.098781108119204,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 7.780029773712158,
      "kl": 52.58126449584961,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -46219425.00628931,
      "logits/rejected": -47191268.9689441,
      "logps/chosen": -455.8256682389937,
      "logps/rejected": -392.7049689440994,
      "loss": 0.319,
      "loss/base_kto": 2.5518600940704346,
      "metrics/advantage_var": 9555.4716796875,
      "rewards/chosen": 2.634859097078911,
      "rewards/margins": 3.1861237000243694,
      "rewards/rejected": -0.5512646029454581,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 8.380097389221191,
      "kl": 22.55097007751465,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -46950616.92211838,
      "logits/rejected": -47377892.71473354,
      "logps/chosen": -465.6770833333333,
      "logps/rejected": -380.56700626959247,
      "loss": 0.3297,
      "loss/base_kto": 2.6379048824310303,
      "metrics/advantage_var": 11059.1005859375,
      "rewards/chosen": 1.678995007666472,
      "rewards/margins": 3.1943280338469187,
      "rewards/rejected": -1.5153330261804467,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 8.63371753692627,
      "kl": 23.489225387573242,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -47105205.8624,
      "logits/rejected": -46306636.995419845,
      "logps/chosen": -475.60705,
      "logps/rejected": -378.4205629770992,
      "loss": 0.3171,
      "loss/base_kto": 2.5367319583892822,
      "metrics/advantage_var": 11255.9462890625,
      "rewards/chosen": 2.1496705078125,
      "rewards/margins": 3.5312632365219465,
      "rewards/rejected": -1.3815927287094465,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 6.207841396331787,
      "kl": 15.559234619140625,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -47738959.87519501,
      "logits/rejected": -46858073.33959311,
      "logps/chosen": -453.6013553042122,
      "logps/rejected": -410.0959996087637,
      "loss": 0.3377,
      "loss/base_kto": 2.701637029647827,
      "metrics/advantage_var": 13703.7685546875,
      "rewards/chosen": 1.087391012730353,
      "rewards/margins": 3.5123001954743867,
      "rewards/rejected": -2.4249091827440337,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 8.613212585449219,
      "kl": 41.37206268310547,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -46806706.08695652,
      "logits/rejected": -46441462.33962264,
      "logps/chosen": -452.17434976708074,
      "logps/rejected": -389.0125786163522,
      "loss": 0.3111,
      "loss/base_kto": 2.4888522624969482,
      "metrics/advantage_var": 11084.3515625,
      "rewards/chosen": 2.6062068583802405,
      "rewards/margins": 3.4839085277493442,
      "rewards/rejected": -0.8777016693691038,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 9.352055549621582,
      "kl": 18.29364585876465,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -48808864.51446945,
      "logits/rejected": -48817167.56231003,
      "logps/chosen": -473.30335610932474,
      "logps/rejected": -386.868944718845,
      "loss": 0.3207,
      "loss/base_kto": 2.5655012130737305,
      "metrics/advantage_var": 14518.6845703125,
      "rewards/chosen": 1.4366721984274518,
      "rewards/margins": 3.508463641484063,
      "rewards/rejected": -2.071791443056611,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 5.842999458312988,
      "kl": 26.263547897338867,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -49060517.43950995,
      "logits/rejected": -48573222.78787879,
      "logps/chosen": -463.4508996937213,
      "logps/rejected": -388.22393341307816,
      "loss": 0.3168,
      "loss/base_kto": 2.5340232849121094,
      "metrics/advantage_var": 13444.134765625,
      "rewards/chosen": 1.9039039319726263,
      "rewards/margins": 3.6761795250746996,
      "rewards/rejected": -1.7722755931020733,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 8.439847946166992,
      "kl": 37.183834075927734,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -46772820.80981595,
      "logits/rejected": -46420320.20382166,
      "logps/chosen": -445.3078029141104,
      "logps/rejected": -388.06966560509557,
      "loss": 0.3295,
      "loss/base_kto": 2.6357457637786865,
      "metrics/advantage_var": 11617.8359375,
      "rewards/chosen": 2.5190156339867715,
      "rewards/margins": 3.17519050631833,
      "rewards/rejected": -0.6561748723315585,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 8.572729110717773,
      "kl": 18.840547561645508,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -47739316.08832808,
      "logits/rejected": -48534800.643962845,
      "logps/chosen": -481.36883872239747,
      "logps/rejected": -363.2466137770898,
      "loss": 0.3124,
      "loss/base_kto": 2.4990785121917725,
      "metrics/advantage_var": 12671.8056640625,
      "rewards/chosen": 2.161652092677938,
      "rewards/margins": 3.838208848192644,
      "rewards/rejected": -1.6765567555147058,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 7.6415181159973145,
      "kl": 23.328641891479492,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -46437382.28220859,
      "logits/rejected": -46758641.32484076,
      "logps/chosen": -447.5244440184049,
      "logps/rejected": -387.3877886146497,
      "loss": 0.3227,
      "loss/base_kto": 2.5819222927093506,
      "metrics/advantage_var": 14022.7490234375,
      "rewards/chosen": 2.143888134166507,
      "rewards/margins": 3.623439510217711,
      "rewards/rejected": -1.4795513760512042,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 6.992530822753906,
      "kl": 26.622074127197266,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -46215003.104669884,
      "logits/rejected": -46835125.96651446,
      "logps/chosen": -422.12354066022544,
      "logps/rejected": -368.4546708523592,
      "loss": 0.3012,
      "loss/base_kto": 2.409684419631958,
      "metrics/advantage_var": 14463.0234375,
      "rewards/chosen": 2.253114463818438,
      "rewards/margins": 4.25722249214321,
      "rewards/rejected": -2.004108028324772,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 6.378375053405762,
      "kl": 24.94471549987793,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -49503190.90850722,
      "logits/rejected": -48349445.84474886,
      "logps/chosen": -458.17962479935795,
      "logps/rejected": -394.6609589041096,
      "loss": 0.2523,
      "loss/base_kto": 2.0187790393829346,
      "metrics/advantage_var": 16326.5107421875,
      "rewards/chosen": 2.497149863939607,
      "rewards/margins": 4.905280341310707,
      "rewards/rejected": -2.4081304773711,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 6.994624614715576,
      "kl": 13.280878067016602,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -47839512.1509434,
      "logits/rejected": -48610097.29192547,
      "logps/chosen": -460.30483490566036,
      "logps/rejected": -395.1667798913044,
      "loss": 0.2539,
      "loss/base_kto": 2.0312325954437256,
      "metrics/advantage_var": 14454.4248046875,
      "rewards/chosen": 2.519423766705975,
      "rewards/margins": 5.028234248375715,
      "rewards/rejected": -2.50881048166974,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 9.424582481384277,
      "kl": 10.85019302368164,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -49302809.56067588,
      "logits/rejected": -49959888.78855326,
      "logps/chosen": -469.22825460829495,
      "logps/rejected": -394.01599761526234,
      "loss": 0.2785,
      "loss/base_kto": 2.227801561355591,
      "metrics/advantage_var": 16311.119140625,
      "rewards/chosen": 1.9709813928091398,
      "rewards/margins": 4.6200011041565165,
      "rewards/rejected": -2.6490197113473766,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 8.003256797790527,
      "kl": 23.4259033203125,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -49873438.30529595,
      "logits/rejected": -49211558.92163009,
      "logps/chosen": -457.47205996884736,
      "logps/rejected": -405.83365987460814,
      "loss": 0.2568,
      "loss/base_kto": 2.054487943649292,
      "metrics/advantage_var": 13718.9765625,
      "rewards/chosen": 2.581324497115946,
      "rewards/margins": 4.676975196934716,
      "rewards/rejected": -2.0956506998187696,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 7.165423393249512,
      "kl": 24.627613067626953,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -48778415.4517134,
      "logits/rejected": -50606927.448275864,
      "logps/chosen": -446.71582943925233,
      "logps/rejected": -396.57908013322884,
      "loss": 0.2638,
      "loss/base_kto": 2.110078811645508,
      "metrics/advantage_var": 14859.9375,
      "rewards/chosen": 2.3450887804833527,
      "rewards/margins": 4.554141652617953,
      "rewards/rejected": -2.2090528721346003,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 7.8962507247924805,
      "kl": 24.107492446899414,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -48750660.1633888,
      "logits/rejected": -48364727.62520194,
      "logps/chosen": -463.73903177004536,
      "logps/rejected": -390.83587439418415,
      "loss": 0.2409,
      "loss/base_kto": 1.9271656274795532,
      "metrics/advantage_var": 15279.6611328125,
      "rewards/chosen": 2.626690701529406,
      "rewards/margins": 5.226181753124721,
      "rewards/rejected": -2.599491051595315,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 6.696859836578369,
      "kl": 24.49211883544922,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -49706159.587253414,
      "logits/rejected": -49836104.55394525,
      "logps/chosen": -460.4776176024279,
      "logps/rejected": -393.373817431562,
      "loss": 0.2466,
      "loss/base_kto": 1.9727410078048706,
      "metrics/advantage_var": 15380.7998046875,
      "rewards/chosen": 2.8233566631496587,
      "rewards/margins": 5.1379878810693445,
      "rewards/rejected": -2.314631217919686,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 7.559670448303223,
      "kl": 13.5224609375,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -51078631.85691057,
      "logits/rejected": -51562779.33233083,
      "logps/chosen": -487.75752032520325,
      "logps/rejected": -395.12109962406015,
      "loss": 0.2634,
      "loss/base_kto": 2.107135534286499,
      "metrics/advantage_var": 14195.0283203125,
      "rewards/chosen": 2.105270658663618,
      "rewards/margins": 4.8840208789408734,
      "rewards/rejected": -2.7787502202772556,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 7.225378513336182,
      "kl": 26.71965980529785,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -50135112.13476264,
      "logits/rejected": -49574805.84370016,
      "logps/chosen": -444.1115045941807,
      "logps/rejected": -381.9628688197767,
      "loss": 0.2421,
      "loss/base_kto": 1.937116265296936,
      "metrics/advantage_var": 13962.541015625,
      "rewards/chosen": 2.7284518792472245,
      "rewards/margins": 4.976747956799059,
      "rewards/rejected": -2.248296077551834,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 6.193775653839111,
      "kl": 31.8278751373291,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -49843278.52760736,
      "logits/rejected": -49576558.87898089,
      "logps/chosen": -470.5159125766871,
      "logps/rejected": -402.9140376194268,
      "loss": 0.2442,
      "loss/base_kto": 1.9534603357315063,
      "metrics/advantage_var": 14674.25,
      "rewards/chosen": 3.012382647742523,
      "rewards/margins": 5.103156114788204,
      "rewards/rejected": -2.090773467045681,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 6.882968425750732,
      "kl": 24.6142578125,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -49777710.618524335,
      "logits/rejected": -50704320.29860031,
      "logps/chosen": -469.02403846153845,
      "logps/rejected": -415.6784603421462,
      "loss": 0.2486,
      "loss/base_kto": 1.9886268377304077,
      "metrics/advantage_var": 16992.470703125,
      "rewards/chosen": 2.6783355281716053,
      "rewards/margins": 5.319612076091512,
      "rewards/rejected": -2.6412765479199067,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 8.8514986038208,
      "kl": 23.941320419311523,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -52166851.35771065,
      "logits/rejected": -50921553.794162825,
      "logps/chosen": -480.11466613672496,
      "logps/rejected": -391.50662442396316,
      "loss": 0.2476,
      "loss/base_kto": 1.980577826499939,
      "metrics/advantage_var": 14342.166015625,
      "rewards/chosen": 2.738708204988076,
      "rewards/margins": 5.102693000014957,
      "rewards/rejected": -2.3639847950268815,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.558167457580566,
      "kl": 17.936880111694336,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -50118090.70031546,
      "logits/rejected": -51773455.851393186,
      "logps/chosen": -455.5176458990536,
      "logps/rejected": -408.96613777089783,
      "loss": 0.2435,
      "loss/base_kto": 1.9483833312988281,
      "metrics/advantage_var": 14109.5107421875,
      "rewards/chosen": 2.2959194243518337,
      "rewards/margins": 5.004700852733219,
      "rewards/rejected": -2.7087814283813856,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 9.361747741699219,
      "kl": 17.557247161865234,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -49804216.25477707,
      "logits/rejected": -50424656.09815951,
      "logps/chosen": -454.2070561305732,
      "logps/rejected": -388.4324674079755,
      "loss": 0.242,
      "loss/base_kto": 1.935681939125061,
      "metrics/advantage_var": 14550.7333984375,
      "rewards/chosen": 2.3316926409484475,
      "rewards/margins": 5.0811738421203225,
      "rewards/rejected": -2.749481201171875,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 9.169776916503906,
      "kl": 20.535934448242188,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -48851373.86506747,
      "logits/rejected": -50709555.78466558,
      "logps/chosen": -436.7430659670165,
      "logps/rejected": -393.1366231647635,
      "loss": 0.2555,
      "loss/base_kto": 2.0440099239349365,
      "metrics/advantage_var": 14528.4658203125,
      "rewards/chosen": 2.7239223405875186,
      "rewards/margins": 4.791540531732503,
      "rewards/rejected": -2.067618191144984,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 8.66065502166748,
      "kl": 25.7312068939209,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -50435637.02009273,
      "logits/rejected": -50470297.276461296,
      "logps/chosen": -479.2921174652241,
      "logps/rejected": -413.2000394944708,
      "loss": 0.2505,
      "loss/base_kto": 2.0043818950653076,
      "metrics/advantage_var": 14897.306640625,
      "rewards/chosen": 2.8491875060374805,
      "rewards/margins": 4.878784832138389,
      "rewards/rejected": -2.0295973261009084,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 8.729217529296875,
      "kl": 12.896934509277344,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -51230771.2,
      "logits/rejected": -48302675.2,
      "logps/chosen": -491.442236328125,
      "logps/rejected": -411.22578125,
      "loss": 0.2505,
      "loss/base_kto": 2.0038585662841797,
      "metrics/advantage_var": 17345.353515625,
      "rewards/chosen": 2.20242862701416,
      "rewards/margins": 5.342057228088379,
      "rewards/rejected": -3.139628601074219,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 7.272515296936035,
      "kl": 20.829553604125977,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -50125110.885758996,
      "logits/rejected": -50515789.079563186,
      "logps/chosen": -442.1692097026604,
      "logps/rejected": -390.7211875975039,
      "loss": 0.2598,
      "loss/base_kto": 2.0783112049102783,
      "metrics/advantage_var": 14066.3505859375,
      "rewards/chosen": 2.8817564676251957,
      "rewards/margins": 4.72465825986876,
      "rewards/rejected": -1.8429017922435647,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 9.126511573791504,
      "kl": 18.096542358398438,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -50236237.91304348,
      "logits/rejected": -50608037.83647799,
      "logps/chosen": -446.0045613354037,
      "logps/rejected": -405.5376130110063,
      "loss": 0.2529,
      "loss/base_kto": 2.023378610610962,
      "metrics/advantage_var": 14046.7861328125,
      "rewards/chosen": 2.22546064483453,
      "rewards/margins": 4.758044205418257,
      "rewards/rejected": -2.5325835605837264,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 7.987075328826904,
      "kl": 33.31155014038086,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -50799396.45696401,
      "logits/rejected": -50986567.08892356,
      "logps/chosen": -466.1683294209703,
      "logps/rejected": -393.43584243369736,
      "loss": 0.2449,
      "loss/base_kto": 1.9588383436203003,
      "metrics/advantage_var": 14299.556640625,
      "rewards/chosen": 2.978382474753032,
      "rewards/margins": 4.966008049111943,
      "rewards/rejected": -1.987625574358912,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 9.176545143127441,
      "kl": 10.651198387145996,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -51713834.40490798,
      "logits/rejected": -51210419.36305732,
      "logps/chosen": -470.2095475460123,
      "logps/rejected": -412.53739550159236,
      "loss": 0.2613,
      "loss/base_kto": 2.0903689861297607,
      "metrics/advantage_var": 14415.9375,
      "rewards/chosen": 2.202017005967216,
      "rewards/margins": 4.79738327133097,
      "rewards/rejected": -2.595366265363754,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 6.654029846191406,
      "kl": 19.941139221191406,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -50187843.47286822,
      "logits/rejected": -51112687.47086614,
      "logps/chosen": -458.1766472868217,
      "logps/rejected": -392.57022637795274,
      "loss": 0.2557,
      "loss/base_kto": 2.0455853939056396,
      "metrics/advantage_var": 14611.9013671875,
      "rewards/chosen": 2.500072674418605,
      "rewards/margins": 4.968738666975199,
      "rewards/rejected": -2.4686659925565944,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 8.606330871582031,
      "kl": 11.365921974182129,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -51410057.46003017,
      "logits/rejected": -51158256.64829822,
      "logps/chosen": -470.796427224736,
      "logps/rejected": -382.7708670988655,
      "loss": 0.2686,
      "loss/base_kto": 2.148906707763672,
      "metrics/advantage_var": 14091.3740234375,
      "rewards/chosen": 2.4390082956259427,
      "rewards/margins": 4.72086251399608,
      "rewards/rejected": -2.2818542183701376,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 7.486184597015381,
      "kl": 27.97551918029785,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -50427826.59527559,
      "logits/rejected": -49912639.900775194,
      "logps/chosen": -449.01761811023624,
      "logps/rejected": -401.06477713178293,
      "loss": 0.2628,
      "loss/base_kto": 2.102313756942749,
      "metrics/advantage_var": 15452.3583984375,
      "rewards/chosen": 2.2096102977362206,
      "rewards/margins": 4.773240347094264,
      "rewards/rejected": -2.563630049358043,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 8.14769458770752,
      "kl": 21.685243606567383,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -51391586.368503936,
      "logits/rejected": -52351553.88527132,
      "logps/chosen": -441.48917322834643,
      "logps/rejected": -404.3329215116279,
      "loss": 0.2538,
      "loss/base_kto": 2.030576467514038,
      "metrics/advantage_var": 14459.845703125,
      "rewards/chosen": 2.7408108929010826,
      "rewards/margins": 4.912655233380734,
      "rewards/rejected": -2.1718443404796512,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 8.10610294342041,
      "kl": 17.0273494720459,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -50129007.825273015,
      "logits/rejected": -52257966.67292645,
      "logps/chosen": -470.134994149766,
      "logps/rejected": -402.0153560250391,
      "loss": 0.263,
      "loss/base_kto": 2.10416316986084,
      "metrics/advantage_var": 14936.8193359375,
      "rewards/chosen": 2.101620392940718,
      "rewards/margins": 4.83664681667409,
      "rewards/rejected": -2.7350264237333723,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 7.606040954589844,
      "kl": 19.883148193359375,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -50607266.01536098,
      "logits/rejected": -51428334.09220985,
      "logps/chosen": -476.41114631336404,
      "logps/rejected": -367.1104680047695,
      "loss": 0.2579,
      "loss/base_kto": 2.0632483959198,
      "metrics/advantage_var": 14255.6953125,
      "rewards/chosen": 2.9189132479478688,
      "rewards/margins": 4.890715199830818,
      "rewards/rejected": -1.971801951882949,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 6.019439697265625,
      "kl": 19.90534782409668,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -51477607.19379845,
      "logits/rejected": -52088490.12913386,
      "logps/chosen": -473.96148255813955,
      "logps/rejected": -417.79862204724407,
      "loss": 0.2387,
      "loss/base_kto": 1.9098351001739502,
      "metrics/advantage_var": 15776.4912109375,
      "rewards/chosen": 2.769668460816376,
      "rewards/margins": 5.314328701650529,
      "rewards/rejected": -2.5446602408341534,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 5.279232978820801,
      "kl": 18.785978317260742,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -49837756.294209704,
      "logits/rejected": -50480509.80343214,
      "logps/chosen": -462.4819053208138,
      "logps/rejected": -391.620027301092,
      "loss": 0.2391,
      "loss/base_kto": 1.912419319152832,
      "metrics/advantage_var": 15797.1923828125,
      "rewards/chosen": 2.4113536470559467,
      "rewards/margins": 5.437673606104308,
      "rewards/rejected": -3.026319959048362,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 6.649168968200684,
      "kl": 25.740583419799805,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -50990672.5129771,
      "logits/rejected": -52533192.2944,
      "logps/chosen": -483.17099236641224,
      "logps/rejected": -405.1268,
      "loss": 0.2562,
      "loss/base_kto": 2.0497548580169678,
      "metrics/advantage_var": 14378.15625,
      "rewards/chosen": 2.62320155951813,
      "rewards/margins": 4.86190702826813,
      "rewards/rejected": -2.23870546875,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.85274600982666,
      "kl": 27.409530639648438,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -52023243.90461049,
      "logits/rejected": -52485766.488479264,
      "logps/chosen": -450.50407392686805,
      "logps/rejected": -399.8531105990783,
      "loss": 0.2642,
      "loss/base_kto": 2.113422155380249,
      "metrics/advantage_var": 14363.2626953125,
      "rewards/chosen": 2.7098499292031,
      "rewards/margins": 4.678663683283361,
      "rewards/rejected": -1.968813754080261,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 6.2643303871154785,
      "kl": 16.370676040649414,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -52536850.428794995,
      "logits/rejected": -51779555.2449298,
      "logps/chosen": -460.3179773082942,
      "logps/rejected": -398.3285150156006,
      "loss": 0.2603,
      "loss/base_kto": 2.082447052001953,
      "metrics/advantage_var": 14686.6943359375,
      "rewards/chosen": 2.232792097637911,
      "rewards/margins": 4.775106505057957,
      "rewards/rejected": -2.5423144074200468,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 8.45455551147461,
      "kl": 12.755212783813477,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -51945536.0,
      "logits/rejected": -51340521.6,
      "logps/chosen": -495.90009765625,
      "logps/rejected": -382.887548828125,
      "loss": 0.2633,
      "loss/base_kto": 2.1066510677337646,
      "metrics/advantage_var": 15828.8095703125,
      "rewards/chosen": 2.346480941772461,
      "rewards/margins": 4.982112312316895,
      "rewards/rejected": -2.6356313705444334,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 7.647529125213623,
      "kl": 26.12347984313965,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -50083490.34146342,
      "logits/rejected": -50876591.54285714,
      "logps/chosen": -425.8041666666667,
      "logps/rejected": -411.9215695488722,
      "loss": 0.2553,
      "loss/base_kto": 2.0427498817443848,
      "metrics/advantage_var": 15325.8642578125,
      "rewards/chosen": 2.465984859311484,
      "rewards/margins": 4.981041911120695,
      "rewards/rejected": -2.5150570518092104,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 7.476153373718262,
      "kl": 16.00942039489746,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -52222755.34006734,
      "logits/rejected": -51787650.6122449,
      "logps/chosen": -454.8621632996633,
      "logps/rejected": -391.1714194606414,
      "loss": 0.257,
      "loss/base_kto": 2.055962324142456,
      "metrics/advantage_var": 15469.765625,
      "rewards/chosen": 2.2594590106797137,
      "rewards/margins": 5.041733262456317,
      "rewards/rejected": -2.7822742517766037,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 8.433756828308105,
      "kl": 17.34130859375,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -51585524.34597876,
      "logits/rejected": -50295051.13043478,
      "logps/chosen": -434.56885432473445,
      "logps/rejected": -388.7183977455717,
      "loss": 0.2523,
      "loss/base_kto": 2.018437147140503,
      "metrics/advantage_var": 15157.1318359375,
      "rewards/chosen": 2.2025817037533195,
      "rewards/margins": 5.203248077923005,
      "rewards/rejected": -3.000666374169686,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 7.5897417068481445,
      "kl": 27.72690200805664,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -51326822.87850467,
      "logits/rejected": -51313021.99373041,
      "logps/chosen": -441.95570482866043,
      "logps/rejected": -380.2719190830721,
      "loss": 0.2653,
      "loss/base_kto": 2.1222727298736572,
      "metrics/advantage_var": 14398.7373046875,
      "rewards/chosen": 2.7608041733596185,
      "rewards/margins": 4.658044886831895,
      "rewards/rejected": -1.8972407134722766,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 8.917332649230957,
      "kl": 15.3670654296875,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -52705762.47775947,
      "logits/rejected": -52573042.49628529,
      "logps/chosen": -477.01328253706754,
      "logps/rejected": -394.1614970282318,
      "loss": 0.2483,
      "loss/base_kto": 1.9862979650497437,
      "metrics/advantage_var": 14811.9443359375,
      "rewards/chosen": 2.8682984001750413,
      "rewards/margins": 5.139281253014473,
      "rewards/rejected": -2.2709828528394316,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 6.401049613952637,
      "kl": 20.232099533081055,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -51152623.58974359,
      "logits/rejected": -52514246.06727829,
      "logps/chosen": -471.6942608173077,
      "logps/rejected": -404.40089831804283,
      "loss": 0.2116,
      "loss/base_kto": 1.6930650472640991,
      "metrics/advantage_var": 16739.052734375,
      "rewards/chosen": 2.6238733927408853,
      "rewards/margins": 5.894996783055296,
      "rewards/rejected": -3.2711233903144112,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 6.078398704528809,
      "kl": 18.59099006652832,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -50931055.8755832,
      "logits/rejected": -51348603.78021978,
      "logps/chosen": -450.2767301710731,
      "logps/rejected": -388.2606456043956,
      "loss": 0.2076,
      "loss/base_kto": 1.660752534866333,
      "metrics/advantage_var": 14589.8154296875,
      "rewards/chosen": 2.7858607646651437,
      "rewards/margins": 5.617628710725779,
      "rewards/rejected": -2.8317679460606358,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 11.037747383117676,
      "kl": 21.137937545776367,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -53843410.67540984,
      "logits/rejected": -53166801.38507463,
      "logps/chosen": -465.16198770491803,
      "logps/rejected": -406.10074626865674,
      "loss": 0.2025,
      "loss/base_kto": 1.6196577548980713,
      "metrics/advantage_var": 16674.591796875,
      "rewards/chosen": 2.7636190445696722,
      "rewards/margins": 6.006351670502508,
      "rewards/rejected": -3.242732625932836,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 7.611783027648926,
      "kl": 13.244036674499512,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -52446179.64307693,
      "logits/rejected": -51852775.61904762,
      "logps/chosen": -444.28432692307695,
      "logps/rejected": -401.10828373015875,
      "loss": 0.2162,
      "loss/base_kto": 1.7293108701705933,
      "metrics/advantage_var": 15236.3466796875,
      "rewards/chosen": 2.6056948617788462,
      "rewards/margins": 5.519433003519918,
      "rewards/rejected": -2.9137381417410713,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 8.240866661071777,
      "kl": 18.3961238861084,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -49095924.111455105,
      "logits/rejected": -49097731.23028391,
      "logps/chosen": -460.3483939628483,
      "logps/rejected": -383.85639294164037,
      "loss": 0.2125,
      "loss/base_kto": 1.6999447345733643,
      "metrics/advantage_var": 14634.9404296875,
      "rewards/chosen": 2.5562453137093653,
      "rewards/margins": 5.478640556586928,
      "rewards/rejected": -2.9223952428775632,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 6.910224914550781,
      "kl": 18.553775787353516,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -52505239.25434439,
      "logits/rejected": -52949324.364760436,
      "logps/chosen": -453.1098933649289,
      "logps/rejected": -390.14055255023186,
      "loss": 0.2146,
      "loss/base_kto": 1.7165441513061523,
      "metrics/advantage_var": 17012.947265625,
      "rewards/chosen": 2.5390918123025275,
      "rewards/margins": 5.7191121284250155,
      "rewards/rejected": -3.1800203161224885,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 5.961915969848633,
      "kl": 17.614856719970703,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -50447468.27794562,
      "logits/rejected": -51201342.13592233,
      "logps/chosen": -427.4541163141994,
      "logps/rejected": -407.8164694579288,
      "loss": 0.2066,
      "loss/base_kto": 1.653010606765747,
      "metrics/advantage_var": 14973.6689453125,
      "rewards/chosen": 2.656560892064766,
      "rewards/margins": 5.582759314394863,
      "rewards/rejected": -2.926198422330097,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 3.847012519836426,
      "kl": 23.62422752380371,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51482255.92626728,
      "logits/rejected": -52466816.61049285,
      "logps/chosen": -459.6874039938556,
      "logps/rejected": -410.0303557233704,
      "loss": 0.2116,
      "loss/base_kto": 1.6924909353256226,
      "metrics/advantage_var": 15904.1904296875,
      "rewards/chosen": 2.9562669510848694,
      "rewards/margins": 5.72156319574405,
      "rewards/rejected": -2.765296244659181,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 9.123897552490234,
      "kl": 23.464460372924805,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -51229029.1712,
      "logits/rejected": -52494498.58931298,
      "logps/chosen": -448.3186,
      "logps/rejected": -407.9988072519084,
      "loss": 0.2057,
      "loss/base_kto": 1.6455026865005493,
      "metrics/advantage_var": 17421.048828125,
      "rewards/chosen": 2.7009166015625,
      "rewards/margins": 5.928064666328721,
      "rewards/rejected": -3.2271480647662214,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 9.547018051147461,
      "kl": 24.59076499938965,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -51683574.138674885,
      "logits/rejected": -52181183.49286846,
      "logps/chosen": -486.013000770416,
      "logps/rejected": -394.2410608161648,
      "loss": 0.2184,
      "loss/base_kto": 1.74701726436615,
      "metrics/advantage_var": 14838.837890625,
      "rewards/chosen": 2.7930456787485554,
      "rewards/margins": 5.485414949375537,
      "rewards/rejected": -2.692369270626981,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 6.961014270782471,
      "kl": 26.59592628479004,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -52061777.361155696,
      "logits/rejected": -51680260.67579909,
      "logps/chosen": -425.9284209470305,
      "logps/rejected": -415.11115867579906,
      "loss": 0.2018,
      "loss/base_kto": 1.614264726638794,
      "metrics/advantage_var": 14626.306640625,
      "rewards/chosen": 2.8435554320701244,
      "rewards/margins": 5.742419156266282,
      "rewards/rejected": -2.898863724196157,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.428321838378906,
      "kl": 20.4986629486084,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -52222980.82260597,
      "logits/rejected": -51456238.88024884,
      "logps/chosen": -455.01800431711143,
      "logps/rejected": -404.5633262052877,
      "loss": 0.2038,
      "loss/base_kto": 1.630610704421997,
      "metrics/advantage_var": 15116.5751953125,
      "rewards/chosen": 2.7454387479150313,
      "rewards/margins": 5.538942860716354,
      "rewards/rejected": -2.793504112801322,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 6.948566436767578,
      "kl": 19.01148796081543,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -52328898.00896861,
      "logits/rejected": -53048723.90180033,
      "logps/chosen": -458.93189461883406,
      "logps/rejected": -408.41865282324056,
      "loss": 0.2116,
      "loss/base_kto": 1.6924980878829956,
      "metrics/advantage_var": 14713.4814453125,
      "rewards/chosen": 2.7260599863368835,
      "rewards/margins": 5.545678024133528,
      "rewards/rejected": -2.8196180377966447,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 6.769558429718018,
      "kl": 19.767709732055664,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -51488283.80830671,
      "logits/rejected": -52025055.90214067,
      "logps/chosen": -455.0748801916933,
      "logps/rejected": -372.0299120795107,
      "loss": 0.1975,
      "loss/base_kto": 1.5798718929290771,
      "metrics/advantage_var": 14985.7646484375,
      "rewards/chosen": 2.8366749918879792,
      "rewards/margins": 5.937662977481443,
      "rewards/rejected": -3.1009879855934632,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 10.205763816833496,
      "kl": 20.990039825439453,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -52205237.57188498,
      "logits/rejected": -52282449.41896024,
      "logps/chosen": -481.22698682108626,
      "logps/rejected": -379.97902331804283,
      "loss": 0.2054,
      "loss/base_kto": 1.643288016319275,
      "metrics/advantage_var": 16388.818359375,
      "rewards/chosen": 2.6462098142971247,
      "rewards/margins": 5.959112235694295,
      "rewards/rejected": -3.312902421397171,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 9.473999977111816,
      "kl": 18.7738094329834,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -50897698.50783699,
      "logits/rejected": -51462290.741433024,
      "logps/chosen": -466.38964537617557,
      "logps/rejected": -401.3882398753894,
      "loss": 0.215,
      "loss/base_kto": 1.71996009349823,
      "metrics/advantage_var": 14777.6533203125,
      "rewards/chosen": 2.3673109096419473,
      "rewards/margins": 5.703103142167259,
      "rewards/rejected": -3.3357922325253115,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 8.424354553222656,
      "kl": 17.50851821899414,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -52425362.86529318,
      "logits/rejected": -53291522.366718024,
      "logps/chosen": -464.77867472266246,
      "logps/rejected": -428.4358628659476,
      "loss": 0.2224,
      "loss/base_kto": 1.7790508270263672,
      "metrics/advantage_var": 15547.15625,
      "rewards/chosen": 2.810249727614897,
      "rewards/margins": 5.644079116578688,
      "rewards/rejected": -2.8338293889637907,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 8.850090026855469,
      "kl": 23.67220115661621,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -51791914.73515249,
      "logits/rejected": -51904091.17808219,
      "logps/chosen": -465.92596308186194,
      "logps/rejected": -405.92142313546424,
      "loss": 0.1973,
      "loss/base_kto": 1.5781967639923096,
      "metrics/advantage_var": 16292.103515625,
      "rewards/chosen": 2.8144715433136036,
      "rewards/margins": 5.905784967852226,
      "rewards/rejected": -3.0913134245386225,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 5.788835525512695,
      "kl": 12.649649620056152,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -51609774.5631659,
      "logits/rejected": -51973779.10754414,
      "logps/chosen": -453.40577435312025,
      "logps/rejected": -421.79268659711073,
      "loss": 0.1977,
      "loss/base_kto": 1.581206202507019,
      "metrics/advantage_var": 15676.1904296875,
      "rewards/chosen": 2.6842882687642695,
      "rewards/margins": 5.96072932545819,
      "rewards/rejected": -3.2764410566939204,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 5.196254253387451,
      "kl": 18.2519588470459,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -53048049.13548387,
      "logits/rejected": -52975417.406060606,
      "logps/chosen": -464.4367943548387,
      "logps/rejected": -399.4305397727273,
      "loss": 0.2091,
      "loss/base_kto": 1.673047661781311,
      "metrics/advantage_var": 15030.0849609375,
      "rewards/chosen": 2.5911819950226813,
      "rewards/margins": 5.556486652929879,
      "rewards/rejected": -2.965304657907197,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 8.297974586486816,
      "kl": 15.502278327941895,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -52658809.97875569,
      "logits/rejected": -53108066.52495974,
      "logps/chosen": -468.6451062215478,
      "logps/rejected": -410.24074074074076,
      "loss": 0.1953,
      "loss/base_kto": 1.5620167255401611,
      "metrics/advantage_var": 15958.2158203125,
      "rewards/chosen": 2.9076016647500946,
      "rewards/margins": 5.964484071528476,
      "rewards/rejected": -3.0568824067783815,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 6.340052604675293,
      "kl": 22.518726348876953,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -52333680.330683626,
      "logits/rejected": -52139670.21812596,
      "logps/chosen": -461.2084658187599,
      "logps/rejected": -376.0010560675883,
      "loss": 0.2126,
      "loss/base_kto": 1.700911521911621,
      "metrics/advantage_var": 14260.2021484375,
      "rewards/chosen": 2.755645509365064,
      "rewards/margins": 5.5544372570369145,
      "rewards/rejected": -2.798791747671851,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 7.071959018707275,
      "kl": 17.614599227905273,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -51984199.4954955,
      "logits/rejected": -54268844.61237785,
      "logps/chosen": -461.6692942942943,
      "logps/rejected": -391.4291276465798,
      "loss": 0.2117,
      "loss/base_kto": 1.693826675415039,
      "metrics/advantage_var": 15238.3154296875,
      "rewards/chosen": 2.8153856981981984,
      "rewards/margins": 5.722483231980263,
      "rewards/rejected": -2.907097533782064,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 7.8359904289245605,
      "kl": 17.305866241455078,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -52183382.39009288,
      "logits/rejected": -52238788.239747636,
      "logps/chosen": -453.1547503869969,
      "logps/rejected": -426.9443020504732,
      "loss": 0.2045,
      "loss/base_kto": 1.6361528635025024,
      "metrics/advantage_var": 14747.8564453125,
      "rewards/chosen": 2.805559376814048,
      "rewards/margins": 5.680909414397842,
      "rewards/rejected": -2.8753500375837935,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 6.994919776916504,
      "kl": 12.713762283325195,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -52167258.71471471,
      "logits/rejected": -51606157.758957654,
      "logps/chosen": -454.4685623123123,
      "logps/rejected": -402.00356270358304,
      "loss": 0.2012,
      "loss/base_kto": 1.6097615957260132,
      "metrics/advantage_var": 14512.5908203125,
      "rewards/chosen": 2.91284399634009,
      "rewards/margins": 5.728330742764662,
      "rewards/rejected": -2.8154867464245723,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 10.00118637084961,
      "kl": 20.05970001220703,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -51249053.696,
      "logits/rejected": -52149540.3480916,
      "logps/chosen": -480.2795,
      "logps/rejected": -398.30271946564886,
      "loss": 0.2203,
      "loss/base_kto": 1.7626689672470093,
      "metrics/advantage_var": 15542.7998046875,
      "rewards/chosen": 2.693503125,
      "rewards/margins": 5.515547532502385,
      "rewards/rejected": -2.8220444075023856,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 7.515979766845703,
      "kl": 21.391738891601562,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -52715101.09090909,
      "logits/rejected": -52747603.27710843,
      "logps/chosen": -484.2689732142857,
      "logps/rejected": -397.6544145331325,
      "loss": 0.2142,
      "loss/base_kto": 1.7132612466812134,
      "metrics/advantage_var": 15169.9345703125,
      "rewards/chosen": 2.683006782036323,
      "rewards/margins": 5.657759920567008,
      "rewards/rejected": -2.9747531385306853,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.797512531280518,
      "kl": 18.113786697387695,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -52307566.11616954,
      "logits/rejected": -53364216.833592534,
      "logps/chosen": -457.19368131868134,
      "logps/rejected": -406.29393468118195,
      "loss": 0.1993,
      "loss/base_kto": 1.5946701765060425,
      "metrics/advantage_var": 14731.962890625,
      "rewards/chosen": 2.7622787020457222,
      "rewards/margins": 5.704829527339754,
      "rewards/rejected": -2.942550825294032,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 8.445562362670898,
      "kl": 14.823893547058105,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -51118715.24394184,
      "logits/rejected": -50332592.9924357,
      "logps/chosen": -444.25504846526655,
      "logps/rejected": -415.2663577912254,
      "loss": 0.2066,
      "loss/base_kto": 1.6527931690216064,
      "metrics/advantage_var": 14747.3642578125,
      "rewards/chosen": 2.554538806921446,
      "rewards/margins": 5.7892395739883895,
      "rewards/rejected": -3.234700767066944,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 8.256867408752441,
      "kl": 17.39256477355957,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -50655182.7987988,
      "logits/rejected": -51788112.88599349,
      "logps/chosen": -464.0256193693694,
      "logps/rejected": -379.0589881921824,
      "loss": 0.2149,
      "loss/base_kto": 1.7192754745483398,
      "metrics/advantage_var": 15078.853515625,
      "rewards/chosen": 2.6980321385838963,
      "rewards/margins": 5.56999032135059,
      "rewards/rejected": -2.871958182766694,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 12.323614120483398,
      "kl": 13.718680381774902,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -51934669.990697674,
      "logits/rejected": -51647728.27716535,
      "logps/chosen": -473.34796511627906,
      "logps/rejected": -391.9261318897638,
      "loss": 0.2148,
      "loss/base_kto": 1.7184391021728516,
      "metrics/advantage_var": 16194.2373046875,
      "rewards/chosen": 2.701825944767442,
      "rewards/margins": 5.7044957282320095,
      "rewards/rejected": -3.002669783464567,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 7.657737731933594,
      "kl": 16.89313507080078,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -52261126.896988906,
      "logits/rejected": -51801802.74884438,
      "logps/chosen": -453.9925713153724,
      "logps/rejected": -408.42045454545456,
      "loss": 0.2062,
      "loss/base_kto": 1.6494368314743042,
      "metrics/advantage_var": 15846.103515625,
      "rewards/chosen": 2.789609978580626,
      "rewards/margins": 5.766267087775059,
      "rewards/rejected": -2.976657109194434,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 6.649723052978516,
      "kl": 16.165952682495117,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -52194091.591331266,
      "logits/rejected": -52126716.76971609,
      "logps/chosen": -469.999661377709,
      "logps/rejected": -394.8598186119874,
      "loss": 0.2136,
      "loss/base_kto": 1.7085020542144775,
      "metrics/advantage_var": 14259.7578125,
      "rewards/chosen": 2.6862430159152475,
      "rewards/margins": 5.366490999637152,
      "rewards/rejected": -2.6802479837219044,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 7.129169940948486,
      "kl": 17.871610641479492,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -53591976.04907975,
      "logits/rejected": -54577712.91719745,
      "logps/chosen": -475.78930214723925,
      "logps/rejected": -424.4720342356688,
      "loss": 0.2101,
      "loss/base_kto": 1.6811965703964233,
      "metrics/advantage_var": 15833.6982421875,
      "rewards/chosen": 2.7122051964508245,
      "rewards/margins": 5.6900568211522575,
      "rewards/rejected": -2.977851624701433,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 8.79556655883789,
      "kl": 13.786334037780762,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -50803324.00594354,
      "logits/rejected": -51990264.83031301,
      "logps/chosen": -429.8521545319465,
      "logps/rejected": -396.9826503294893,
      "loss": 0.2138,
      "loss/base_kto": 1.710473895072937,
      "metrics/advantage_var": 14456.3095703125,
      "rewards/chosen": 2.7941658734212482,
      "rewards/margins": 5.510056170219024,
      "rewards/rejected": -2.715890296797776,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 6.54217529296875,
      "kl": 13.671980857849121,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -50993193.66510172,
      "logits/rejected": -51762870.914196566,
      "logps/chosen": -463.6511150234742,
      "logps/rejected": -396.309379875195,
      "loss": 0.1945,
      "loss/base_kto": 1.5562013387680054,
      "metrics/advantage_var": 14981.1943359375,
      "rewards/chosen": 2.8755658407912756,
      "rewards/margins": 5.85812178365399,
      "rewards/rejected": -2.9825559428627146,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.645531177520752,
      "kl": 19.706483840942383,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -51603658.47649919,
      "logits/rejected": -50926791.239819005,
      "logps/chosen": -475.88614262560776,
      "logps/rejected": -404.70046191553547,
      "loss": 0.2135,
      "loss/base_kto": 1.708304762840271,
      "metrics/advantage_var": 15029.6640625,
      "rewards/chosen": 2.6179828365579416,
      "rewards/margins": 5.423700477430396,
      "rewards/rejected": -2.8057176408724547,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 6.593998908996582,
      "kl": 23.683530807495117,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -53067135.21472393,
      "logits/rejected": -52689486.26751592,
      "logps/chosen": -448.24467983128835,
      "logps/rejected": -419.56822253184714,
      "loss": 0.211,
      "loss/base_kto": 1.6882925033569336,
      "metrics/advantage_var": 16097.6591796875,
      "rewards/chosen": 2.571448577693635,
      "rewards/margins": 5.724529997814555,
      "rewards/rejected": -3.1530814201209196,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.98294195064275e+17,
      "train_loss": 0.27245468189473393,
      "train_runtime": 11042.7298,
      "train_samples_per_second": 13.422,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.98294195064275e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}