{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 9.878138542175293,
      "kl": 1.9441527128219604,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36347301.26582278,
      "logits/rejected": -37878834.56790123,
      "logps/chosen": -506.0238825158228,
      "logps/rejected": -381.65121045524694,
      "loss": 0.4676,
      "loss/base_kto": 3.740809679031372,
      "metrics/advantage_var": 518.8466186523438,
      "rewards/chosen": -0.3824396978450727,
      "rewards/margins": 0.3086688173880967,
      "rewards/rejected": -0.6911085152331694,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 9.061779975891113,
      "kl": 36.309051513671875,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37144238.348228045,
      "logits/rejected": -38884373.09667195,
      "logps/chosen": -485.66212442218796,
      "logps/rejected": -357.98058637083994,
      "loss": 0.4366,
      "loss/base_kto": 3.492482900619507,
      "metrics/advantage_var": 762.9915161132812,
      "rewards/chosen": 0.8913009949200693,
      "rewards/margins": 0.49241496874014656,
      "rewards/rejected": 0.39888602617992275,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 9.688044548034668,
      "kl": 15.731460571289062,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36332691.692307696,
      "logits/rejected": -36808822.63414634,
      "logps/chosen": -465.1143329326923,
      "logps/rejected": -365.1838557545732,
      "loss": 0.4249,
      "loss/base_kto": 3.398885726928711,
      "metrics/advantage_var": 1136.6014404296875,
      "rewards/chosen": 0.5513690068171575,
      "rewards/margins": 0.6965122831247985,
      "rewards/rejected": -0.145143276307641,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 9.593290328979492,
      "kl": 7.7284698486328125,
      "learning_rate": 7.9e-07,
      "logits/chosen": -39501343.44566929,
      "logits/rejected": -39199355.03875969,
      "logps/chosen": -503.3968996062992,
      "logps/rejected": -388.6171511627907,
      "loss": 0.4287,
      "loss/base_kto": 3.4294159412384033,
      "metrics/advantage_var": 1320.1468505859375,
      "rewards/chosen": 0.357440521961122,
      "rewards/margins": 0.7577194857450368,
      "rewards/rejected": -0.40027896378391475,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 11.081856727600098,
      "kl": 4.866374492645264,
      "learning_rate": 9.9e-07,
      "logits/chosen": -38675630.06431853,
      "logits/rejected": -39611403.432216905,
      "logps/chosen": -491.76311255742723,
      "logps/rejected": -380.7640051834131,
      "loss": 0.4353,
      "loss/base_kto": 3.482758045196533,
      "metrics/advantage_var": 2065.534423828125,
      "rewards/chosen": -0.05095923804944726,
      "rewards/margins": 0.79693024996391,
      "rewards/rejected": -0.8478894880133573,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 7.145315170288086,
      "kl": 42.45624923706055,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -40589290.1129771,
      "logits/rejected": -39799858.7904,
      "logps/chosen": -479.55500954198476,
      "logps/rejected": -358.55725,
      "loss": 0.4017,
      "loss/base_kto": 3.213905096054077,
      "metrics/advantage_var": 3067.258544921875,
      "rewards/chosen": 1.443365897841126,
      "rewards/margins": 1.0941520306536259,
      "rewards/rejected": 0.3492138671875,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 9.371260643005371,
      "kl": 19.413406372070312,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -39658741.00613497,
      "logits/rejected": -41928746.39490446,
      "logps/chosen": -479.15423696319016,
      "logps/rejected": -389.13002587579615,
      "loss": 0.4023,
      "loss/base_kto": 3.2183144092559814,
      "metrics/advantage_var": 3842.186767578125,
      "rewards/chosen": 0.8319151708684815,
      "rewards/margins": 1.3273419562828674,
      "rewards/rejected": -0.49542678541438595,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 7.865825176239014,
      "kl": 42.96455001831055,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -41660416.0,
      "logits/rejected": -40298753.25285481,
      "logps/chosen": -480.15606259370315,
      "logps/rejected": -371.54175163132135,
      "loss": 0.3747,
      "loss/base_kto": 2.997523069381714,
      "metrics/advantage_var": 4104.15380859375,
      "rewards/chosen": 1.8185332382636807,
      "rewards/margins": 1.4788250500340467,
      "rewards/rejected": 0.33970818822963395,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 7.099153518676758,
      "kl": 48.65797805786133,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -40652091.898658715,
      "logits/rejected": -40906916.78160919,
      "logps/chosen": -451.25363263785397,
      "logps/rejected": -357.00413074712645,
      "loss": 0.3747,
      "loss/base_kto": 2.9979875087738037,
      "metrics/advantage_var": 4341.8828125,
      "rewards/chosen": 1.846562892953614,
      "rewards/margins": 1.5223601283516794,
      "rewards/rejected": 0.32420276460193453,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 6.9005632400512695,
      "kl": 35.48698043823242,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -43210083.51335312,
      "logits/rejected": -43863675.35313531,
      "logps/chosen": -473.6956602373887,
      "logps/rejected": -381.14686468646863,
      "loss": 0.3735,
      "loss/base_kto": 2.988220453262329,
      "metrics/advantage_var": 4227.18359375,
      "rewards/chosen": 1.754835360835497,
      "rewards/margins": 1.5429234385946888,
      "rewards/rejected": 0.21191192224080807,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 11.100954055786133,
      "kl": 13.9872407913208,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -42800803.18381113,
      "logits/rejected": -42655946.713245995,
      "logps/chosen": -471.9735455311973,
      "logps/rejected": -391.8405203784571,
      "loss": 0.3627,
      "loss/base_kto": 2.901885747909546,
      "metrics/advantage_var": 5783.517578125,
      "rewards/chosen": 0.8845758293370574,
      "rewards/margins": 2.0644699782066898,
      "rewards/rejected": -1.1798941488696324,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 6.646932125091553,
      "kl": 10.82445240020752,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -41599086.3518225,
      "logits/rejected": -41190625.62711865,
      "logps/chosen": -460.8179477020602,
      "logps/rejected": -390.94602272727275,
      "loss": 0.3665,
      "loss/base_kto": 2.9323196411132812,
      "metrics/advantage_var": 5771.20068359375,
      "rewards/chosen": 0.5926709545395453,
      "rewards/margins": 2.086920859742261,
      "rewards/rejected": -1.4942499052027156,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 8.745616912841797,
      "kl": 12.242945671081543,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -41989474.093457945,
      "logits/rejected": -41297611.836990595,
      "logps/chosen": -471.8758761682243,
      "logps/rejected": -389.6364615987461,
      "loss": 0.3707,
      "loss/base_kto": 2.965242862701416,
      "metrics/advantage_var": 5933.3544921875,
      "rewards/chosen": 0.8538461845611858,
      "rewards/margins": 2.0028409412777317,
      "rewards/rejected": -1.148994756716546,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 7.0273518562316895,
      "kl": 19.097814559936523,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -42896259.92891761,
      "logits/rejected": -44195567.34644478,
      "logps/chosen": -496.05331179321485,
      "logps/rejected": -374.2357696671709,
      "loss": 0.3581,
      "loss/base_kto": 2.864837646484375,
      "metrics/advantage_var": 5337.912109375,
      "rewards/chosen": 1.04482228613439,
      "rewards/margins": 2.0973767874817804,
      "rewards/rejected": -1.0525545013473903,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 5.808931827545166,
      "kl": 9.683107376098633,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -43473398.80239521,
      "logits/rejected": -44432296.99346405,
      "logps/chosen": -459.32185628742513,
      "logps/rejected": -388.02706290849676,
      "loss": 0.373,
      "loss/base_kto": 2.98384952545166,
      "metrics/advantage_var": 5935.91357421875,
      "rewards/chosen": 0.4494812559938716,
      "rewards/margins": 1.9479031182283488,
      "rewards/rejected": -1.4984218622344772,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 8.083940505981445,
      "kl": 32.34806442260742,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -44761008.59450727,
      "logits/rejected": -43924521.82753404,
      "logps/chosen": -506.91735662358644,
      "logps/rejected": -400.2647976550681,
      "loss": 0.3519,
      "loss/base_kto": 2.8153538703918457,
      "metrics/advantage_var": 7049.96728515625,
      "rewards/chosen": 1.8274850675863288,
      "rewards/margins": 2.3638658065530933,
      "rewards/rejected": -0.5363807389667644,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 7.935533046722412,
      "kl": 11.448060989379883,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -41366528.0,
      "logits/rejected": -42194198.70336392,
      "logps/chosen": -462.01277955271564,
      "logps/rejected": -379.04319571865443,
      "loss": 0.3455,
      "loss/base_kto": 2.763831377029419,
      "metrics/advantage_var": 7012.91552734375,
      "rewards/chosen": 0.9539188470322484,
      "rewards/margins": 2.6009064354305664,
      "rewards/rejected": -1.646987588398318,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 8.790483474731445,
      "kl": 13.847442626953125,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -41813509.789983846,
      "logits/rejected": -42020693.59152799,
      "logps/chosen": -485.4153877221325,
      "logps/rejected": -379.9791036308623,
      "loss": 0.3311,
      "loss/base_kto": 2.648519992828369,
      "metrics/advantage_var": 7620.4296875,
      "rewards/chosen": 1.1262229709902565,
      "rewards/margins": 2.844358187222764,
      "rewards/rejected": -1.7181352162325076,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 8.30214786529541,
      "kl": 15.4894380569458,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -43492787.642072216,
      "logits/rejected": -42438321.56765163,
      "logps/chosen": -472.0441522762951,
      "logps/rejected": -367.4388851088647,
      "loss": 0.3532,
      "loss/base_kto": 2.82547926902771,
      "metrics/advantage_var": 7682.71875,
      "rewards/chosen": 1.021260643304798,
      "rewards/margins": 2.4276647024939018,
      "rewards/rejected": -1.4064040591891038,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 8.737303733825684,
      "kl": 27.308744430541992,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -44707931.02222222,
      "logits/rejected": -43532987.470769234,
      "logps/chosen": -465.1904761904762,
      "logps/rejected": -398.69610576923077,
      "loss": 0.3264,
      "loss/base_kto": 2.6114189624786377,
      "metrics/advantage_var": 8271.7685546875,
      "rewards/chosen": 1.7188228546626985,
      "rewards/margins": 2.914910876748035,
      "rewards/rejected": -1.1960880220853365,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 7.870279788970947,
      "kl": 13.653708457946777,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -46274614.9148265,
      "logits/rejected": -45654735.653250776,
      "logps/chosen": -470.99980283911674,
      "logps/rejected": -372.77447755417955,
      "loss": 0.3567,
      "loss/base_kto": 2.853562831878662,
      "metrics/advantage_var": 7688.44921875,
      "rewards/chosen": 0.8195493680069499,
      "rewards/margins": 2.347240999501242,
      "rewards/rejected": -1.5276916314942919,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 9.080231666564941,
      "kl": 19.786767959594727,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -43581364.14814815,
      "logits/rejected": -43813042.2278481,
      "logps/chosen": -442.10792824074076,
      "logps/rejected": -387.9043710443038,
      "loss": 0.3467,
      "loss/base_kto": 2.7739083766937256,
      "metrics/advantage_var": 9266.0595703125,
      "rewards/chosen": 1.423888971776138,
      "rewards/margins": 2.5969124370896273,
      "rewards/rejected": -1.173023465313489,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 8.503353118896484,
      "kl": 19.65529441833496,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -43856134.64427481,
      "logits/rejected": -43512653.4144,
      "logps/chosen": -452.02118320610685,
      "logps/rejected": -370.3334,
      "loss": 0.3258,
      "loss/base_kto": 2.6065726280212402,
      "metrics/advantage_var": 9236.5703125,
      "rewards/chosen": 1.5968776091364503,
      "rewards/margins": 3.0344336638239504,
      "rewards/rejected": -1.4375560546875,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 6.571924686431885,
      "kl": 33.894203186035156,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -46261658.5437788,
      "logits/rejected": -45714837.3672496,
      "logps/chosen": -473.17857142857144,
      "logps/rejected": -392.7427464228935,
      "loss": 0.341,
      "loss/base_kto": 2.7280545234680176,
      "metrics/advantage_var": 9399.193359375,
      "rewards/chosen": 2.450667655229935,
      "rewards/margins": 2.9155829201904377,
      "rewards/rejected": -0.4649152649605028,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 9.13626766204834,
      "kl": 20.319122314453125,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -45136019.85867895,
      "logits/rejected": -46194492.64228935,
      "logps/chosen": -452.5185291858679,
      "logps/rejected": -397.5857015103339,
      "loss": 0.3428,
      "loss/base_kto": 2.7425696849823,
      "metrics/advantage_var": 9139.390625,
      "rewards/chosen": 1.4733294180827572,
      "rewards/margins": 2.7737037023509408,
      "rewards/rejected": -1.3003742842681836,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.005231857299805,
      "kl": 21.759937286376953,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -46905621.8044164,
      "logits/rejected": -46833293.07739938,
      "logps/chosen": -450.3179219242902,
      "logps/rejected": -391.17748645510835,
      "loss": 0.3391,
      "loss/base_kto": 2.71299147605896,
      "metrics/advantage_var": 9798.2314453125,
      "rewards/chosen": 1.6295716679810726,
      "rewards/margins": 2.907357781141386,
      "rewards/rejected": -1.2777861131603134,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 8.948205947875977,
      "kl": 21.718387603759766,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -46783021.97869102,
      "logits/rejected": -45139701.72712681,
      "logps/chosen": -484.298848934551,
      "logps/rejected": -372.73266954253614,
      "loss": 0.3111,
      "loss/base_kto": 2.489197015762329,
      "metrics/advantage_var": 9011.4482421875,
      "rewards/chosen": 1.8880607802392504,
      "rewards/margins": 3.203403274658893,
      "rewards/rejected": -1.3153424944196428,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 7.8195061683654785,
      "kl": 30.360706329345703,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -46205451.61875945,
      "logits/rejected": -45409423.92245557,
      "logps/chosen": -478.77704236006053,
      "logps/rejected": -396.2744850565428,
      "loss": 0.3272,
      "loss/base_kto": 2.6175248622894287,
      "metrics/advantage_var": 10326.212890625,
      "rewards/chosen": 2.0666980368286687,
      "rewards/margins": 3.1215515840065766,
      "rewards/rejected": -1.054853547177908,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 6.63793420791626,
      "kl": 52.894161224365234,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -46940785.777777776,
      "logits/rejected": -46880508.75949367,
      "logps/chosen": -480.3076774691358,
      "logps/rejected": -374.3922320015823,
      "loss": 0.3066,
      "loss/base_kto": 2.4528987407684326,
      "metrics/advantage_var": 10011.7216796875,
      "rewards/chosen": 2.773547137225116,
      "rewards/margins": 3.3432198786925555,
      "rewards/rejected": -0.5696727414674396,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 8.211414337158203,
      "kl": 24.846866607666016,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -46749095.506172836,
      "logits/rejected": -46511726.17721519,
      "logps/chosen": -505.8790509259259,
      "logps/rejected": -378.9453619462025,
      "loss": 0.3305,
      "loss/base_kto": 2.643768787384033,
      "metrics/advantage_var": 11103.37109375,
      "rewards/chosen": 2.026964352454668,
      "rewards/margins": 3.399310482053165,
      "rewards/rejected": -1.3723461295984969,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 10.186355590820312,
      "kl": 35.13449478149414,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -43608122.05354331,
      "logits/rejected": -44839264.446511626,
      "logps/chosen": -456.92529527559054,
      "logps/rejected": -393.90515988372096,
      "loss": 0.3321,
      "loss/base_kto": 2.657120943069458,
      "metrics/advantage_var": 10315.302734375,
      "rewards/chosen": 2.076808178518701,
      "rewards/margins": 3.154547190298013,
      "rewards/rejected": -1.077739011779312,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 8.631143569946289,
      "kl": 36.79061508178711,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -45510366.791461416,
      "logits/rejected": -46906311.535022356,
      "logps/chosen": -460.31958128078816,
      "logps/rejected": -377.0022820417288,
      "loss": 0.3293,
      "loss/base_kto": 2.634350299835205,
      "metrics/advantage_var": 10736.0703125,
      "rewards/chosen": 2.139907010083128,
      "rewards/margins": 3.1054689919665113,
      "rewards/rejected": -0.965561981883383,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 6.811507225036621,
      "kl": 16.8293399810791,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -45059225.12149533,
      "logits/rejected": -45106856.526645765,
      "logps/chosen": -440.123734423676,
      "logps/rejected": -401.2125048981191,
      "loss": 0.3373,
      "loss/base_kto": 2.69854474067688,
      "metrics/advantage_var": 12008.3115234375,
      "rewards/chosen": 1.570432193554079,
      "rewards/margins": 3.2257186416829975,
      "rewards/rejected": -1.6552864481289185,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 7.607090950012207,
      "kl": 25.78407859802246,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -45679349.567398116,
      "logits/rejected": -46405013.13395639,
      "logps/chosen": -476.6412617554859,
      "logps/rejected": -389.7828563084112,
      "loss": 0.3237,
      "loss/base_kto": 2.5895638465881348,
      "metrics/advantage_var": 11293.404296875,
      "rewards/chosen": 2.03739312151009,
      "rewards/margins": 3.2593509848156783,
      "rewards/rejected": -1.221957863305588,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 6.921399116516113,
      "kl": 16.107458114624023,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -45514713.2976378,
      "logits/rejected": -45708515.02635659,
      "logps/chosen": -475.71624015748034,
      "logps/rejected": -367.9354651162791,
      "loss": 0.3122,
      "loss/base_kto": 2.4977047443389893,
      "metrics/advantage_var": 12347.6845703125,
      "rewards/chosen": 1.3382462629183072,
      "rewards/margins": 3.6617011744977646,
      "rewards/rejected": -2.323454911579457,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 9.065701484680176,
      "kl": 24.047327041625977,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -46841501.47758887,
      "logits/rejected": -47183131.09636651,
      "logps/chosen": -479.98551004636784,
      "logps/rejected": -390.7399289099526,
      "loss": 0.324,
      "loss/base_kto": 2.5916531085968018,
      "metrics/advantage_var": 12090.9658203125,
      "rewards/chosen": 2.2090130530332304,
      "rewards/margins": 3.4206983060138287,
      "rewards/rejected": -1.2116852529805984,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 7.208115577697754,
      "kl": 21.597766876220703,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -47840753.08737864,
      "logits/rejected": -47484934.18731118,
      "logps/chosen": -478.4880663430421,
      "logps/rejected": -403.3761329305136,
      "loss": 0.3316,
      "loss/base_kto": 2.652467727661133,
      "metrics/advantage_var": 13014.21875,
      "rewards/chosen": 1.5764168057241101,
      "rewards/margins": 3.3739137742498846,
      "rewards/rejected": -1.7974969685257742,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 7.711939811706543,
      "kl": 28.2313175201416,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -46367084.08888889,
      "logits/rejected": -46945661.24307692,
      "logps/chosen": -453.5967261904762,
      "logps/rejected": -374.50927884615385,
      "loss": 0.297,
      "loss/base_kto": 2.3756587505340576,
      "metrics/advantage_var": 13752.7392578125,
      "rewards/chosen": 2.4194787403893847,
      "rewards/margins": 4.005452673682654,
      "rewards/rejected": -1.5859739332932692,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 7.007646560668945,
      "kl": 30.02141761779785,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -45276892.37540453,
      "logits/rejected": -45065901.76969697,
      "logps/chosen": -446.30157766990294,
      "logps/rejected": -402.1261837121212,
      "loss": 0.3006,
      "loss/base_kto": 2.4051828384399414,
      "metrics/advantage_var": 13163.7060546875,
      "rewards/chosen": 2.2396643184921117,
      "rewards/margins": 4.016955356311714,
      "rewards/rejected": -1.7772910378196023,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 8.173478126525879,
      "kl": 17.41115379333496,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -46656193.6661562,
      "logits/rejected": -47212380.68261563,
      "logps/chosen": -445.1758231240429,
      "logps/rejected": -404.7448664274322,
      "loss": 0.279,
      "loss/base_kto": 2.232347249984741,
      "metrics/advantage_var": 14762.0546875,
      "rewards/chosen": 2.299037495214395,
      "rewards/margins": 4.4196192118461735,
      "rewards/rejected": -2.1205817166317784,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 6.80936861038208,
      "kl": 17.929452896118164,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -45475994.56603774,
      "logits/rejected": -46844374.65838509,
      "logps/chosen": -426.599597091195,
      "logps/rejected": -415.8588412267081,
      "loss": 0.2593,
      "loss/base_kto": 2.0747039318084717,
      "metrics/advantage_var": 14235.2373046875,
      "rewards/chosen": 2.141762595506584,
      "rewards/margins": 4.803825341163611,
      "rewards/rejected": -2.6620627456570265,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 4.544503211975098,
      "kl": 22.975421905517578,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -47021896.24796085,
      "logits/rejected": -46036727.940029986,
      "logps/chosen": -491.61449836867865,
      "logps/rejected": -387.09075149925036,
      "loss": 0.2567,
      "loss/base_kto": 2.0539605617523193,
      "metrics/advantage_var": 14759.0185546875,
      "rewards/chosen": 2.3486837912928222,
      "rewards/margins": 5.012761664465611,
      "rewards/rejected": -2.6640778731727885,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 6.556363582611084,
      "kl": 15.817468643188477,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -48574518.3151326,
      "logits/rejected": -47992947.38028169,
      "logps/chosen": -441.83502340093605,
      "logps/rejected": -393.63113751956183,
      "loss": 0.2591,
      "loss/base_kto": 2.072817087173462,
      "metrics/advantage_var": 13400.6279296875,
      "rewards/chosen": 2.281471288148401,
      "rewards/margins": 4.700414117214813,
      "rewards/rejected": -2.4189428290664123,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 7.777688503265381,
      "kl": 14.315707206726074,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -48459913.57557252,
      "logits/rejected": -48805127.7824,
      "logps/chosen": -440.69608778625957,
      "logps/rejected": -391.97955,
      "loss": 0.2579,
      "loss/base_kto": 2.063169479370117,
      "metrics/advantage_var": 14690.3310546875,
      "rewards/chosen": 2.5303023243678435,
      "rewards/margins": 5.0792437306178435,
      "rewards/rejected": -2.54894140625,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 12.152023315429688,
      "kl": 22.047971725463867,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -50287049.86102719,
      "logits/rejected": -48711812.55663431,
      "logps/chosen": -487.5616503021148,
      "logps/rejected": -401.6143810679612,
      "loss": 0.2559,
      "loss/base_kto": 2.04754638671875,
      "metrics/advantage_var": 13613.640625,
      "rewards/chosen": 2.5347214436603096,
      "rewards/margins": 4.730875794262049,
      "rewards/rejected": -2.1961543506017396,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 6.6538896560668945,
      "kl": 11.32634162902832,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -47334103.578947365,
      "logits/rejected": -46812358.0952381,
      "logps/chosen": -472.7476356907895,
      "logps/rejected": -401.02167038690476,
      "loss": 0.2566,
      "loss/base_kto": 2.052682638168335,
      "metrics/advantage_var": 13816.6689453125,
      "rewards/chosen": 2.0561051619680306,
      "rewards/margins": 4.973276821891766,
      "rewards/rejected": -2.917171659923735,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 6.753261566162109,
      "kl": 20.307422637939453,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -49337350.56410257,
      "logits/rejected": -49350955.70731708,
      "logps/chosen": -468.1454326923077,
      "logps/rejected": -390.9633193597561,
      "loss": 0.2608,
      "loss/base_kto": 2.086202621459961,
      "metrics/advantage_var": 13973.15625,
      "rewards/chosen": 2.541408049754607,
      "rewards/margins": 4.750835860051387,
      "rewards/rejected": -2.2094278102967797,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 6.562922954559326,
      "kl": 18.85896873474121,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -51021522.356589146,
      "logits/rejected": -51042939.36377953,
      "logps/chosen": -473.3996124031008,
      "logps/rejected": -400.0395915354331,
      "loss": 0.2626,
      "loss/base_kto": 2.1009585857391357,
      "metrics/advantage_var": 15106.8251953125,
      "rewards/chosen": 2.31993777252907,
      "rewards/margins": 4.723193493493637,
      "rewards/rejected": -2.403255720964567,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 8.071813583374023,
      "kl": 22.20347785949707,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -49203014.97819315,
      "logits/rejected": -49981135.04702194,
      "logps/chosen": -453.9005062305296,
      "logps/rejected": -412.33821512539186,
      "loss": 0.2573,
      "loss/base_kto": 2.0587565898895264,
      "metrics/advantage_var": 14139.6328125,
      "rewards/chosen": 2.6057438835548092,
      "rewards/margins": 4.811828801633768,
      "rewards/rejected": -2.206084918078958,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 8.019198417663574,
      "kl": 24.09480094909668,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -48533296.93522907,
      "logits/rejected": -47720226.42349304,
      "logps/chosen": -449.9792654028436,
      "logps/rejected": -422.0738021638331,
      "loss": 0.2723,
      "loss/base_kto": 2.1783738136291504,
      "metrics/advantage_var": 13183.6513671875,
      "rewards/chosen": 2.459912340787915,
      "rewards/margins": 4.491786842695759,
      "rewards/rejected": -2.031874501907844,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 8.52418041229248,
      "kl": 16.32106590270996,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -49187318.80239521,
      "logits/rejected": -48029050.14379085,
      "logps/chosen": -460.259309505988,
      "logps/rejected": -408.9749285130719,
      "loss": 0.2512,
      "loss/base_kto": 2.009221315383911,
      "metrics/advantage_var": 14127.5673828125,
      "rewards/chosen": 2.2250684178517965,
      "rewards/margins": 4.883991997049101,
      "rewards/rejected": -2.658923579197304,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 5.944615840911865,
      "kl": 19.7237548828125,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -48926827.78947368,
      "logits/rejected": -48995842.49756098,
      "logps/chosen": -488.859022556391,
      "logps/rejected": -380.9029471544715,
      "loss": 0.2604,
      "loss/base_kto": 2.0830843448638916,
      "metrics/advantage_var": 12477.3466796875,
      "rewards/chosen": 2.6787015757166355,
      "rewards/margins": 4.541950670609928,
      "rewards/rejected": -1.8632490948932927,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 8.016765594482422,
      "kl": 26.5404052734375,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -49199443.15923567,
      "logits/rejected": -48270034.45398773,
      "logps/chosen": -470.86912818471336,
      "logps/rejected": -393.27779907975463,
      "loss": 0.2685,
      "loss/base_kto": 2.1477668285369873,
      "metrics/advantage_var": 14028.2255859375,
      "rewards/chosen": 2.6989994899482483,
      "rewards/margins": 4.560636347993206,
      "rewards/rejected": -1.8616368580449578,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 7.640358924865723,
      "kl": 22.410078048706055,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -50904264.24165342,
      "logits/rejected": -50952012.68202765,
      "logps/chosen": -484.72043918918916,
      "logps/rejected": -387.44239631336404,
      "loss": 0.2606,
      "loss/base_kto": 2.0846333503723145,
      "metrics/advantage_var": 15510.6748046875,
      "rewards/chosen": 2.4037286367249604,
      "rewards/margins": 4.797877362243394,
      "rewards/rejected": -2.3941487255184333,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 6.453657627105713,
      "kl": 26.574853897094727,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -48059187.83800623,
      "logits/rejected": -48764972.940438874,
      "logps/chosen": -447.25292056074767,
      "logps/rejected": -395.70219435736675,
      "loss": 0.245,
      "loss/base_kto": 1.9602657556533813,
      "metrics/advantage_var": 13271.8349609375,
      "rewards/chosen": 2.6893097589320485,
      "rewards/margins": 4.852744687419509,
      "rewards/rejected": -2.1634349284874608,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 11.55324649810791,
      "kl": 17.889604568481445,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -48340237.647819065,
      "logits/rejected": -48361713.670196675,
      "logps/chosen": -455.0253937802908,
      "logps/rejected": -400.8607696671709,
      "loss": 0.2562,
      "loss/base_kto": 2.049893856048584,
      "metrics/advantage_var": 12405.05078125,
      "rewards/chosen": 1.9586449840342286,
      "rewards/margins": 4.61523220096407,
      "rewards/rejected": -2.6565872169298412,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 7.1196184158325195,
      "kl": 20.197782516479492,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -47066458.75993641,
      "logits/rejected": -47454664.159754224,
      "logps/chosen": -467.860393481717,
      "logps/rejected": -399.22633448540705,
      "loss": 0.244,
      "loss/base_kto": 1.9522743225097656,
      "metrics/advantage_var": 12976.525390625,
      "rewards/chosen": 2.20633880663752,
      "rewards/margins": 4.915033288084333,
      "rewards/rejected": -2.7086944814468126,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 8.951573371887207,
      "kl": 20.724227905273438,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -47757560.14216478,
      "logits/rejected": -48288456.6172466,
      "logps/chosen": -464.625,
      "logps/rejected": -395.11682110438727,
      "loss": 0.2513,
      "loss/base_kto": 2.010521411895752,
      "metrics/advantage_var": 13302.541015625,
      "rewards/chosen": 2.286919663141155,
      "rewards/margins": 4.8331484845883,
      "rewards/rejected": -2.5462288214471447,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 9.4291353225708,
      "kl": 22.72542381286621,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -47091389.32475884,
      "logits/rejected": -47570663.87841945,
      "logps/chosen": -478.8822347266881,
      "logps/rejected": -365.6098736702128,
      "loss": 0.239,
      "loss/base_kto": 1.9119118452072144,
      "metrics/advantage_var": 12811.125,
      "rewards/chosen": 2.425092004119775,
      "rewards/margins": 4.811326925282389,
      "rewards/rejected": -2.386234921162614,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 6.64006233215332,
      "kl": 19.150110244750977,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -47568567.346210994,
      "logits/rejected": -49767010.68863262,
      "logps/chosen": -483.8380850668648,
      "logps/rejected": -384.9492637973641,
      "loss": 0.2549,
      "loss/base_kto": 2.039292812347412,
      "metrics/advantage_var": 14007.6123046875,
      "rewards/chosen": 2.5177968720978825,
      "rewards/margins": 4.919014035787133,
      "rewards/rejected": -2.4012171636892505,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 8.059784889221191,
      "kl": 21.792648315429688,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -47696774.7788162,
      "logits/rejected": -48640969.42946708,
      "logps/chosen": -445.69314641744546,
      "logps/rejected": -396.8174471003135,
      "loss": 0.2522,
      "loss/base_kto": 2.0174732208251953,
      "metrics/advantage_var": 12939.958984375,
      "rewards/chosen": 2.2641485576689058,
      "rewards/margins": 4.6352878448701205,
      "rewards/rejected": -2.3711392872012147,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 8.086423873901367,
      "kl": 20.496225357055664,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -47647981.307936504,
      "logits/rejected": -49093603.64307693,
      "logps/chosen": -490.843501984127,
      "logps/rejected": -401.04983173076926,
      "loss": 0.2477,
      "loss/base_kto": 1.9819554090499878,
      "metrics/advantage_var": 13979.4912109375,
      "rewards/chosen": 2.251154048859127,
      "rewards/margins": 5.037465084015377,
      "rewards/rejected": -2.78631103515625,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 8.792497634887695,
      "kl": 30.228851318359375,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -47447484.43879173,
      "logits/rejected": -48123139.5391705,
      "logps/chosen": -460.79113672496027,
      "logps/rejected": -372.7542722734255,
      "loss": 0.2564,
      "loss/base_kto": 2.0514838695526123,
      "metrics/advantage_var": 13797.9755859375,
      "rewards/chosen": 2.6598088638960653,
      "rewards/margins": 4.687813406186772,
      "rewards/rejected": -2.0280045422907067,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 7.578653335571289,
      "kl": 25.090009689331055,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -49122948.62166405,
      "logits/rejected": -49207139.93157076,
      "logps/chosen": -454.2995486656201,
      "logps/rejected": -390.2356629082426,
      "loss": 0.2587,
      "loss/base_kto": 2.069631576538086,
      "metrics/advantage_var": 13317.5,
      "rewards/chosen": 2.299419888147567,
      "rewards/margins": 4.770764689129294,
      "rewards/rejected": -2.4713448009817265,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 14.963940620422363,
      "kl": 17.4893856048584,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -47609887.65378671,
      "logits/rejected": -47908110.154818326,
      "logps/chosen": -450.29921754250387,
      "logps/rejected": -389.2090244865719,
      "loss": 0.2503,
      "loss/base_kto": 2.002574920654297,
      "metrics/advantage_var": 13172.140625,
      "rewards/chosen": 2.4579678141904946,
      "rewards/margins": 4.867302355499243,
      "rewards/rejected": -2.409334541308748,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 9.611598014831543,
      "kl": 19.83905601501465,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -49030802.28571428,
      "logits/rejected": -49205513.253012046,
      "logps/chosen": -464.34293831168833,
      "logps/rejected": -408.77179028614455,
      "loss": 0.2459,
      "loss/base_kto": 1.9675463438034058,
      "metrics/advantage_var": 13988.8271484375,
      "rewards/chosen": 2.4432099577668427,
      "rewards/margins": 4.986179626971003,
      "rewards/rejected": -2.5429696692041603,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 7.579370975494385,
      "kl": 14.656468391418457,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -49270376.28177641,
      "logits/rejected": -49555152.22966507,
      "logps/chosen": -466.990141653905,
      "logps/rejected": -396.08756977671453,
      "loss": 0.2616,
      "loss/base_kto": 2.0930798053741455,
      "metrics/advantage_var": 12933.798828125,
      "rewards/chosen": 2.0334977388016844,
      "rewards/margins": 4.576472913554974,
      "rewards/rejected": -2.5429751747532894,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 6.426728248596191,
      "kl": 21.355215072631836,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -48487235.53374233,
      "logits/rejected": -48920758.62420382,
      "logps/chosen": -463.2795245398773,
      "logps/rejected": -401.8110569267516,
      "loss": 0.2608,
      "loss/base_kto": 2.086047410964966,
      "metrics/advantage_var": 13996.095703125,
      "rewards/chosen": 2.549413762940951,
      "rewards/margins": 4.723553283865513,
      "rewards/rejected": -2.174139520924562,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 8.00083065032959,
      "kl": 29.541763305664062,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -49463453.538461536,
      "logits/rejected": -49750379.09797823,
      "logps/chosen": -454.42327315541604,
      "logps/rejected": -412.99912519440124,
      "loss": 0.2473,
      "loss/base_kto": 1.9786356687545776,
      "metrics/advantage_var": 13827.681640625,
      "rewards/chosen": 2.412167631475667,
      "rewards/margins": 4.759135503632549,
      "rewards/rejected": -2.3469678721568816,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.693033218383789,
      "kl": 15.414921760559082,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -49728912.02413273,
      "logits/rejected": -49519315.60453809,
      "logps/chosen": -455.81061463046757,
      "logps/rejected": -374.9399311183144,
      "loss": 0.2398,
      "loss/base_kto": 1.9187382459640503,
      "metrics/advantage_var": 14239.2685546875,
      "rewards/chosen": 2.254514392145079,
      "rewards/margins": 5.03609360608349,
      "rewards/rejected": -2.7815792139384117,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 6.900964736938477,
      "kl": 20.708576202392578,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -50192860.88676236,
      "logits/rejected": -49531403.7611026,
      "logps/chosen": -466.3460925039872,
      "logps/rejected": -379.18082408116385,
      "loss": 0.2688,
      "loss/base_kto": 2.150141716003418,
      "metrics/advantage_var": 13455.515625,
      "rewards/chosen": 2.2594463342304625,
      "rewards/margins": 4.589688216648151,
      "rewards/rejected": -2.330241882417688,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 8.05775260925293,
      "kl": 35.416603088378906,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -47505559.004608296,
      "logits/rejected": -47748973.48171701,
      "logps/chosen": -449.55587557603684,
      "logps/rejected": -367.85159976152624,
      "loss": 0.2338,
      "loss/base_kto": 1.8703714609146118,
      "metrics/advantage_var": 12229.8173828125,
      "rewards/chosen": 3.1599852390552994,
      "rewards/margins": 5.009129039047847,
      "rewards/rejected": -1.8491437999925477,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 10.813697814941406,
      "kl": 20.462011337280273,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -49673536.19814242,
      "logits/rejected": -50293795.53312303,
      "logps/chosen": -452.44615905572755,
      "logps/rejected": -416.8113170347003,
      "loss": 0.2643,
      "loss/base_kto": 2.1144533157348633,
      "metrics/advantage_var": 14252.142578125,
      "rewards/chosen": 2.1609387471579913,
      "rewards/margins": 4.722467575775894,
      "rewards/rejected": -2.5615288286179023,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 7.989013195037842,
      "kl": 17.5428466796875,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -50056393.64923077,
      "logits/rejected": -49990984.33015873,
      "logps/chosen": -474.66769230769233,
      "logps/rejected": -413.96671626984124,
      "loss": 0.2435,
      "loss/base_kto": 1.9483126401901245,
      "metrics/advantage_var": 14465.009765625,
      "rewards/chosen": 2.5189391150841347,
      "rewards/margins": 5.101884536091079,
      "rewards/rejected": -2.5829454210069445,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 8.70841121673584,
      "kl": 32.39929962158203,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -50555225.681528665,
      "logits/rejected": -52477776.09815951,
      "logps/chosen": -451.25084593949043,
      "logps/rejected": -388.8933569785276,
      "loss": 0.2402,
      "loss/base_kto": 1.921789526939392,
      "metrics/advantage_var": 12992.45703125,
      "rewards/chosen": 2.8987142963773884,
      "rewards/margins": 4.943832757003348,
      "rewards/rejected": -2.0451184606259587,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 8.169147491455078,
      "kl": 25.643451690673828,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -49150005.58139535,
      "logits/rejected": -49291436.972972974,
      "logps/chosen": -454.4925054505814,
      "logps/rejected": -391.3290223817568,
      "loss": 0.2583,
      "loss/base_kto": 2.0667595863342285,
      "metrics/advantage_var": 12927.4765625,
      "rewards/chosen": 2.863457968068677,
      "rewards/margins": 4.700496162879699,
      "rewards/rejected": -1.837038194811022,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 7.993652820587158,
      "kl": 25.68218994140625,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -51835381.418124005,
      "logits/rejected": -51349979.03533026,
      "logps/chosen": -465.550079491256,
      "logps/rejected": -408.51060867895546,
      "loss": 0.2639,
      "loss/base_kto": 2.1111161708831787,
      "metrics/advantage_var": 14188.736328125,
      "rewards/chosen": 2.582842464477345,
      "rewards/margins": 4.783263541426269,
      "rewards/rejected": -2.2004210769489245,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 7.127334117889404,
      "kl": 16.307735443115234,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -49595238.06535948,
      "logits/rejected": -49896564.85285285,
      "logps/chosen": -468.554891748366,
      "logps/rejected": -403.67328265765764,
      "loss": 0.2215,
      "loss/base_kto": 1.7723366022109985,
      "metrics/advantage_var": 15443.9345703125,
      "rewards/chosen": 2.419900931564032,
      "rewards/margins": 5.460144910894925,
      "rewards/rejected": -3.0402439793308935,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 6.784162521362305,
      "kl": 10.496627807617188,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -49782327.20827179,
      "logits/rejected": -49491845.731343284,
      "logps/chosen": -436.35293574593794,
      "logps/rejected": -407.7500777363184,
      "loss": 0.212,
      "loss/base_kto": 1.6957463026046753,
      "metrics/advantage_var": 14359.087890625,
      "rewards/chosen": 2.7411344857828657,
      "rewards/margins": 5.641210521619309,
      "rewards/rejected": -2.9000760358364426,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 7.4223833084106445,
      "kl": 18.540119171142578,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -49057408.80503145,
      "logits/rejected": -50834272.99378882,
      "logps/chosen": -478.2899960691824,
      "logps/rejected": -385.6640139751553,
      "loss": 0.2166,
      "loss/base_kto": 1.7324780225753784,
      "metrics/advantage_var": 14026.9814453125,
      "rewards/chosen": 2.643333003206073,
      "rewards/margins": 5.287066626070979,
      "rewards/rejected": -2.643733622864907,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 7.456380844116211,
      "kl": 19.18268394470215,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -50089974.59111792,
      "logits/rejected": -51495983.362041466,
      "logps/chosen": -460.1427067381317,
      "logps/rejected": -408.0125598086124,
      "loss": 0.2258,
      "loss/base_kto": 1.8063572645187378,
      "metrics/advantage_var": 15073.0185546875,
      "rewards/chosen": 2.8757813995501533,
      "rewards/margins": 5.380927821624516,
      "rewards/rejected": -2.505146422074362,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 7.515079498291016,
      "kl": 15.434982299804688,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -50601090.21036107,
      "logits/rejected": -49526793.55520996,
      "logps/chosen": -463.3057299843014,
      "logps/rejected": -399.95956454121307,
      "loss": 0.2194,
      "loss/base_kto": 1.7549737691879272,
      "metrics/advantage_var": 13595.9375,
      "rewards/chosen": 2.455097863213795,
      "rewards/margins": 5.31824090607927,
      "rewards/rejected": -2.8631430428654743,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 6.709864616394043,
      "kl": 11.128249168395996,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -50074304.83116883,
      "logits/rejected": -49689544.48192771,
      "logps/chosen": -454.7149959415584,
      "logps/rejected": -400.812311746988,
      "loss": 0.2063,
      "loss/base_kto": 1.6505447626113892,
      "metrics/advantage_var": 14310.376953125,
      "rewards/chosen": 2.2841396579494724,
      "rewards/margins": 5.6345535204306465,
      "rewards/rejected": -3.3504138624811746,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 6.725008010864258,
      "kl": 10.414727210998535,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -49975073.39130435,
      "logits/rejected": -51824623.89937107,
      "logps/chosen": -471.26674107142856,
      "logps/rejected": -390.125393081761,
      "loss": 0.2155,
      "loss/base_kto": 1.723760962486267,
      "metrics/advantage_var": 14411.134765625,
      "rewards/chosen": 2.309495629731172,
      "rewards/margins": 5.520320272272446,
      "rewards/rejected": -3.2108246425412736,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 6.696305751800537,
      "kl": 19.083173751831055,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51039629.06122449,
      "logits/rejected": -52006752.746500775,
      "logps/chosen": -500.1241660125589,
      "logps/rejected": -397.9802196734059,
      "loss": 0.1986,
      "loss/base_kto": 1.5888489484786987,
      "metrics/advantage_var": 15714.0361328125,
      "rewards/chosen": 3.0086947789933283,
      "rewards/margins": 5.956221630665179,
      "rewards/rejected": -2.9475268516718507,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 7.990821838378906,
      "kl": 16.3265323638916,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -50054883.46870229,
      "logits/rejected": -51335384.2688,
      "logps/chosen": -452.41808206106873,
      "logps/rejected": -417.467625,
      "loss": 0.2201,
      "loss/base_kto": 1.7606159448623657,
      "metrics/advantage_var": 15616.5,
      "rewards/chosen": 2.714349505009542,
      "rewards/margins": 5.649598333134542,
      "rewards/rejected": -2.935248828125,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 6.282646656036377,
      "kl": 14.768630981445312,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -50328071.035114504,
      "logits/rejected": -50948728.4224,
      "logps/chosen": -472.78134541984736,
      "logps/rejected": -390.61135,
      "loss": 0.2196,
      "loss/base_kto": 1.7569602727890015,
      "metrics/advantage_var": 14497.3505859375,
      "rewards/chosen": 2.513952170801527,
      "rewards/margins": 5.425104905176527,
      "rewards/rejected": -2.911152734375,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 7.860062599182129,
      "kl": 18.362916946411133,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -49646857.01684533,
      "logits/rejected": -49474414.64752791,
      "logps/chosen": -423.1053790199081,
      "logps/rejected": -397.5823116028708,
      "loss": 0.2189,
      "loss/base_kto": 1.750977873802185,
      "metrics/advantage_var": 15536.078125,
      "rewards/chosen": 2.5981626642060682,
      "rewards/margins": 5.376900780359378,
      "rewards/rejected": -2.7787381161533093,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 8.946330070495605,
      "kl": 24.330366134643555,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -51386275.489096574,
      "logits/rejected": -50967632.2507837,
      "logps/chosen": -464.68092873831773,
      "logps/rejected": -414.86162813479626,
      "loss": 0.203,
      "loss/base_kto": 1.6239101886749268,
      "metrics/advantage_var": 13769.6953125,
      "rewards/chosen": 2.8734166986103,
      "rewards/margins": 5.41564441676028,
      "rewards/rejected": -2.5422277181499804,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 7.575611114501953,
      "kl": 14.73845386505127,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -52410310.66874028,
      "logits/rejected": -50013941.14913658,
      "logps/chosen": -467.96491057542767,
      "logps/rejected": -395.2499754709576,
      "loss": 0.2125,
      "loss/base_kto": 1.699928879737854,
      "metrics/advantage_var": 15204.1005859375,
      "rewards/chosen": 2.6341455913442844,
      "rewards/margins": 5.696759613064751,
      "rewards/rejected": -3.062614021720467,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 8.54968547821045,
      "kl": 20.515256881713867,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -50502933.50237718,
      "logits/rejected": -51343561.95993837,
      "logps/chosen": -443.70106973058637,
      "logps/rejected": -378.1875481510015,
      "loss": 0.2262,
      "loss/base_kto": 1.8092983961105347,
      "metrics/advantage_var": 14298.4443359375,
      "rewards/chosen": 2.5944340580427894,
      "rewards/margins": 5.181517965677131,
      "rewards/rejected": -2.5870839076343413,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 8.126795768737793,
      "kl": 19.532196044921875,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -51061946.03475513,
      "logits/rejected": -51500096.89026275,
      "logps/chosen": -433.7949249605055,
      "logps/rejected": -422.66653786707883,
      "loss": 0.2046,
      "loss/base_kto": 1.637192964553833,
      "metrics/advantage_var": 14092.0693359375,
      "rewards/chosen": 2.652627616508689,
      "rewards/margins": 5.654991478866204,
      "rewards/rejected": -3.0023638623575155,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 5.897149085998535,
      "kl": 21.163227081298828,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -51688446.40746501,
      "logits/rejected": -51002498.21036107,
      "logps/chosen": -450.73950233281494,
      "logps/rejected": -417.90124607535324,
      "loss": 0.2114,
      "loss/base_kto": 1.6915730237960815,
      "metrics/advantage_var": 13375.6953125,
      "rewards/chosen": 2.561840858281493,
      "rewards/margins": 5.410429595158456,
      "rewards/rejected": -2.8485887368769625,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 8.426025390625,
      "kl": 22.875829696655273,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -47735147.663551405,
      "logits/rejected": -50488265.42946708,
      "logps/chosen": -445.261292834891,
      "logps/rejected": -409.0168740203762,
      "loss": 0.2198,
      "loss/base_kto": 1.7581509351730347,
      "metrics/advantage_var": 13349.6689453125,
      "rewards/chosen": 2.66365179614486,
      "rewards/margins": 5.254863766841862,
      "rewards/rejected": -2.5912119706970023,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 7.257813930511475,
      "kl": 19.541242599487305,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -49885532.455858745,
      "logits/rejected": -50699527.40334856,
      "logps/chosen": -451.54303772070625,
      "logps/rejected": -408.70933219178085,
      "loss": 0.2043,
      "loss/base_kto": 1.6344120502471924,
      "metrics/advantage_var": 13755.4189453125,
      "rewards/chosen": 2.823837647660012,
      "rewards/margins": 5.598703946294906,
      "rewards/rejected": -2.7748662986348935,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 7.962888240814209,
      "kl": 10.586474418640137,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -51680913.762517884,
      "logits/rejected": -51276505.66609294,
      "logps/chosen": -461.40334406294704,
      "logps/rejected": -414.9785660499139,
      "loss": 0.214,
      "loss/base_kto": 1.7116488218307495,
      "metrics/advantage_var": 14825.0048828125,
      "rewards/chosen": 2.555452404104077,
      "rewards/margins": 5.493936950548784,
      "rewards/rejected": -2.9384845464447076,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 7.1761298179626465,
      "kl": 15.605143547058105,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -51403234.077922076,
      "logits/rejected": -51802593.15662651,
      "logps/chosen": -507.03059050324674,
      "logps/rejected": -390.7486351656627,
      "loss": 0.1924,
      "loss/base_kto": 1.5393751859664917,
      "metrics/advantage_var": 15480.7099609375,
      "rewards/chosen": 2.8869965788605927,
      "rewards/margins": 6.081190591238228,
      "rewards/rejected": -3.1941940123776353,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 7.228409767150879,
      "kl": 15.267898559570312,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -48605112.933753945,
      "logits/rejected": -49457862.14241486,
      "logps/chosen": -452.8354199526814,
      "logps/rejected": -385.3189821981424,
      "loss": 0.2023,
      "loss/base_kto": 1.6183490753173828,
      "metrics/advantage_var": 14819.5283203125,
      "rewards/chosen": 2.5347646237923898,
      "rewards/margins": 5.6482885027107335,
      "rewards/rejected": -3.1135238789183437,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 6.922834396362305,
      "kl": 16.767065048217773,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -50873499.4428795,
      "logits/rejected": -49571018.88299532,
      "logps/chosen": -442.87377738654146,
      "logps/rejected": -409.1465483619345,
      "loss": 0.2299,
      "loss/base_kto": 1.8388410806655884,
      "metrics/advantage_var": 13165.7158203125,
      "rewards/chosen": 2.2737991261370305,
      "rewards/margins": 5.173699946388591,
      "rewards/rejected": -2.89990082025156,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 6.087769508361816,
      "kl": 17.345020294189453,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -50924598.05731523,
      "logits/rejected": -51076071.10534846,
      "logps/chosen": -451.93999811463044,
      "logps/rejected": -398.00182333873585,
      "loss": 0.1927,
      "loss/base_kto": 1.5418506860733032,
      "metrics/advantage_var": 15268.994140625,
      "rewards/chosen": 2.669620841876414,
      "rewards/margins": 5.844617834633707,
      "rewards/rejected": -3.1749969927572934,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 7.265406608581543,
      "kl": 17.250513076782227,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -50358195.913312696,
      "logits/rejected": -50521524.08832808,
      "logps/chosen": -470.1187113003096,
      "logps/rejected": -375.2814964511041,
      "loss": 0.2118,
      "loss/base_kto": 1.6946674585342407,
      "metrics/advantage_var": 13960.9951171875,
      "rewards/chosen": 2.7360907870549536,
      "rewards/margins": 5.317020323603652,
      "rewards/rejected": -2.5809295365486986,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 7.966547966003418,
      "kl": 12.166990280151367,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -51448348.86646884,
      "logits/rejected": -50085110.70627063,
      "logps/chosen": -448.09375,
      "logps/rejected": -415.8252887788779,
      "loss": 0.2227,
      "loss/base_kto": 1.7817531824111938,
      "metrics/advantage_var": 15057.634765625,
      "rewards/chosen": 2.597703701664503,
      "rewards/margins": 5.362192601726385,
      "rewards/rejected": -2.764488900061881,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 6.479636192321777,
      "kl": 21.107120513916016,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -51183870.3483871,
      "logits/rejected": -51802614.69090909,
      "logps/chosen": -458.6647681451613,
      "logps/rejected": -418.3832859848485,
      "loss": 0.2142,
      "loss/base_kto": 1.7134571075439453,
      "metrics/advantage_var": 14570.9716796875,
      "rewards/chosen": 2.7536784510458667,
      "rewards/margins": 5.491471789705905,
      "rewards/rejected": -2.7377933386600377,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 6.600201606750488,
      "kl": 18.848928451538086,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -51338816.99071208,
      "logits/rejected": -51748482.82649842,
      "logps/chosen": -448.76683436532505,
      "logps/rejected": -409.4468404968454,
      "loss": 0.2105,
      "loss/base_kto": 1.6841938495635986,
      "metrics/advantage_var": 14403.134765625,
      "rewards/chosen": 2.503087471882256,
      "rewards/margins": 5.406129756730442,
      "rewards/rejected": -2.903042284848186,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 6.502203941345215,
      "kl": 19.20033836364746,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -49168930.98136646,
      "logits/rejected": -50254120.251572326,
      "logps/chosen": -479.9908288043478,
      "logps/rejected": -378.2589426100629,
      "loss": 0.2073,
      "loss/base_kto": 1.658690094947815,
      "metrics/advantage_var": 14569.7236328125,
      "rewards/chosen": 2.8965670188761647,
      "rewards/margins": 5.645065323711071,
      "rewards/rejected": -2.748498304834906,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 9.374115943908691,
      "kl": 16.311094284057617,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -50253843.629392974,
      "logits/rejected": -52141566.434250765,
      "logps/chosen": -469.2500499201278,
      "logps/rejected": -407.09193425076455,
      "loss": 0.2019,
      "loss/base_kto": 1.6155580282211304,
      "metrics/advantage_var": 14986.9287109375,
      "rewards/chosen": 2.7423072303065097,
      "rewards/margins": 5.90321232754466,
      "rewards/rejected": -3.16090509723815,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 7.66606330871582,
      "kl": 20.138084411621094,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -50138638.5362776,
      "logits/rejected": -50570610.92260062,
      "logps/chosen": -468.341285488959,
      "logps/rejected": -399.02713815789474,
      "loss": 0.1985,
      "loss/base_kto": 1.5876652002334595,
      "metrics/advantage_var": 14451.103515625,
      "rewards/chosen": 2.6708017824576102,
      "rewards/margins": 5.763690676553489,
      "rewards/rejected": -3.0928888940958785,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 7.446461200714111,
      "kl": 21.64815330505371,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -50349630.0355412,
      "logits/rejected": -50701945.6096823,
      "logps/chosen": -476.92790791599356,
      "logps/rejected": -398.7458396369138,
      "loss": 0.2051,
      "loss/base_kto": 1.6408064365386963,
      "metrics/advantage_var": 14884.3896484375,
      "rewards/chosen": 2.6866137576989093,
      "rewards/margins": 5.743195840657968,
      "rewards/rejected": -3.0565820829590584,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 8.375565528869629,
      "kl": 23.32106590270996,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -51635112.70819672,
      "logits/rejected": -52851250.43582089,
      "logps/chosen": -470.435143442623,
      "logps/rejected": -406.24967350746266,
      "loss": 0.1974,
      "loss/base_kto": 1.5795862674713135,
      "metrics/advantage_var": 15001.556640625,
      "rewards/chosen": 2.739492347592213,
      "rewards/margins": 5.774384779961617,
      "rewards/rejected": -3.034892432369403,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 6.117384433746338,
      "kl": 19.464855194091797,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -51135410.499194846,
      "logits/rejected": -52004487.96358118,
      "logps/chosen": -449.73661433172305,
      "logps/rejected": -397.84754362670714,
      "loss": 0.2139,
      "loss/base_kto": 1.7114804983139038,
      "metrics/advantage_var": 13869.8876953125,
      "rewards/chosen": 2.5128948316098025,
      "rewards/margins": 5.446001411773877,
      "rewards/rejected": -2.9331065801640746,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 7.1891303062438965,
      "kl": 23.11102867126465,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -51682678.87163233,
      "logits/rejected": -51322785.33127889,
      "logps/chosen": -445.8949088748019,
      "logps/rejected": -414.2937692604006,
      "loss": 0.2002,
      "loss/base_kto": 1.6013821363449097,
      "metrics/advantage_var": 15572.7216796875,
      "rewards/chosen": 2.7213179260350633,
      "rewards/margins": 5.883317080383099,
      "rewards/rejected": -3.1619991543480355,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 7.130632400512695,
      "kl": 19.88947105407715,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -52005329.45454545,
      "logits/rejected": -52686891.18072289,
      "logps/chosen": -470.73787540584414,
      "logps/rejected": -424.3311370481928,
      "loss": 0.2134,
      "loss/base_kto": 1.7074934244155884,
      "metrics/advantage_var": 14958.361328125,
      "rewards/chosen": 2.825003091391031,
      "rewards/margins": 5.454456267866464,
      "rewards/rejected": -2.629453176475433,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.399202346801758,
      "kl": 17.426616668701172,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -50207483.987460814,
      "logits/rejected": -51213363.04049844,
      "logps/chosen": -443.9810442789969,
      "logps/rejected": -393.54239680685356,
      "loss": 0.213,
      "loss/base_kto": 1.7043474912643433,
      "metrics/advantage_var": 14574.78125,
      "rewards/chosen": 2.522138733101489,
      "rewards/margins": 5.6771853327777935,
      "rewards/rejected": -3.1550465996763046,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 6.621180534362793,
      "kl": 12.879165649414062,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -51292347.33643411,
      "logits/rejected": -51794236.06929134,
      "logps/chosen": -472.0147286821705,
      "logps/rejected": -419.4152559055118,
      "loss": 0.2145,
      "loss/base_kto": 1.7159721851348877,
      "metrics/advantage_var": 14325.78125,
      "rewards/chosen": 2.685220218628876,
      "rewards/margins": 5.562849055673659,
      "rewards/rejected": -2.8776288370447833,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 8.082818031311035,
      "kl": 14.013216972351074,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -50113596.60091047,
      "logits/rejected": -48584469.84863124,
      "logps/chosen": -449.9308611532625,
      "logps/rejected": -370.71844806763283,
      "loss": 0.2059,
      "loss/base_kto": 1.6473382711410522,
      "metrics/advantage_var": 14097.734375,
      "rewards/chosen": 2.5490370708459786,
      "rewards/margins": 5.652958614034888,
      "rewards/rejected": -3.103921543188909,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.975174191323546e+17,
      "train_loss": 0.274414178620966,
      "train_runtime": 11069.1641,
      "train_samples_per_second": 13.39,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.975174191323546e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}