{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 9.96064281463623,
      "kl": 57.65949249267578,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37995714.74846626,
      "logits/rejected": -38779241.98726115,
      "logps/chosen": -450.8286042944785,
      "logps/rejected": -374.59121218152865,
      "loss": 0.4564,
      "loss/base_kto": 3.6511714458465576,
      "metrics/advantage_var": 538.0438842773438,
      "rewards/chosen": 0.9948832506051093,
      "rewards/margins": 0.29010821180696533,
      "rewards/rejected": 0.704775038798144,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 10.330055236816406,
      "kl": 16.56190299987793,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37086065.86750789,
      "logits/rejected": -37968436.30959752,
      "logps/chosen": -475.0192724763407,
      "logps/rejected": -362.4485777863777,
      "loss": 0.4488,
      "loss/base_kto": 3.590278387069702,
      "metrics/advantage_var": 806.6080932617188,
      "rewards/chosen": 0.528348327059099,
      "rewards/margins": 0.4430706338686572,
      "rewards/rejected": 0.08527769319044178,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 8.40238094329834,
      "kl": 10.559950828552246,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36252843.67058823,
      "logits/rejected": -37554100.906666666,
      "logps/chosen": -460.3076746323529,
      "logps/rejected": -384.36171875,
      "loss": 0.4338,
      "loss/base_kto": 3.47013783454895,
      "metrics/advantage_var": 966.3760986328125,
      "rewards/chosen": 0.5126130047966452,
      "rewards/margins": 0.6342480349073223,
      "rewards/rejected": -0.12163503011067708,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 9.587729454040527,
      "kl": 28.615070343017578,
      "learning_rate": 7.9e-07,
      "logits/chosen": -37204144.38277512,
      "logits/rejected": -38282552.061255746,
      "logps/chosen": -479.75458532695376,
      "logps/rejected": -382.3811255742726,
      "loss": 0.441,
      "loss/base_kto": 3.5277810096740723,
      "metrics/advantage_var": 1355.2171630859375,
      "rewards/chosen": 0.8166142757239334,
      "rewards/margins": 0.5394598468709233,
      "rewards/rejected": 0.27715442885301017,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 10.16214656829834,
      "kl": 38.51455307006836,
      "learning_rate": 9.9e-07,
      "logits/chosen": -39859357.77643505,
      "logits/rejected": -39609748.297734626,
      "logps/chosen": -471.56891993957703,
      "logps/rejected": -368.18031957928804,
      "loss": 0.4119,
      "loss/base_kto": 3.294884204864502,
      "metrics/advantage_var": 2121.011474609375,
      "rewards/chosen": 1.214173930649311,
      "rewards/margins": 0.8701046318464691,
      "rewards/rejected": 0.3440692988028418,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 9.195523262023926,
      "kl": 18.842559814453125,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -39613262.46439628,
      "logits/rejected": -38695997.37539432,
      "logps/chosen": -462.6353037925697,
      "logps/rejected": -347.01944499211356,
      "loss": 0.4002,
      "loss/base_kto": 3.2016677856445312,
      "metrics/advantage_var": 3064.429443359375,
      "rewards/chosen": 0.9390506331027477,
      "rewards/margins": 1.1935873791549092,
      "rewards/rejected": -0.25453674605216137,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 10.514404296875,
      "kl": 8.773940086364746,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -40551948.17115689,
      "logits/rejected": -40892785.20801233,
      "logps/chosen": -469.73514263074486,
      "logps/rejected": -388.4011941448382,
      "loss": 0.3942,
      "loss/base_kto": 3.1532838344573975,
      "metrics/advantage_var": 4229.67578125,
      "rewards/chosen": 0.5272595969319532,
      "rewards/margins": 1.4671285885897922,
      "rewards/rejected": -0.939868991657839,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 8.357895851135254,
      "kl": 15.100295066833496,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -40738180.8607595,
      "logits/rejected": -41197637.530864194,
      "logps/chosen": -483.00178006329116,
      "logps/rejected": -376.13421103395063,
      "loss": 0.3844,
      "loss/base_kto": 3.0754406452178955,
      "metrics/advantage_var": 5060.587890625,
      "rewards/chosen": 0.8644038333168512,
      "rewards/margins": 1.6502909039609002,
      "rewards/rejected": -0.785887070644049,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 9.702012062072754,
      "kl": 6.5621843338012695,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -40952274.63291139,
      "logits/rejected": -42449069.82716049,
      "logps/chosen": -518.0392602848101,
      "logps/rejected": -407.48533950617286,
      "loss": 0.3861,
      "loss/base_kto": 3.088801622390747,
      "metrics/advantage_var": 4415.6943359375,
      "rewards/chosen": 0.32813692696486846,
      "rewards/margins": 1.63254242193738,
      "rewards/rejected": -1.3044054949725117,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 7.299619674682617,
      "kl": 16.3758544921875,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -42427388.73885351,
      "logits/rejected": -42330005.202453986,
      "logps/chosen": -513.141421178344,
      "logps/rejected": -372.5459164110429,
      "loss": 0.3698,
      "loss/base_kto": 2.9581849575042725,
      "metrics/advantage_var": 5061.00439453125,
      "rewards/chosen": 0.7296892882912023,
      "rewards/margins": 1.8715177667708802,
      "rewards/rejected": -1.141828478479678,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 8.850095748901367,
      "kl": 12.437146186828613,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -44007936.0,
      "logits/rejected": -43484697.28395062,
      "logps/chosen": -464.53258504746833,
      "logps/rejected": -422.64699074074076,
      "loss": 0.3858,
      "loss/base_kto": 3.086326837539673,
      "metrics/advantage_var": 5249.66796875,
      "rewards/chosen": 0.7787781244591822,
      "rewards/margins": 1.7817974808924384,
      "rewards/rejected": -1.0030193564332561,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 9.223854064941406,
      "kl": 34.16802978515625,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -41462785.627980925,
      "logits/rejected": -39973147.13364055,
      "logps/chosen": -478.3248211446741,
      "logps/rejected": -391.4754704301075,
      "loss": 0.3687,
      "loss/base_kto": 2.9497230052948,
      "metrics/advantage_var": 5468.2666015625,
      "rewards/chosen": 1.6289144009588632,
      "rewards/margins": 1.9584869936047926,
      "rewards/rejected": -0.32957259264592936,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 7.754208564758301,
      "kl": 18.512327194213867,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -42944346.443113774,
      "logits/rejected": -42855497.62091503,
      "logps/chosen": -464.94938248502996,
      "logps/rejected": -359.9943831699346,
      "loss": 0.3545,
      "loss/base_kto": 2.8356099128723145,
      "metrics/advantage_var": 5849.23486328125,
      "rewards/chosen": 1.4158094029226702,
      "rewards/margins": 2.1239714488689785,
      "rewards/rejected": -0.7081620459463082,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 8.134888648986816,
      "kl": 19.836334228515625,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -42694010.56969697,
      "logits/rejected": -42238341.78064516,
      "logps/chosen": -475.8674715909091,
      "logps/rejected": -368.44627016129033,
      "loss": 0.3552,
      "loss/base_kto": 2.841984987258911,
      "metrics/advantage_var": 5558.55859375,
      "rewards/chosen": 1.2543638287168561,
      "rewards/margins": 2.0791629541421335,
      "rewards/rejected": -0.8247991254252772,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 7.4902472496032715,
      "kl": 20.3782958984375,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -42808046.158833064,
      "logits/rejected": -42098033.13423831,
      "logps/chosen": -483.73936385737437,
      "logps/rejected": -379.45880467571646,
      "loss": 0.3512,
      "loss/base_kto": 2.809675693511963,
      "metrics/advantage_var": 6742.35791015625,
      "rewards/chosen": 1.2607057840356564,
      "rewards/margins": 2.351388560301211,
      "rewards/rejected": -1.0906827762655542,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 11.804646492004395,
      "kl": 15.350687026977539,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -43782101.4016,
      "logits/rejected": -43546999.20610687,
      "logps/chosen": -461.6126,
      "logps/rejected": -410.2317748091603,
      "loss": 0.3685,
      "loss/base_kto": 2.9478862285614014,
      "metrics/advantage_var": 7228.64404296875,
      "rewards/chosen": 0.73833515625,
      "rewards/margins": 2.2523628056416984,
      "rewards/rejected": -1.5140276493916984,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 7.086822986602783,
      "kl": 26.88038444519043,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -44641121.61715161,
      "logits/rejected": -44243701.79266348,
      "logps/chosen": -453.5135432618683,
      "logps/rejected": -369.793610446571,
      "loss": 0.3503,
      "loss/base_kto": 2.802610397338867,
      "metrics/advantage_var": 6313.2421875,
      "rewards/chosen": 1.4890733049985643,
      "rewards/margins": 2.2028278646265047,
      "rewards/rejected": -0.7137545596279405,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 10.928313255310059,
      "kl": 26.3825740814209,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -44585356.38709678,
      "logits/rejected": -44008938.51230101,
      "logps/chosen": -493.0664261460102,
      "logps/rejected": -377.9354875180897,
      "loss": 0.3429,
      "loss/base_kto": 2.743035316467285,
      "metrics/advantage_var": 7014.70947265625,
      "rewards/chosen": 1.4837874459491722,
      "rewards/margins": 2.5707175842790386,
      "rewards/rejected": -1.0869301383298662,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 8.422404289245605,
      "kl": 4.848169803619385,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -43597803.921568625,
      "logits/rejected": -42906237.7005988,
      "logps/chosen": -452.0413602941176,
      "logps/rejected": -378.8748128742515,
      "loss": 0.3572,
      "loss/base_kto": 2.8578033447265625,
      "metrics/advantage_var": 8165.21875,
      "rewards/chosen": 0.0707818324269812,
      "rewards/margins": 2.411239406049268,
      "rewards/rejected": -2.340457573622287,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 6.798046588897705,
      "kl": 18.345449447631836,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -43084744.38338658,
      "logits/rejected": -43517958.26299694,
      "logps/chosen": -465.7491513578275,
      "logps/rejected": -387.7618501529052,
      "loss": 0.3413,
      "loss/base_kto": 2.7300755977630615,
      "metrics/advantage_var": 7656.23583984375,
      "rewards/chosen": 1.2981610282922325,
      "rewards/margins": 2.6447027800721443,
      "rewards/rejected": -1.346541751779912,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 7.999556064605713,
      "kl": 21.403776168823242,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -45108135.69742814,
      "logits/rejected": -44685381.47980614,
      "logps/chosen": -470.1483547655068,
      "logps/rejected": -384.3038166397415,
      "loss": 0.3412,
      "loss/base_kto": 2.729660749435425,
      "metrics/advantage_var": 8261.6982421875,
      "rewards/chosen": 1.6109587745839637,
      "rewards/margins": 2.6943660271776126,
      "rewards/rejected": -1.0834072525936491,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 9.10338306427002,
      "kl": 17.625547409057617,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -45012449.88235294,
      "logits/rejected": -44789968.47904192,
      "logps/chosen": -475.2092524509804,
      "logps/rejected": -393.1947979041916,
      "loss": 0.3477,
      "loss/base_kto": 2.7819411754608154,
      "metrics/advantage_var": 9123.5576171875,
      "rewards/chosen": 1.1569894030203227,
      "rewards/margins": 2.65279250506818,
      "rewards/rejected": -1.4958031020478575,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 7.185104846954346,
      "kl": 39.15957260131836,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -45259307.60883281,
      "logits/rejected": -45467172.45820434,
      "logps/chosen": -475.2610410094637,
      "logps/rejected": -367.40707236842104,
      "loss": 0.3101,
      "loss/base_kto": 2.4805924892425537,
      "metrics/advantage_var": 9365.8955078125,
      "rewards/chosen": 2.171943929293178,
      "rewards/margins": 3.200332457556046,
      "rewards/rejected": -1.0283885282628678,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 8.035014152526855,
      "kl": 32.20772171020508,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -46044747.64541213,
      "logits/rejected": -45784483.566718996,
      "logps/chosen": -448.41854587869364,
      "logps/rejected": -384.4145898744113,
      "loss": 0.3403,
      "loss/base_kto": 2.7223238945007324,
      "metrics/advantage_var": 8992.5966796875,
      "rewards/chosen": 1.9568911444158243,
      "rewards/margins": 2.6815834007504495,
      "rewards/rejected": -0.7246922563346252,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 8.956879615783691,
      "kl": 37.35211181640625,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -46692901.62300319,
      "logits/rejected": -47316591.16819572,
      "logps/chosen": -488.006589456869,
      "logps/rejected": -369.25248470948014,
      "loss": 0.3429,
      "loss/base_kto": 2.7429778575897217,
      "metrics/advantage_var": 9564.0224609375,
      "rewards/chosen": 1.9750145860373403,
      "rewards/margins": 2.805674971495817,
      "rewards/rejected": -0.8306603854584766,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.948576927185059,
      "kl": 21.236835479736328,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -45227771.52305246,
      "logits/rejected": -44795060.89093702,
      "logps/chosen": -474.6163056438792,
      "logps/rejected": -375.4060579877112,
      "loss": 0.3341,
      "loss/base_kto": 2.6726529598236084,
      "metrics/advantage_var": 10510.228515625,
      "rewards/chosen": 1.8970902629421702,
      "rewards/margins": 3.1445029723555726,
      "rewards/rejected": -1.2474127094134024,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 7.4264655113220215,
      "kl": 23.38250160217285,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -49304830.78288431,
      "logits/rejected": -48731349.0046225,
      "logps/chosen": -486.996236133122,
      "logps/rejected": -385.19794876733437,
      "loss": 0.3096,
      "loss/base_kto": 2.4769375324249268,
      "metrics/advantage_var": 10881.8251953125,
      "rewards/chosen": 1.862114714367076,
      "rewards/margins": 3.4746163319397843,
      "rewards/rejected": -1.612501617572708,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 8.825889587402344,
      "kl": 29.91910171508789,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -46931557.75394322,
      "logits/rejected": -47508308.80495356,
      "logps/chosen": -453.5905954258675,
      "logps/rejected": -374.0875096749226,
      "loss": 0.3183,
      "loss/base_kto": 2.546102285385132,
      "metrics/advantage_var": 11163.6689453125,
      "rewards/chosen": 1.95512703014097,
      "rewards/margins": 3.2599436865784215,
      "rewards/rejected": -1.3048166564374517,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 9.196476936340332,
      "kl": 32.796630859375,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -47745985.30612245,
      "logits/rejected": -47962675.75738725,
      "logps/chosen": -449.0790816326531,
      "logps/rejected": -392.8316728227061,
      "loss": 0.3159,
      "loss/base_kto": 2.527033567428589,
      "metrics/advantage_var": 9498.0673828125,
      "rewards/chosen": 2.36832139331093,
      "rewards/margins": 3.255964498445553,
      "rewards/rejected": -0.8876431051346229,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 8.986213684082031,
      "kl": 26.47309684753418,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -47960802.30721003,
      "logits/rejected": -48642245.78193147,
      "logps/chosen": -485.68696120689657,
      "logps/rejected": -394.78448695482865,
      "loss": 0.3287,
      "loss/base_kto": 2.6298370361328125,
      "metrics/advantage_var": 10601.69921875,
      "rewards/chosen": 1.781288649221199,
      "rewards/margins": 3.213277897146384,
      "rewards/rejected": -1.431989247925185,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 7.825324058532715,
      "kl": 47.7818717956543,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -47315026.17283951,
      "logits/rejected": -47319033.51898734,
      "logps/chosen": -487.53462577160496,
      "logps/rejected": -359.762089596519,
      "loss": 0.319,
      "loss/base_kto": 2.551891803741455,
      "metrics/advantage_var": 9312.2255859375,
      "rewards/chosen": 2.5791514832296487,
      "rewards/margins": 3.1936893621111757,
      "rewards/rejected": -0.6145378788815269,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 6.205320835113525,
      "kl": 39.29474639892578,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -46901672.36036036,
      "logits/rejected": -47044868.169381104,
      "logps/chosen": -454.5004222972973,
      "logps/rejected": -378.09731270358304,
      "loss": 0.3192,
      "loss/base_kto": 2.5539467334747314,
      "metrics/advantage_var": 10707.734375,
      "rewards/chosen": 2.507337965406813,
      "rewards/margins": 3.290744872280286,
      "rewards/rejected": -0.7834069068734731,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 5.915700435638428,
      "kl": 25.44234848022461,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -48082336.28486647,
      "logits/rejected": -47447279.94719472,
      "logps/chosen": -454.9650871661721,
      "logps/rejected": -394.4505466171617,
      "loss": 0.3307,
      "loss/base_kto": 2.645360231399536,
      "metrics/advantage_var": 10400.4521484375,
      "rewards/chosen": 1.958246338615078,
      "rewards/margins": 3.180263028809488,
      "rewards/rejected": -1.22201669019441,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 8.638898849487305,
      "kl": 44.32819747924805,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -48419350.92537314,
      "logits/rejected": -47859839.58032787,
      "logps/chosen": -463.88768656716417,
      "logps/rejected": -381.01787909836065,
      "loss": 0.3098,
      "loss/base_kto": 2.478761672973633,
      "metrics/advantage_var": 10415.7021484375,
      "rewards/chosen": 2.7349866269239738,
      "rewards/margins": 3.2418057547415966,
      "rewards/rejected": -0.506819127817623,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 8.295904159545898,
      "kl": 15.3886079788208,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -49457854.996865205,
      "logits/rejected": -49579368.47352025,
      "logps/chosen": -491.1561520376176,
      "logps/rejected": -419.11808800623055,
      "loss": 0.3314,
      "loss/base_kto": 2.651231288909912,
      "metrics/advantage_var": 13674.0029296875,
      "rewards/chosen": 1.5671528305005877,
      "rewards/margins": 3.690570289673582,
      "rewards/rejected": -2.1234174591729946,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 8.249643325805664,
      "kl": 36.11222457885742,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -47823306.27160494,
      "logits/rejected": -48412691.44303797,
      "logps/chosen": -468.3174672067901,
      "logps/rejected": -374.06002768987344,
      "loss": 0.3015,
      "loss/base_kto": 2.412252426147461,
      "metrics/advantage_var": 12290.0556640625,
      "rewards/chosen": 2.395194536373939,
      "rewards/margins": 3.7916094742262434,
      "rewards/rejected": -1.396414937852304,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 8.483682632446289,
      "kl": 19.95432472229004,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -49081974.15384615,
      "logits/rejected": -47577837.26829268,
      "logps/chosen": -472.70347556089746,
      "logps/rejected": -404.7442835365854,
      "loss": 0.3209,
      "loss/base_kto": 2.5670926570892334,
      "metrics/advantage_var": 13209.6298828125,
      "rewards/chosen": 1.3851619622646234,
      "rewards/margins": 3.574410582274627,
      "rewards/rejected": -2.189248620010004,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 6.4333648681640625,
      "kl": 21.872238159179688,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -47805700.13509545,
      "logits/rejected": -47969828.75459099,
      "logps/chosen": -473.998669236417,
      "logps/rejected": -388.78177170283806,
      "loss": 0.3352,
      "loss/base_kto": 2.6814286708831787,
      "metrics/advantage_var": 14399.0498046875,
      "rewards/chosen": 1.7137072951082966,
      "rewards/margins": 3.7160327447509305,
      "rewards/rejected": -2.0023254496426337,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 7.0378828048706055,
      "kl": 44.050533294677734,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -47909045.75594295,
      "logits/rejected": -47041594.55950541,
      "logps/chosen": -446.0818145800317,
      "logps/rejected": -369.4335394126739,
      "loss": 0.3069,
      "loss/base_kto": 2.455458164215088,
      "metrics/advantage_var": 12130.0087890625,
      "rewards/chosen": 2.7584470334786055,
      "rewards/margins": 3.6354136484745,
      "rewards/rejected": -0.8769666149958946,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 7.128819942474365,
      "kl": 8.479227066040039,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -49175786.102643855,
      "logits/rejected": -49501764.320251174,
      "logps/chosen": -472.32008164852255,
      "logps/rejected": -406.70111361852435,
      "loss": 0.2763,
      "loss/base_kto": 2.2100369930267334,
      "metrics/advantage_var": 14933.9052734375,
      "rewards/chosen": 1.5404610878572123,
      "rewards/margins": 4.540625509094458,
      "rewards/rejected": -3.000164421237245,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 6.075253486633301,
      "kl": 22.863582611083984,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -48807158.518518515,
      "logits/rejected": -49843264.81012658,
      "logps/chosen": -472.3656925154321,
      "logps/rejected": -378.02037183544303,
      "loss": 0.2606,
      "loss/base_kto": 2.084904670715332,
      "metrics/advantage_var": 14587.2138671875,
      "rewards/chosen": 2.5730581401306907,
      "rewards/margins": 4.949998130983143,
      "rewards/rejected": -2.3769399908524527,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 8.257942199707031,
      "kl": 24.318510055541992,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -50012448.0,
      "logits/rejected": -49810028.8,
      "logps/chosen": -448.85087890625,
      "logps/rejected": -413.91298828125,
      "loss": 0.2571,
      "loss/base_kto": 2.0571820735931396,
      "metrics/advantage_var": 14592.2490234375,
      "rewards/chosen": 2.8325862884521484,
      "rewards/margins": 4.869905662536621,
      "rewards/rejected": -2.037319374084473,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 9.605294227600098,
      "kl": 9.2108793258667,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -49713686.5408805,
      "logits/rejected": -50184201.54037267,
      "logps/chosen": -483.2286261792453,
      "logps/rejected": -399.96535326086956,
      "loss": 0.2791,
      "loss/base_kto": 2.2326481342315674,
      "metrics/advantage_var": 14464.5341796875,
      "rewards/chosen": 2.036675027331466,
      "rewards/margins": 4.535472141922887,
      "rewards/rejected": -2.498797114591421,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 8.508650779724121,
      "kl": 23.149219512939453,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -48690886.37059724,
      "logits/rejected": -49778936.24242424,
      "logps/chosen": -426.35892036753444,
      "logps/rejected": -397.50652910685807,
      "loss": 0.2715,
      "loss/base_kto": 2.1716575622558594,
      "metrics/advantage_var": 13795.7470703125,
      "rewards/chosen": 2.4378465824799003,
      "rewards/margins": 4.29703390180805,
      "rewards/rejected": -1.8591873193281498,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 9.159284591674805,
      "kl": 8.661831855773926,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -49975672.1632653,
      "logits/rejected": -49788434.31415241,
      "logps/chosen": -456.75132456828885,
      "logps/rejected": -394.9606580482115,
      "loss": 0.271,
      "loss/base_kto": 2.1682369709014893,
      "metrics/advantage_var": 14302.1611328125,
      "rewards/chosen": 1.9851262019230769,
      "rewards/margins": 4.683231321358341,
      "rewards/rejected": -2.6981051194352643,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 8.35326099395752,
      "kl": 20.57655906677246,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -48238076.18479881,
      "logits/rejected": -48799912.14449918,
      "logps/chosen": -457.77868852459017,
      "logps/rejected": -390.96544027093597,
      "loss": 0.262,
      "loss/base_kto": 2.096139669418335,
      "metrics/advantage_var": 14034.4755859375,
      "rewards/chosen": 2.3559140974292103,
      "rewards/margins": 4.828909097557494,
      "rewards/rejected": -2.472995000128284,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 7.199108123779297,
      "kl": 16.808950424194336,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -50340605.72106825,
      "logits/rejected": -49781594.40264026,
      "logps/chosen": -470.5954191394659,
      "logps/rejected": -382.4571730610561,
      "loss": 0.2585,
      "loss/base_kto": 2.068336248397827,
      "metrics/advantage_var": 13719.765625,
      "rewards/chosen": 2.7827637443202895,
      "rewards/margins": 4.609518172692814,
      "rewards/rejected": -1.8267544283725248,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 7.581564426422119,
      "kl": 20.133108139038086,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -50324385.42675159,
      "logits/rejected": -49409947.48466258,
      "logps/chosen": -476.8251393312102,
      "logps/rejected": -413.8957055214724,
      "loss": 0.2575,
      "loss/base_kto": 2.059846878051758,
      "metrics/advantage_var": 14377.228515625,
      "rewards/chosen": 2.26127595354797,
      "rewards/margins": 5.040482024711217,
      "rewards/rejected": -2.7792060711632476,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 6.550929069519043,
      "kl": 18.76786231994629,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -50265547.880239524,
      "logits/rejected": -51629383.94771242,
      "logps/chosen": -467.468001497006,
      "logps/rejected": -375.1931168300654,
      "loss": 0.2578,
      "loss/base_kto": 2.0624940395355225,
      "metrics/advantage_var": 13318.0908203125,
      "rewards/chosen": 2.5435257414857784,
      "rewards/margins": 4.6974037477664154,
      "rewards/rejected": -2.153878006280637,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 7.907017230987549,
      "kl": 26.3935604095459,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -49740227.99391172,
      "logits/rejected": -49904444.40449438,
      "logps/chosen": -470.5619292237443,
      "logps/rejected": -384.57641954253614,
      "loss": 0.2526,
      "loss/base_kto": 2.020489454269409,
      "metrics/advantage_var": 14414.7890625,
      "rewards/chosen": 2.637302903824201,
      "rewards/margins": 4.767513523118442,
      "rewards/rejected": -2.1302106192942416,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 11.054191589355469,
      "kl": 25.65165138244629,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -49833681.7833595,
      "logits/rejected": -48465050.47589424,
      "logps/chosen": -455.166601255887,
      "logps/rejected": -377.271967340591,
      "loss": 0.2636,
      "loss/base_kto": 2.1088874340057373,
      "metrics/advantage_var": 15088.5439453125,
      "rewards/chosen": 2.379607244039443,
      "rewards/margins": 4.768339869428246,
      "rewards/rejected": -2.3887326253888026,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 8.053932189941406,
      "kl": 21.888080596923828,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -49561057.11746032,
      "logits/rejected": -49377758.916923076,
      "logps/chosen": -458.37624007936506,
      "logps/rejected": -390.8678125,
      "loss": 0.2553,
      "loss/base_kto": 2.0426578521728516,
      "metrics/advantage_var": 13139.5380859375,
      "rewards/chosen": 2.727308679005456,
      "rewards/margins": 4.735687960856417,
      "rewards/rejected": -2.0083792818509614,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 7.707869529724121,
      "kl": 17.26511001586914,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -52352816.231884055,
      "logits/rejected": -50427248.26707132,
      "logps/chosen": -503.0871578099839,
      "logps/rejected": -401.6888277693475,
      "loss": 0.2615,
      "loss/base_kto": 2.0917253494262695,
      "metrics/advantage_var": 13824.634765625,
      "rewards/chosen": 2.185942374949678,
      "rewards/margins": 4.890966411131393,
      "rewards/rejected": -2.705024036181715,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 8.309268951416016,
      "kl": 18.66203498840332,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -49922970.10114192,
      "logits/rejected": -50169263.40029985,
      "logps/chosen": -443.8905995106036,
      "logps/rejected": -399.65423538230885,
      "loss": 0.2728,
      "loss/base_kto": 2.182373523712158,
      "metrics/advantage_var": 13755.0751953125,
      "rewards/chosen": 2.116479173570045,
      "rewards/margins": 4.563443923616896,
      "rewards/rejected": -2.4469647500468517,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 8.86742115020752,
      "kl": 19.643617630004883,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -49900602.32911392,
      "logits/rejected": -50785434.86419753,
      "logps/chosen": -470.07352650316454,
      "logps/rejected": -404.669319058642,
      "loss": 0.2568,
      "loss/base_kto": 2.0543911457061768,
      "metrics/advantage_var": 13535.3701171875,
      "rewards/chosen": 2.5262949496884888,
      "rewards/margins": 4.746610388418234,
      "rewards/rejected": -2.2203154387297452,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 8.328465461730957,
      "kl": 24.861005783081055,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -48342120.818897635,
      "logits/rejected": -49282109.91627907,
      "logps/chosen": -436.20187007874017,
      "logps/rejected": -389.5715600775194,
      "loss": 0.2647,
      "loss/base_kto": 2.1176109313964844,
      "metrics/advantage_var": 13335.603515625,
      "rewards/chosen": 2.118761534202756,
      "rewards/margins": 4.478910024694519,
      "rewards/rejected": -2.360148490491764,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 8.225176811218262,
      "kl": 16.95220375061035,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -48880521.66409861,
      "logits/rejected": -48887918.3518225,
      "logps/chosen": -454.7324730354391,
      "logps/rejected": -381.3492224643423,
      "loss": 0.2655,
      "loss/base_kto": 2.123788833618164,
      "metrics/advantage_var": 13949.244140625,
      "rewards/chosen": 2.2439794319987483,
      "rewards/margins": 4.5772557604951825,
      "rewards/rejected": -2.333276328496434,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 8.387892723083496,
      "kl": 21.560420989990234,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -51606187.77309562,
      "logits/rejected": -51349188.92307692,
      "logps/chosen": -462.6977816045381,
      "logps/rejected": -391.9108691553545,
      "loss": 0.2572,
      "loss/base_kto": 2.0576674938201904,
      "metrics/advantage_var": 14644.9970703125,
      "rewards/chosen": 2.266873005723258,
      "rewards/margins": 4.8239275282534235,
      "rewards/rejected": -2.5570545225301657,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 7.858390808105469,
      "kl": 15.610109329223633,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -50757154.027820714,
      "logits/rejected": -51382676.423380725,
      "logps/chosen": -465.051970633694,
      "logps/rejected": -391.02142575039494,
      "loss": 0.2454,
      "loss/base_kto": 1.9634945392608643,
      "metrics/advantage_var": 15700.1689453125,
      "rewards/chosen": 2.402788636857612,
      "rewards/margins": 5.1408103187049665,
      "rewards/rejected": -2.738021681847354,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 9.813386917114258,
      "kl": 25.023880004882812,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -50744911.50310559,
      "logits/rejected": -52875898.364779875,
      "logps/chosen": -453.86655667701865,
      "logps/rejected": -409.5614435927673,
      "loss": 0.2585,
      "loss/base_kto": 2.068122625350952,
      "metrics/advantage_var": 14208.1142578125,
      "rewards/chosen": 2.5509571525621118,
      "rewards/margins": 4.723011035472882,
      "rewards/rejected": -2.1720538829107703,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 9.781533241271973,
      "kl": 18.06972312927246,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -50727252.22149837,
      "logits/rejected": -50698756.61261261,
      "logps/chosen": -470.33092426710095,
      "logps/rejected": -399.44237987987987,
      "loss": 0.2447,
      "loss/base_kto": 1.9572668075561523,
      "metrics/advantage_var": 14845.8779296875,
      "rewards/chosen": 2.0454725830873373,
      "rewards/margins": 5.094310972257757,
      "rewards/rejected": -3.0488383891704203,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 7.7680559158325195,
      "kl": 25.265140533447266,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -50514493.94180705,
      "logits/rejected": -51570998.303030305,
      "logps/chosen": -452.76732388973966,
      "logps/rejected": -400.32675438596493,
      "loss": 0.2464,
      "loss/base_kto": 1.9708786010742188,
      "metrics/advantage_var": 14639.181640625,
      "rewards/chosen": 2.538979686602699,
      "rewards/margins": 4.9464379296001075,
      "rewards/rejected": -2.4074582429974085,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 9.849201202392578,
      "kl": 30.48682975769043,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -51322437.451104105,
      "logits/rejected": -52218033.53560372,
      "logps/chosen": -479.2074625394322,
      "logps/rejected": -386.64439821981426,
      "loss": 0.2403,
      "loss/base_kto": 1.9221832752227783,
      "metrics/advantage_var": 13286.900390625,
      "rewards/chosen": 3.028706162509858,
      "rewards/margins": 4.989388939938264,
      "rewards/rejected": -1.9606827774284055,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 13.191734313964844,
      "kl": 14.286864280700684,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -51283517.44,
      "logits/rejected": -50448852.114285715,
      "logps/chosen": -453.09653846153844,
      "logps/rejected": -413.8794642857143,
      "loss": 0.2527,
      "loss/base_kto": 2.0218100547790527,
      "metrics/advantage_var": 14516.5849609375,
      "rewards/chosen": 2.2963213641826923,
      "rewards/margins": 4.983963198260073,
      "rewards/rejected": -2.687641834077381,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 8.107991218566895,
      "kl": 22.3076114654541,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -51279175.19526627,
      "logits/rejected": -50293064.90066225,
      "logps/chosen": -459.0122503698225,
      "logps/rejected": -396.8199503311258,
      "loss": 0.2464,
      "loss/base_kto": 1.97113037109375,
      "metrics/advantage_var": 14052.2333984375,
      "rewards/chosen": 2.767571985368898,
      "rewards/margins": 4.981228783472686,
      "rewards/rejected": -2.2136567981037873,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 7.876888751983643,
      "kl": 23.63214683532715,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -51858099.57585645,
      "logits/rejected": -50774759.05247376,
      "logps/chosen": -460.11154159869494,
      "logps/rejected": -404.99058283358323,
      "loss": 0.2449,
      "loss/base_kto": 1.9594535827636719,
      "metrics/advantage_var": 14345.2470703125,
      "rewards/chosen": 2.6476365753721454,
      "rewards/margins": 5.056325968468566,
      "rewards/rejected": -2.4086893930964206,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 8.337098121643066,
      "kl": 19.482269287109375,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -49769627.5443038,
      "logits/rejected": -50932467.358024694,
      "logps/chosen": -467.47542523734177,
      "logps/rejected": -385.0291521990741,
      "loss": 0.2689,
      "loss/base_kto": 2.151118040084839,
      "metrics/advantage_var": 15071.4501953125,
      "rewards/chosen": 2.0300922635235366,
      "rewards/margins": 4.762695736951391,
      "rewards/rejected": -2.732603473427855,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 8.486111640930176,
      "kl": 18.892194747924805,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -51199834.63354037,
      "logits/rejected": -51612607.597484276,
      "logps/chosen": -451.39314829192546,
      "logps/rejected": -402.25515919811323,
      "loss": 0.2552,
      "loss/base_kto": 2.041901111602783,
      "metrics/advantage_var": 13988.0927734375,
      "rewards/chosen": 2.632206318541343,
      "rewards/margins": 4.877813322766972,
      "rewards/rejected": -2.245607004225629,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 5.810118198394775,
      "kl": 18.350488662719727,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -50581396.377125196,
      "logits/rejected": -51894417.59241706,
      "logps/chosen": -479.9468701700155,
      "logps/rejected": -409.77739928909955,
      "loss": 0.2533,
      "loss/base_kto": 2.0267603397369385,
      "metrics/advantage_var": 15810.369140625,
      "rewards/chosen": 2.52049875627898,
      "rewards/margins": 5.100344118455618,
      "rewards/rejected": -2.579845362176639,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 8.731647491455078,
      "kl": 19.821945190429688,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -50769305.271268055,
      "logits/rejected": -50592647.98782344,
      "logps/chosen": -472.839386035313,
      "logps/rejected": -380.9224457762557,
      "loss": 0.2307,
      "loss/base_kto": 1.8458435535430908,
      "metrics/advantage_var": 15380.0625,
      "rewards/chosen": 2.470528542899779,
      "rewards/margins": 5.446726689957903,
      "rewards/rejected": -2.976198147058124,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 12.344839096069336,
      "kl": 20.61775016784668,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -51296505.91442155,
      "logits/rejected": -52072858.23112481,
      "logps/chosen": -460.2765946909667,
      "logps/rejected": -402.63467835130973,
      "loss": 0.262,
      "loss/base_kto": 2.095961093902588,
      "metrics/advantage_var": 14151.8955078125,
      "rewards/chosen": 2.6214164334885104,
      "rewards/margins": 4.732221668706153,
      "rewards/rejected": -2.1108052352176427,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 10.104900360107422,
      "kl": 21.741413116455078,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -49976328.16586922,
      "logits/rejected": -50593735.5467075,
      "logps/chosen": -460.589812599681,
      "logps/rejected": -404.05915007656967,
      "loss": 0.2625,
      "loss/base_kto": 2.1003079414367676,
      "metrics/advantage_var": 14812.681640625,
      "rewards/chosen": 2.3763505610172446,
      "rewards/margins": 4.787454585112766,
      "rewards/rejected": -2.4111040240955206,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 9.464112281799316,
      "kl": 26.843799591064453,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -51003599.06477093,
      "logits/rejected": -50613699.06646059,
      "logps/chosen": -435.2705371248025,
      "logps/rejected": -421.655477202473,
      "loss": 0.2546,
      "loss/base_kto": 2.036956548690796,
      "metrics/advantage_var": 13646.5810546875,
      "rewards/chosen": 2.445724800676343,
      "rewards/margins": 4.735024837806849,
      "rewards/rejected": -2.289300037130506,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 8.945937156677246,
      "kl": 18.546194076538086,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -51285871.35794543,
      "logits/rejected": -50765618.264840186,
      "logps/chosen": -477.2027989566613,
      "logps/rejected": -391.77960901826486,
      "loss": 0.2609,
      "loss/base_kto": 2.0872085094451904,
      "metrics/advantage_var": 13895.5595703125,
      "rewards/chosen": 2.4925454814782304,
      "rewards/margins": 4.680363570329067,
      "rewards/rejected": -2.187818088850837,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 8.474348068237305,
      "kl": 16.511445999145508,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -51369294.923317686,
      "logits/rejected": -51402240.79875195,
      "logps/chosen": -462.3911384976526,
      "logps/rejected": -414.3921606864275,
      "loss": 0.2603,
      "loss/base_kto": 2.0823142528533936,
      "metrics/advantage_var": 14479.283203125,
      "rewards/chosen": 2.201315914148083,
      "rewards/margins": 4.8964726364326,
      "rewards/rejected": -2.6951567222845165,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 7.912738800048828,
      "kl": 25.281246185302734,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -52288512.0,
      "logits/rejected": -53545018.32911392,
      "logps/chosen": -473.1086516203704,
      "logps/rejected": -385.10087025316454,
      "loss": 0.2629,
      "loss/base_kto": 2.1033740043640137,
      "metrics/advantage_var": 12775.525390625,
      "rewards/chosen": 2.3238201376832563,
      "rewards/margins": 4.495245486278984,
      "rewards/rejected": -2.171425348595728,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 8.972837448120117,
      "kl": 24.49571418762207,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -50361548.15698587,
      "logits/rejected": -51107225.281493,
      "logps/chosen": -474.48606750392463,
      "logps/rejected": -402.76513899688956,
      "loss": 0.2375,
      "loss/base_kto": 1.9002567529678345,
      "metrics/advantage_var": 14855.8740234375,
      "rewards/chosen": 2.919538033813285,
      "rewards/margins": 5.343848744714334,
      "rewards/rejected": -2.4243107109010498,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 7.34293794631958,
      "kl": 18.954492568969727,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -50771734.56534954,
      "logits/rejected": -51986418.787096776,
      "logps/chosen": -466.31449468085106,
      "logps/rejected": -376.00982862903226,
      "loss": 0.2148,
      "loss/base_kto": 1.7182190418243408,
      "metrics/advantage_var": 15409.087890625,
      "rewards/chosen": 2.7397511027141905,
      "rewards/margins": 5.701999283472759,
      "rewards/rejected": -2.9622481807585683,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 5.555119037628174,
      "kl": 14.898538589477539,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -50034223.87321711,
      "logits/rejected": -50908884.21571649,
      "logps/chosen": -441.70032686212363,
      "logps/rejected": -415.6199441448382,
      "loss": 0.2126,
      "loss/base_kto": 1.7008146047592163,
      "metrics/advantage_var": 13632.4970703125,
      "rewards/chosen": 2.8393784899341323,
      "rewards/margins": 5.480414842435577,
      "rewards/rejected": -2.6410363525014446,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 7.122386455535889,
      "kl": 17.392332077026367,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -52030957.14874816,
      "logits/rejected": -52453399.85357737,
      "logps/chosen": -468.22505522827686,
      "logps/rejected": -391.0408433860233,
      "loss": 0.2097,
      "loss/base_kto": 1.6779264211654663,
      "metrics/advantage_var": 14165.556640625,
      "rewards/chosen": 2.5735638131098124,
      "rewards/margins": 5.60465457330116,
      "rewards/rejected": -3.031090760191348,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 8.374948501586914,
      "kl": 18.164281845092773,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -54053437.923303835,
      "logits/rejected": -53630322.81727575,
      "logps/chosen": -474.5418510324484,
      "logps/rejected": -419.30411648671094,
      "loss": 0.2087,
      "loss/base_kto": 1.6698551177978516,
      "metrics/advantage_var": 13417.6748046875,
      "rewards/chosen": 2.791623095847161,
      "rewards/margins": 5.391831061482543,
      "rewards/rejected": -2.600207965635382,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 9.291142463684082,
      "kl": 19.005338668823242,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -52156198.78787879,
      "logits/rejected": -53144311.741935484,
      "logps/chosen": -463.5467803030303,
      "logps/rejected": -409.1705141129032,
      "loss": 0.2125,
      "loss/base_kto": 1.6997795104980469,
      "metrics/advantage_var": 13266.9892578125,
      "rewards/chosen": 2.6969493519176138,
      "rewards/margins": 5.313385528832937,
      "rewards/rejected": -2.6164361769153226,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 8.366868019104004,
      "kl": 14.552406311035156,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -52351320.3539823,
      "logits/rejected": -52654488.239202656,
      "logps/chosen": -473.98101032448375,
      "logps/rejected": -418.17794850498336,
      "loss": 0.2051,
      "loss/base_kto": 1.6408668756484985,
      "metrics/advantage_var": 16153.1005859375,
      "rewards/chosen": 2.771032462781158,
      "rewards/margins": 5.9631924259897335,
      "rewards/rejected": -3.1921599632085758,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 7.207845211029053,
      "kl": 21.19478416442871,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -52147415.31942215,
      "logits/rejected": -52342840.10958904,
      "logps/chosen": -453.4293739967897,
      "logps/rejected": -401.2667903348554,
      "loss": 0.2149,
      "loss/base_kto": 1.718897819519043,
      "metrics/advantage_var": 14809.8955078125,
      "rewards/chosen": 2.812826435217697,
      "rewards/margins": 5.515159371683165,
      "rewards/rejected": -2.702332936465468,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 7.801844596862793,
      "kl": 19.278751373291016,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51519835.172628306,
      "logits/rejected": -50894349.66405024,
      "logps/chosen": -465.26360808709177,
      "logps/rejected": -372.2115875196232,
      "loss": 0.2053,
      "loss/base_kto": 1.642542839050293,
      "metrics/advantage_var": 12793.9970703125,
      "rewards/chosen": 2.8394145105341173,
      "rewards/margins": 5.430397711206213,
      "rewards/rejected": -2.590983200672096,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 9.775045394897461,
      "kl": 20.726667404174805,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -51509880.662420385,
      "logits/rejected": -51625760.98159509,
      "logps/chosen": -422.7342257165605,
      "logps/rejected": -416.6057323619632,
      "loss": 0.2219,
      "loss/base_kto": 1.7754486799240112,
      "metrics/advantage_var": 14590.0625,
      "rewards/chosen": 2.514944283066282,
      "rewards/margins": 5.329098597914825,
      "rewards/rejected": -2.814154314848543,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 7.695766925811768,
      "kl": 18.871171951293945,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -51663503.74763407,
      "logits/rejected": -51968903.52941176,
      "logps/chosen": -447.028785488959,
      "logps/rejected": -400.92431791795667,
      "loss": 0.2171,
      "loss/base_kto": 1.7367476224899292,
      "metrics/advantage_var": 15574.3642578125,
      "rewards/chosen": 2.6369784863589314,
      "rewards/margins": 5.576990186968452,
      "rewards/rejected": -2.9400117006095203,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 7.314939022064209,
      "kl": 19.452167510986328,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -52745370.752411574,
      "logits/rejected": -53523259.91489362,
      "logps/chosen": -471.878165192926,
      "logps/rejected": -386.12419262917933,
      "loss": 0.1992,
      "loss/base_kto": 1.593440294265747,
      "metrics/advantage_var": 16043.1455078125,
      "rewards/chosen": 2.777079198904743,
      "rewards/margins": 5.964584393386126,
      "rewards/rejected": -3.187505194481383,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.1406474113464355,
      "kl": 11.1013765335083,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -52880064.68817204,
      "logits/rejected": -53137740.10810811,
      "logps/chosen": -439.2087653609831,
      "logps/rejected": -397.033286963434,
      "loss": 0.2087,
      "loss/base_kto": 1.669519066810608,
      "metrics/advantage_var": 14999.6923828125,
      "rewards/chosen": 2.5516636814756146,
      "rewards/margins": 5.534063238373966,
      "rewards/rejected": -2.9823995568983506,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 6.2611212730407715,
      "kl": 14.011194229125977,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -51661088.90282132,
      "logits/rejected": -50629182.205607474,
      "logps/chosen": -458.0458463949843,
      "logps/rejected": -380.8788454049844,
      "loss": 0.2145,
      "loss/base_kto": 1.716043472290039,
      "metrics/advantage_var": 15494.1279296875,
      "rewards/chosen": 2.860801194528801,
      "rewards/margins": 5.573951551689042,
      "rewards/rejected": -2.7131503571602416,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 9.701900482177734,
      "kl": 15.361149787902832,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -51675350.58243451,
      "logits/rejected": -53768451.65134707,
      "logps/chosen": -491.8400423728813,
      "logps/rejected": -406.3640798335975,
      "loss": 0.2072,
      "loss/base_kto": 1.6577314138412476,
      "metrics/advantage_var": 15171.2412109375,
      "rewards/chosen": 2.656865241898594,
      "rewards/margins": 5.613896104987936,
      "rewards/rejected": -2.9570308630893423,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 6.971599102020264,
      "kl": 22.8885555267334,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -52703075.26530612,
      "logits/rejected": -53073008.46242774,
      "logps/chosen": -465.74192176870747,
      "logps/rejected": -418.8117774566474,
      "loss": 0.2017,
      "loss/base_kto": 1.6138557195663452,
      "metrics/advantage_var": 15352.720703125,
      "rewards/chosen": 2.726814944727891,
      "rewards/margins": 5.802486895933635,
      "rewards/rejected": -3.075671951205744,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 13.588480949401855,
      "kl": 13.888775825500488,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -52804236.0832,
      "logits/rejected": -52442382.461068705,
      "logps/chosen": -480.8757,
      "logps/rejected": -408.2770992366412,
      "loss": 0.2267,
      "loss/base_kto": 1.8138843774795532,
      "metrics/advantage_var": 16470.36328125,
      "rewards/chosen": 2.4228767578125,
      "rewards/margins": 5.583166073771469,
      "rewards/rejected": -3.1602893159589693,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 8.179054260253906,
      "kl": 14.419031143188477,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -51960324.16260163,
      "logits/rejected": -52883646.94135338,
      "logps/chosen": -472.74481707317074,
      "logps/rejected": -375.7159539473684,
      "loss": 0.2233,
      "loss/base_kto": 1.7862293720245361,
      "metrics/advantage_var": 14972.625,
      "rewards/chosen": 2.4842916349085367,
      "rewards/margins": 5.570328259673574,
      "rewards/rejected": -3.0860366247650375,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 7.958951950073242,
      "kl": 17.662368774414062,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -51976939.882896766,
      "logits/rejected": -53508605.56576862,
      "logps/chosen": -453.66833590138674,
      "logps/rejected": -387.3939926703645,
      "loss": 0.2156,
      "loss/base_kto": 1.7250031232833862,
      "metrics/advantage_var": 13712.3671875,
      "rewards/chosen": 2.6780703034716873,
      "rewards/margins": 5.375076117965051,
      "rewards/rejected": -2.6970058144933637,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 6.839268207550049,
      "kl": 19.165042877197266,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -51127809.501466274,
      "logits/rejected": -51394149.03010034,
      "logps/chosen": -457.106900659824,
      "logps/rejected": -380.3718122909699,
      "loss": 0.2113,
      "loss/base_kto": 1.6907148361206055,
      "metrics/advantage_var": 14822.4345703125,
      "rewards/chosen": 2.7296326936514848,
      "rewards/margins": 5.532632030634135,
      "rewards/rejected": -2.8029993369826505,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 8.150351524353027,
      "kl": 15.535231590270996,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -50620527.03614458,
      "logits/rejected": -51386268.25974026,
      "logps/chosen": -465.2983810240964,
      "logps/rejected": -405.39891943993507,
      "loss": 0.2084,
      "loss/base_kto": 1.6675866842269897,
      "metrics/advantage_var": 15500.0830078125,
      "rewards/chosen": 2.8343193329960465,
      "rewards/margins": 5.77271345840189,
      "rewards/rejected": -2.938394125405844,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 7.899462699890137,
      "kl": 23.955821990966797,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -53088461.473684214,
      "logits/rejected": -52787882.666666664,
      "logps/chosen": -452.7037417763158,
      "logps/rejected": -396.98456101190476,
      "loss": 0.2092,
      "loss/base_kto": 1.6736135482788086,
      "metrics/advantage_var": 13926.71875,
      "rewards/chosen": 2.632309361508018,
      "rewards/margins": 5.417852894106604,
      "rewards/rejected": -2.7855435325985862,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 7.126053333282471,
      "kl": 23.701719284057617,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -51701858.126738794,
      "logits/rejected": -52832971.02053712,
      "logps/chosen": -450.41373647604325,
      "logps/rejected": -414.1920912322275,
      "loss": 0.2111,
      "loss/base_kto": 1.6890560388565063,
      "metrics/advantage_var": 14298.392578125,
      "rewards/chosen": 2.787808968073802,
      "rewards/margins": 5.523550075646867,
      "rewards/rejected": -2.7357411075730647,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 9.15418529510498,
      "kl": 16.745986938476562,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -52566420.463949844,
      "logits/rejected": -52571710.205607474,
      "logps/chosen": -462.2469631661442,
      "logps/rejected": -392.3098958333333,
      "loss": 0.2137,
      "loss/base_kto": 1.7097481489181519,
      "metrics/advantage_var": 15660.7626953125,
      "rewards/chosen": 2.7696104617701804,
      "rewards/margins": 5.616432184353417,
      "rewards/rejected": -2.846821722583236,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 6.503498077392578,
      "kl": 18.828340530395508,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -52250798.98734177,
      "logits/rejected": -52860463.40740741,
      "logps/chosen": -467.80241297468353,
      "logps/rejected": -398.2588493441358,
      "loss": 0.2111,
      "loss/base_kto": 1.6891224384307861,
      "metrics/advantage_var": 14132.4580078125,
      "rewards/chosen": 2.7144435447982596,
      "rewards/margins": 5.519844628662071,
      "rewards/rejected": -2.8054010838638117,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 8.204117774963379,
      "kl": 16.509063720703125,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -53636154.74108527,
      "logits/rejected": -52339868.42204724,
      "logps/chosen": -467.9199612403101,
      "logps/rejected": -396.39783464566926,
      "loss": 0.2051,
      "loss/base_kto": 1.6408218145370483,
      "metrics/advantage_var": 15810.578125,
      "rewards/chosen": 2.7192469870397287,
      "rewards/margins": 5.853811916911775,
      "rewards/rejected": -3.134564929872047,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 9.347222328186035,
      "kl": 23.602487564086914,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -49904713.728,
      "logits/rejected": -52168260.00610687,
      "logps/chosen": -432.02695,
      "logps/rejected": -413.0551526717557,
      "loss": 0.2353,
      "loss/base_kto": 1.8824348449707031,
      "metrics/advantage_var": 15216.1328125,
      "rewards/chosen": 2.405577734375,
      "rewards/margins": 5.20243633408874,
      "rewards/rejected": -2.7968585997137403,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 7.752729892730713,
      "kl": 17.468936920166016,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -52361436.35443038,
      "logits/rejected": -53112332.641975306,
      "logps/chosen": -439.2915842563291,
      "logps/rejected": -390.4963348765432,
      "loss": 0.2215,
      "loss/base_kto": 1.771693468093872,
      "metrics/advantage_var": 14805.1787109375,
      "rewards/chosen": 2.445236012905459,
      "rewards/margins": 5.354064559876909,
      "rewards/rejected": -2.9088285469714505,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.719018459320068,
      "kl": 15.244857788085938,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -53175440.52852853,
      "logits/rejected": -52999641.64169381,
      "logps/chosen": -479.3825075075075,
      "logps/rejected": -403.0092121335505,
      "loss": 0.1956,
      "loss/base_kto": 1.5649467706680298,
      "metrics/advantage_var": 15854.3876953125,
      "rewards/chosen": 2.8091128237612613,
      "rewards/margins": 6.051144654291086,
      "rewards/rejected": -3.242031830529825,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 8.016149520874023,
      "kl": 13.679033279418945,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -51404154.43478261,
      "logits/rejected": -51658449.3081761,
      "logps/chosen": -453.99427406832297,
      "logps/rejected": -410.07323604559747,
      "loss": 0.2189,
      "loss/base_kto": 1.7515852451324463,
      "metrics/advantage_var": 15158.03125,
      "rewards/chosen": 2.611411716627038,
      "rewards/margins": 5.540414586430988,
      "rewards/rejected": -2.9290028698039503,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 46.4044189453125,
      "kl": 13.768518447875977,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -52033393.86750789,
      "logits/rejected": -50642709.3993808,
      "logps/chosen": -457.0539235015773,
      "logps/rejected": -400.0735294117647,
      "loss": 0.2263,
      "loss/base_kto": 1.8101516962051392,
      "metrics/advantage_var": 15886.5654296875,
      "rewards/chosen": 2.604241115437697,
      "rewards/margins": 5.479861671020128,
      "rewards/rejected": -2.8756205555824303,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 6.903535842895508,
      "kl": 17.980924606323242,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -52394683.94936709,
      "logits/rejected": -53570484.14814815,
      "logps/chosen": -457.2773931962025,
      "logps/rejected": -413.1441936728395,
      "loss": 0.2028,
      "loss/base_kto": 1.6224689483642578,
      "metrics/advantage_var": 15232.3779296875,
      "rewards/chosen": 2.61376528196697,
      "rewards/margins": 5.606020222047024,
      "rewards/rejected": -2.992254940080054,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 8.415434837341309,
      "kl": 16.8980770111084,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -52471641.41626794,
      "logits/rejected": -53361188.851454824,
      "logps/chosen": -456.282745215311,
      "logps/rejected": -405.41213629402756,
      "loss": 0.2095,
      "loss/base_kto": 1.6762361526489258,
      "metrics/advantage_var": 15096.353515625,
      "rewards/chosen": 2.530509011787281,
      "rewards/margins": 5.562195511296757,
      "rewards/rejected": -3.0316864995094757,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 10.105056762695312,
      "kl": 16.623977661132812,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -54464518.29185868,
      "logits/rejected": -53723507.17965024,
      "logps/chosen": -462.01612903225805,
      "logps/rejected": -417.56369236883944,
      "loss": 0.2251,
      "loss/base_kto": 1.801042914390564,
      "metrics/advantage_var": 15147.4130859375,
      "rewards/chosen": 2.5503351589501726,
      "rewards/margins": 5.4693309297669455,
      "rewards/rejected": -2.918995770816773,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 11.006617546081543,
      "kl": 18.074350357055664,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -54645943.63525836,
      "logits/rejected": -52249079.76848874,
      "logps/chosen": -481.00949848024317,
      "logps/rejected": -411.04677451768487,
      "loss": 0.2147,
      "loss/base_kto": 1.7173265218734741,
      "metrics/advantage_var": 16321.2412109375,
      "rewards/chosen": 2.7446502407271085,
      "rewards/margins": 5.886056922487056,
      "rewards/rejected": -3.1414066817599475,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 27.938901901245117,
      "kl": 20.423267364501953,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -52610920.41574803,
      "logits/rejected": -51983479.069767445,
      "logps/chosen": -442.2832677165354,
      "logps/rejected": -414.7684108527132,
      "loss": 0.2157,
      "loss/base_kto": 1.7258579730987549,
      "metrics/advantage_var": 14801.845703125,
      "rewards/chosen": 2.5830262672244095,
      "rewards/margins": 5.560088971621212,
      "rewards/rejected": -2.9770627043968023,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 8.298346519470215,
      "kl": 19.364654541015625,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -53583106.08130081,
      "logits/rejected": -54432458.49022556,
      "logps/chosen": -469.030487804878,
      "logps/rejected": -403.2602913533835,
      "loss": 0.2137,
      "loss/base_kto": 1.7098503112792969,
      "metrics/advantage_var": 16005.7880859375,
      "rewards/chosen": 2.6091197440294716,
      "rewards/margins": 5.647543072859358,
      "rewards/rejected": -3.038423328829887,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.369647026062012,
      "kl": 14.8077974319458,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -51136837.89079563,
      "logits/rejected": -51940427.31768388,
      "logps/chosen": -461.8348771450858,
      "logps/rejected": -388.8576144366197,
      "loss": 0.2349,
      "loss/base_kto": 1.8791515827178955,
      "metrics/advantage_var": 14433.1123046875,
      "rewards/chosen": 2.4296322731815523,
      "rewards/margins": 5.145286386664534,
      "rewards/rejected": -2.715654113482981,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 7.4242634773254395,
      "kl": 20.830245971679688,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -51962868.640253566,
      "logits/rejected": -52445777.25731895,
      "logps/chosen": -459.29576069730587,
      "logps/rejected": -395.55602369029276,
      "loss": 0.1987,
      "loss/base_kto": 1.5892614126205444,
      "metrics/advantage_var": 13583.431640625,
      "rewards/chosen": 2.650785467326169,
      "rewards/margins": 5.544644484203095,
      "rewards/rejected": -2.893859016876926,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.978042558275912e+17,
      "train_loss": 0.27575122105642924,
      "train_runtime": 11063.995,
      "train_samples_per_second": 13.396,
      "train_steps_per_second": 0.209
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.978042558275912e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}