{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 12.468659400939941,
      "kl": 29.8879337310791,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -37374458.35590551,
      "logits/rejected": -38401060.51472868,
      "logps/chosen": -477.19443897637797,
      "logps/rejected": -352.93997093023256,
      "loss": 0.4634,
      "loss/base_kto": 3.707191228866577,
      "metrics/advantage_var": 531.9630126953125,
      "rewards/chosen": 0.5312210683747539,
      "rewards/margins": 0.2895921110255533,
      "rewards/rejected": 0.2416289573492006,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 10.705573081970215,
      "kl": 7.3258957862854,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -37314006.4,
      "logits/rejected": -37823193.6,
      "logps/chosen": -475.64013671875,
      "logps/rejected": -414.605908203125,
      "loss": 0.4578,
      "loss/base_kto": 3.662597417831421,
      "metrics/advantage_var": 619.570068359375,
      "rewards/chosen": 0.08362504839897156,
      "rewards/margins": 0.38068429827690126,
      "rewards/rejected": -0.2970592498779297,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 9.74264907836914,
      "kl": 7.171512126922607,
      "learning_rate": 5.9e-07,
      "logits/chosen": -37287739.57232705,
      "logits/rejected": -39389584.69565217,
      "logps/chosen": -486.7311320754717,
      "logps/rejected": -376.1077251552795,
      "loss": 0.449,
      "loss/base_kto": 3.5922298431396484,
      "metrics/advantage_var": 765.0824584960938,
      "rewards/chosen": 0.156025748582756,
      "rewards/margins": 0.48436920515908555,
      "rewards/rejected": -0.3283434565763296,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 11.212510108947754,
      "kl": 30.199316024780273,
      "learning_rate": 7.9e-07,
      "logits/chosen": -39358154.86792453,
      "logits/rejected": -38897091.57763975,
      "logps/chosen": -496.57139347484275,
      "logps/rejected": -381.1135238742236,
      "loss": 0.4283,
      "loss/base_kto": 3.426403760910034,
      "metrics/advantage_var": 1222.6826171875,
      "rewards/chosen": 0.9127416070902122,
      "rewards/margins": 0.6644606850120145,
      "rewards/rejected": 0.2482809220781978,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 9.89513874053955,
      "kl": 27.885400772094727,
      "learning_rate": 9.9e-07,
      "logits/chosen": -38983038.80373832,
      "logits/rejected": -39081836.338557996,
      "logps/chosen": -500.18433605919006,
      "logps/rejected": -378.2253379702194,
      "loss": 0.4129,
      "loss/base_kto": 3.303199529647827,
      "metrics/advantage_var": 1563.1497802734375,
      "rewards/chosen": 0.9240179804626655,
      "rewards/margins": 0.8425314548822384,
      "rewards/rejected": 0.08148652558042711,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 7.232778072357178,
      "kl": 26.04160499572754,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -39328741.0942813,
      "logits/rejected": -40328264.79620853,
      "logps/chosen": -462.09457109737247,
      "logps/rejected": -370.9913605845182,
      "loss": 0.4159,
      "loss/base_kto": 3.327594041824341,
      "metrics/advantage_var": 2382.521484375,
      "rewards/chosen": 0.8227335297527048,
      "rewards/margins": 0.9499181881949564,
      "rewards/rejected": -0.12718465844225169,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 8.212059020996094,
      "kl": 10.217510223388672,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -39942811.00917431,
      "logits/rejected": -40622629.62300319,
      "logps/chosen": -475.33371559633025,
      "logps/rejected": -390.1128943690096,
      "loss": 0.4073,
      "loss/base_kto": 3.258467197418213,
      "metrics/advantage_var": 2937.027099609375,
      "rewards/chosen": 0.5755939075341647,
      "rewards/margins": 1.1537541051242706,
      "rewards/rejected": -0.5781601975901058,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 7.369364261627197,
      "kl": 15.60644817352295,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -39963147.02769231,
      "logits/rejected": -40619862.95873016,
      "logps/chosen": -486.4833173076923,
      "logps/rejected": -367.47792658730157,
      "loss": 0.376,
      "loss/base_kto": 3.008207321166992,
      "metrics/advantage_var": 3660.644287109375,
      "rewards/chosen": 0.8883199368990384,
      "rewards/margins": 1.604301556597699,
      "rewards/rejected": -0.7159816196986607,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 8.021268844604492,
      "kl": 29.99273681640625,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -39923759.92511701,
      "logits/rejected": -40120464.225352116,
      "logps/chosen": -513.3507215288612,
      "logps/rejected": -368.61678403755866,
      "loss": 0.3689,
      "loss/base_kto": 2.9508259296417236,
      "metrics/advantage_var": 4051.784423828125,
      "rewards/chosen": 1.4986730328588143,
      "rewards/margins": 1.6529032969165767,
      "rewards/rejected": -0.15423026405776238,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 8.382246971130371,
      "kl": 33.228641510009766,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -40510072.13858268,
      "logits/rejected": -41386263.417054266,
      "logps/chosen": -448.5313976377953,
      "logps/rejected": -377.7218265503876,
      "loss": 0.3599,
      "loss/base_kto": 2.8793251514434814,
      "metrics/advantage_var": 5004.10888671875,
      "rewards/chosen": 1.4610107421875,
      "rewards/margins": 1.873297355710998,
      "rewards/rejected": -0.41228661352349805,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 9.600776672363281,
      "kl": 22.71444320678711,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -41444090.347003154,
      "logits/rejected": -42096608.29721362,
      "logps/chosen": -463.39880717665613,
      "logps/rejected": -382.3298664860681,
      "loss": 0.3704,
      "loss/base_kto": 2.9630680084228516,
      "metrics/advantage_var": 5203.06201171875,
      "rewards/chosen": 1.451242729692429,
      "rewards/margins": 1.8382827409773193,
      "rewards/rejected": -0.3870400112848902,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 6.537970066070557,
      "kl": 15.437616348266602,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -39280118.930756845,
      "logits/rejected": -40898034.79210926,
      "logps/chosen": -473.702999194847,
      "logps/rejected": -360.13294290591807,
      "loss": 0.3601,
      "loss/base_kto": 2.880817174911499,
      "metrics/advantage_var": 5864.35546875,
      "rewards/chosen": 0.9577954180190721,
      "rewards/margins": 2.0404741593352234,
      "rewards/rejected": -1.0826787413161514,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 9.806045532226562,
      "kl": 19.706167221069336,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -42583149.829652995,
      "logits/rejected": -42101725.12693498,
      "logps/chosen": -473.08142744479494,
      "logps/rejected": -396.03990905572755,
      "loss": 0.3702,
      "loss/base_kto": 2.961411952972412,
      "metrics/advantage_var": 5851.69775390625,
      "rewards/chosen": 1.1792202056018335,
      "rewards/margins": 1.917102884693721,
      "rewards/rejected": -0.7378826790918875,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 10.039080619812012,
      "kl": 15.152414321899414,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -42944540.18348624,
      "logits/rejected": -43340040.99680511,
      "logps/chosen": -482.88054281345563,
      "logps/rejected": -386.05176717252397,
      "loss": 0.3623,
      "loss/base_kto": 2.8985350131988525,
      "metrics/advantage_var": 5968.19091796875,
      "rewards/chosen": 1.2328592002938648,
      "rewards/margins": 2.250269527364302,
      "rewards/rejected": -1.0174103270704373,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 9.22908878326416,
      "kl": 34.14554977416992,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -41032895.6928,
      "logits/rejected": -41480729.79541985,
      "logps/chosen": -479.8308,
      "logps/rejected": -400.47323473282444,
      "loss": 0.3558,
      "loss/base_kto": 2.846527576446533,
      "metrics/advantage_var": 5473.09716796875,
      "rewards/chosen": 1.7591283203125,
      "rewards/margins": 2.1426396728515624,
      "rewards/rejected": -0.3835113525390625,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 9.44801139831543,
      "kl": 5.6502299308776855,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -41668222.79623824,
      "logits/rejected": -41938136.92211838,
      "logps/chosen": -464.014302507837,
      "logps/rejected": -380.6324230919003,
      "loss": 0.3456,
      "loss/base_kto": 2.7647082805633545,
      "metrics/advantage_var": 6785.89453125,
      "rewards/chosen": 0.8642528378477664,
      "rewards/margins": 2.4993839132225717,
      "rewards/rejected": -1.6351310753748054,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 8.25853443145752,
      "kl": 27.141498565673828,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -42697218.29596413,
      "logits/rejected": -42946503.01800327,
      "logps/chosen": -440.3116591928251,
      "logps/rejected": -367.64192921440264,
      "loss": 0.3266,
      "loss/base_kto": 2.6130597591400146,
      "metrics/advantage_var": 7005.6376953125,
      "rewards/chosen": 1.804233157581278,
      "rewards/margins": 2.6989783802567078,
      "rewards/rejected": -0.8947452226754297,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 10.800997734069824,
      "kl": 18.460763931274414,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -42541564.71794872,
      "logits/rejected": -42184426.146341465,
      "logps/chosen": -474.4635416666667,
      "logps/rejected": -387.2720322027439,
      "loss": 0.3465,
      "loss/base_kto": 2.771848440170288,
      "metrics/advantage_var": 7674.00732421875,
      "rewards/chosen": 1.1859318659855769,
      "rewards/margins": 2.5386900409748585,
      "rewards/rejected": -1.3527581749892816,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 7.289452075958252,
      "kl": 19.30964469909668,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -44670307.96937213,
      "logits/rejected": -43826612.05741627,
      "logps/chosen": -469.2593797856049,
      "logps/rejected": -383.1033692185008,
      "loss": 0.3388,
      "loss/base_kto": 2.7102856636047363,
      "metrics/advantage_var": 7789.09375,
      "rewards/chosen": 1.5137171139213246,
      "rewards/margins": 2.627849546387433,
      "rewards/rejected": -1.1141324324661084,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 11.180095672607422,
      "kl": 8.698614120483398,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -43742993.234811164,
      "logits/rejected": -44725041.97913562,
      "logps/chosen": -488.38161945812806,
      "logps/rejected": -398.0107116244411,
      "loss": 0.3474,
      "loss/base_kto": 2.779360294342041,
      "metrics/advantage_var": 9631.0576171875,
      "rewards/chosen": 0.4435019908084462,
      "rewards/margins": 2.8141602172195865,
      "rewards/rejected": -2.37065822641114,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 8.808432579040527,
      "kl": 19.872203826904297,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -43714582.02150538,
      "logits/rejected": -43255111.22416534,
      "logps/chosen": -474.78638632872503,
      "logps/rejected": -376.85358704292526,
      "loss": 0.3379,
      "loss/base_kto": 2.7029435634613037,
      "metrics/advantage_var": 8687.8916015625,
      "rewards/chosen": 1.690362558203725,
      "rewards/margins": 2.747389965608832,
      "rewards/rejected": -1.0570274074051074,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 8.22370433807373,
      "kl": 12.967694282531738,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -44093455.375375375,
      "logits/rejected": -45716296.54723127,
      "logps/chosen": -494.614771021021,
      "logps/rejected": -378.99407064332246,
      "loss": 0.3262,
      "loss/base_kto": 2.609593629837036,
      "metrics/advantage_var": 9493.0263671875,
      "rewards/chosen": 1.4646418574336055,
      "rewards/margins": 3.2769947368615373,
      "rewards/rejected": -1.8123528794279316,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 9.493188858032227,
      "kl": 28.234481811523438,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -43742895.61352657,
      "logits/rejected": -44186697.03186646,
      "logps/chosen": -454.02123590982285,
      "logps/rejected": -362.98392450682854,
      "loss": 0.3324,
      "loss/base_kto": 2.659339189529419,
      "metrics/advantage_var": 10535.546875,
      "rewards/chosen": 2.081637126044183,
      "rewards/margins": 3.0719833833943344,
      "rewards/rejected": -0.9903462573501517,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 8.444293022155762,
      "kl": 12.679133415222168,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -45683208.23151126,
      "logits/rejected": -45501421.32522796,
      "logps/chosen": -472.53878617363347,
      "logps/rejected": -407.61084726443767,
      "loss": 0.3209,
      "loss/base_kto": 2.5670223236083984,
      "metrics/advantage_var": 11255.380859375,
      "rewards/chosen": 1.2575028103647508,
      "rewards/margins": 3.4119206425740973,
      "rewards/rejected": -2.1544178322093464,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 6.553300380706787,
      "kl": 21.521692276000977,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -44957182.41486068,
      "logits/rejected": -45334599.066246055,
      "logps/chosen": -486.2410023219814,
      "logps/rejected": -369.3547417192429,
      "loss": 0.3386,
      "loss/base_kto": 2.7085347175598145,
      "metrics/advantage_var": 9516.822265625,
      "rewards/chosen": 1.316057801615712,
      "rewards/margins": 2.804156138686887,
      "rewards/rejected": -1.4880983370711751,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 8.086736679077148,
      "kl": 25.90850257873535,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -44885738.30508474,
      "logits/rejected": -44230659.245641835,
      "logps/chosen": -468.65167565485365,
      "logps/rejected": -380.25049524564184,
      "loss": 0.3197,
      "loss/base_kto": 2.5576512813568115,
      "metrics/advantage_var": 10727.76171875,
      "rewards/chosen": 1.882302400327427,
      "rewards/margins": 3.272062655007498,
      "rewards/rejected": -1.3897602546800714,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 7.290363311767578,
      "kl": 46.841773986816406,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -46048638.45783132,
      "logits/rejected": -45202997.1948052,
      "logps/chosen": -459.4590549698795,
      "logps/rejected": -363.3772575081169,
      "loss": 0.3306,
      "loss/base_kto": 2.644582986831665,
      "metrics/advantage_var": 8934.9560546875,
      "rewards/chosen": 2.396558830536992,
      "rewards/margins": 2.956978681072199,
      "rewards/rejected": -0.5604198505352069,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 9.326342582702637,
      "kl": 34.701419830322266,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -46096527.6097561,
      "logits/rejected": -46722238.35897436,
      "logps/chosen": -448.9080125762195,
      "logps/rejected": -356.9432592147436,
      "loss": 0.3373,
      "loss/base_kto": 2.698012351989746,
      "metrics/advantage_var": 8884.0751953125,
      "rewards/chosen": 2.0205548914467415,
      "rewards/margins": 2.9187025722673043,
      "rewards/rejected": -0.898147680820563,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 7.9763569831848145,
      "kl": 30.290571212768555,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -47435029.08235294,
      "logits/rejected": -48232424.10666667,
      "logps/chosen": -453.71875,
      "logps/rejected": -392.1332552083333,
      "loss": 0.3347,
      "loss/base_kto": 2.6772916316986084,
      "metrics/advantage_var": 9017.8701171875,
      "rewards/chosen": 2.336901496438419,
      "rewards/margins": 2.92593978582644,
      "rewards/rejected": -0.5890382893880208,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 8.198352813720703,
      "kl": 42.193519592285156,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -45948783.064615384,
      "logits/rejected": -45057124.77460317,
      "logps/chosen": -428.4231730769231,
      "logps/rejected": -398.4294642857143,
      "loss": 0.3403,
      "loss/base_kto": 2.72217059135437,
      "metrics/advantage_var": 9118.69140625,
      "rewards/chosen": 2.434313401442308,
      "rewards/margins": 2.8220942600422583,
      "rewards/rejected": -0.3877808585999504,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 7.566533088684082,
      "kl": 24.0041561126709,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -44345575.8490566,
      "logits/rejected": -45049067.32919255,
      "logps/chosen": -472.1415585691824,
      "logps/rejected": -404.17330648291926,
      "loss": 0.3262,
      "loss/base_kto": 2.609588384628296,
      "metrics/advantage_var": 10468.0185546875,
      "rewards/chosen": 1.6340403046997838,
      "rewards/margins": 3.2025497086630503,
      "rewards/rejected": -1.5685094039632668,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 6.906414031982422,
      "kl": 29.112890243530273,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -44022749.26009693,
      "logits/rejected": -44602871.4795764,
      "logps/chosen": -474.4382067851373,
      "logps/rejected": -381.9731467473525,
      "loss": 0.3129,
      "loss/base_kto": 2.5033178329467773,
      "metrics/advantage_var": 11300.2333984375,
      "rewards/chosen": 2.0221373229881867,
      "rewards/margins": 3.574880931748588,
      "rewards/rejected": -1.552743608760401,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 7.620474338531494,
      "kl": 24.406091690063477,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -45466683.49847095,
      "logits/rejected": -46020585.09904154,
      "logps/chosen": -477.24870986238534,
      "logps/rejected": -381.3620207667732,
      "loss": 0.3229,
      "loss/base_kto": 2.583230495452881,
      "metrics/advantage_var": 9361.6982421875,
      "rewards/chosen": 2.0213107887758026,
      "rewards/margins": 3.1422668684233663,
      "rewards/rejected": -1.120956079647564,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 11.329891204833984,
      "kl": 26.5103816986084,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -43904365.3632,
      "logits/rejected": -45610836.03053435,
      "logps/chosen": -458.59245,
      "logps/rejected": -390.3072996183206,
      "loss": 0.3241,
      "loss/base_kto": 2.5929224491119385,
      "metrics/advantage_var": 11055.96875,
      "rewards/chosen": 1.6681978515625,
      "rewards/margins": 3.202909989265267,
      "rewards/rejected": -1.5347121377027673,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 8.696304321289062,
      "kl": 15.144403457641602,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -44265897.30351438,
      "logits/rejected": -46273955.62079511,
      "logps/chosen": -468.4798821884984,
      "logps/rejected": -381.43926796636083,
      "loss": 0.3069,
      "loss/base_kto": 2.4550883769989014,
      "metrics/advantage_var": 12411.6318359375,
      "rewards/chosen": 1.7593678239816295,
      "rewards/margins": 3.7511917261700853,
      "rewards/rejected": -1.9918239021884556,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 9.53442096710205,
      "kl": 13.843756675720215,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -46577823.03645008,
      "logits/rejected": -47045128.6779661,
      "logps/chosen": -467.5065372424723,
      "logps/rejected": -386.2795165639445,
      "loss": 0.3309,
      "loss/base_kto": 2.6470870971679688,
      "metrics/advantage_var": 12080.7734375,
      "rewards/chosen": 1.70285745128021,
      "rewards/margins": 3.3992557939829258,
      "rewards/rejected": -1.6963983427027156,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 6.700616359710693,
      "kl": 23.6582088470459,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -48391386.96855346,
      "logits/rejected": -48613763.97515528,
      "logps/chosen": -496.71255896226415,
      "logps/rejected": -369.32375776397515,
      "loss": 0.3146,
      "loss/base_kto": 2.5171942710876465,
      "metrics/advantage_var": 13365.2724609375,
      "rewards/chosen": 2.063871755539996,
      "rewards/margins": 3.7268028305472747,
      "rewards/rejected": -1.6629310750072788,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 6.985821723937988,
      "kl": 23.956912994384766,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -48765427.751196176,
      "logits/rejected": -47536871.301684536,
      "logps/chosen": -473.1124401913876,
      "logps/rejected": -416.80647013782544,
      "loss": 0.3062,
      "loss/base_kto": 2.449873685836792,
      "metrics/advantage_var": 13375.8642578125,
      "rewards/chosen": 1.7407265500398723,
      "rewards/margins": 3.816490153121802,
      "rewards/rejected": -2.0757636030819295,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 8.316628456115723,
      "kl": 27.635818481445312,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -47696105.15692308,
      "logits/rejected": -47566792.560509555,
      "logps/chosen": -459.5833653846154,
      "logps/rejected": -418.2669685509554,
      "loss": 0.2826,
      "loss/base_kto": 2.2607383728027344,
      "metrics/advantage_var": 14893.017578125,
      "rewards/chosen": 2.2947506009615384,
      "rewards/margins": 4.421246934187056,
      "rewards/rejected": -2.1264963332255173,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 8.086918830871582,
      "kl": 18.65052604675293,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -47266483.120743036,
      "logits/rejected": -46471193.84227129,
      "logps/chosen": -447.69045085139317,
      "logps/rejected": -406.4086652208202,
      "loss": 0.2638,
      "loss/base_kto": 2.110551118850708,
      "metrics/advantage_var": 14927.890625,
      "rewards/chosen": 2.344045727614648,
      "rewards/margins": 4.815705776042289,
      "rewards/rejected": -2.471660048427642,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 9.235642433166504,
      "kl": 14.318849563598633,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -47330193.20655738,
      "logits/rejected": -48266255.28358209,
      "logps/chosen": -478.18575819672134,
      "logps/rejected": -398.50046641791045,
      "loss": 0.2707,
      "loss/base_kto": 2.165867805480957,
      "metrics/advantage_var": 16995.412109375,
      "rewards/chosen": 1.6457587570440573,
      "rewards/margins": 4.922583252729692,
      "rewards/rejected": -3.2768244956856343,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 6.9475836753845215,
      "kl": 18.910648345947266,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -48754031.46411483,
      "logits/rejected": -48983170.156202145,
      "logps/chosen": -449.504485645933,
      "logps/rejected": -377.55764261102604,
      "loss": 0.2572,
      "loss/base_kto": 2.057651996612549,
      "metrics/advantage_var": 16152.619140625,
      "rewards/chosen": 2.3972954514304226,
      "rewards/margins": 5.001059703560017,
      "rewards/rejected": -2.603764252129594,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 7.026848793029785,
      "kl": 18.607139587402344,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -46669212.90322581,
      "logits/rejected": -47008243.587878786,
      "logps/chosen": -477.9242943548387,
      "logps/rejected": -392.5854640151515,
      "loss": 0.2558,
      "loss/base_kto": 2.0461461544036865,
      "metrics/advantage_var": 13921.1953125,
      "rewards/chosen": 2.056893034904234,
      "rewards/margins": 4.811471226488041,
      "rewards/rejected": -2.754578191583807,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 7.500204563140869,
      "kl": 20.632123947143555,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -48184230.28398792,
      "logits/rejected": -48479195.54692557,
      "logps/chosen": -472.469458081571,
      "logps/rejected": -404.5207827669903,
      "loss": 0.2658,
      "loss/base_kto": 2.126520872116089,
      "metrics/advantage_var": 14187.9189453125,
      "rewards/chosen": 2.431369009334875,
      "rewards/margins": 4.628053911489001,
      "rewards/rejected": -2.1966849021541264,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 11.07607650756836,
      "kl": 19.66994285583496,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -49061644.641975306,
      "logits/rejected": -49284076.55696203,
      "logps/chosen": -453.7958140432099,
      "logps/rejected": -417.20149327531647,
      "loss": 0.2483,
      "loss/base_kto": 1.9867771863937378,
      "metrics/advantage_var": 14991.2001953125,
      "rewards/chosen": 2.400873631606867,
      "rewards/margins": 4.973577705355878,
      "rewards/rejected": -2.5727040737490112,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 7.448911666870117,
      "kl": 15.462172508239746,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -48214747.88714734,
      "logits/rejected": -49171829.23364486,
      "logps/chosen": -455.8012343260188,
      "logps/rejected": -429.9959598909657,
      "loss": 0.2591,
      "loss/base_kto": 2.072408676147461,
      "metrics/advantage_var": 14586.7568359375,
      "rewards/chosen": 2.2276615154780566,
      "rewards/margins": 4.818913447307301,
      "rewards/rejected": -2.5912519318292446,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 6.72298002243042,
      "kl": 23.561777114868164,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -47956512.0,
      "logits/rejected": -49148969.6,
      "logps/chosen": -450.888916015625,
      "logps/rejected": -404.3291015625,
      "loss": 0.2624,
      "loss/base_kto": 2.099287509918213,
      "metrics/advantage_var": 13910.462890625,
      "rewards/chosen": 2.486882781982422,
      "rewards/margins": 4.68602180480957,
      "rewards/rejected": -2.1991390228271483,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 6.384796619415283,
      "kl": 18.171024322509766,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -49109873.3374613,
      "logits/rejected": -48812248.42902208,
      "logps/chosen": -462.2097523219814,
      "logps/rejected": -390.073738170347,
      "loss": 0.2643,
      "loss/base_kto": 2.114152431488037,
      "metrics/advantage_var": 14593.4248046875,
      "rewards/chosen": 2.215952208905766,
      "rewards/margins": 4.692166981801074,
      "rewards/rejected": -2.4762147728953074,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 6.847940921783447,
      "kl": 17.695087432861328,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -50544042.93416928,
      "logits/rejected": -48933128.77258567,
      "logps/chosen": -456.51489028213166,
      "logps/rejected": -395.2147098909657,
      "loss": 0.2613,
      "loss/base_kto": 2.0907790660858154,
      "metrics/advantage_var": 12825.6767578125,
      "rewards/chosen": 2.1193005792025863,
      "rewards/margins": 4.628709690439443,
      "rewards/rejected": -2.5094091112368573,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 6.785857677459717,
      "kl": 23.224470138549805,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -48401680.440366976,
      "logits/rejected": -47403619.782747604,
      "logps/chosen": -463.9189124617737,
      "logps/rejected": -383.25039936102235,
      "loss": 0.2638,
      "loss/base_kto": 2.110729694366455,
      "metrics/advantage_var": 13884.853515625,
      "rewards/chosen": 2.27902361668578,
      "rewards/margins": 4.50523706579221,
      "rewards/rejected": -2.22621344910643,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 8.263350486755371,
      "kl": 29.09259033203125,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -50162247.24919094,
      "logits/rejected": -51190753.06344411,
      "logps/chosen": -466.3296925566343,
      "logps/rejected": -381.7652473564955,
      "loss": 0.2527,
      "loss/base_kto": 2.0212230682373047,
      "metrics/advantage_var": 13454.7890625,
      "rewards/chosen": 2.5315537931078076,
      "rewards/margins": 4.647886247731581,
      "rewards/rejected": -2.1163324546237727,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 7.771735191345215,
      "kl": 27.689910888671875,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -49508589.54231975,
      "logits/rejected": -50506238.40498442,
      "logps/chosen": -458.10452586206895,
      "logps/rejected": -415.4616433021807,
      "loss": 0.2494,
      "loss/base_kto": 1.9951435327529907,
      "metrics/advantage_var": 15280.3505859375,
      "rewards/chosen": 2.685712951851489,
      "rewards/margins": 4.9627353641129766,
      "rewards/rejected": -2.2770224122614877,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 6.784241676330566,
      "kl": 18.85918426513672,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -50499053.278219394,
      "logits/rejected": -51514501.70199693,
      "logps/chosen": -453.55176868044515,
      "logps/rejected": -394.48804723502303,
      "loss": 0.2559,
      "loss/base_kto": 2.047353744506836,
      "metrics/advantage_var": 14907.603515625,
      "rewards/chosen": 2.608219116405008,
      "rewards/margins": 4.9608688109854615,
      "rewards/rejected": -2.352649694580453,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 9.232665061950684,
      "kl": 20.594545364379883,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -49949756.04938272,
      "logits/rejected": -50106912.405063294,
      "logps/chosen": -474.619212962963,
      "logps/rejected": -386.55075652689874,
      "loss": 0.2712,
      "loss/base_kto": 2.1697463989257812,
      "metrics/advantage_var": 13390.572265625,
      "rewards/chosen": 2.2712521023220487,
      "rewards/margins": 4.5504941229411155,
      "rewards/rejected": -2.2792420206190664,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 7.8734869956970215,
      "kl": 20.734603881835938,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -49311651.469879515,
      "logits/rejected": -49996404.36363637,
      "logps/chosen": -454.7604951054217,
      "logps/rejected": -383.43514103084414,
      "loss": 0.241,
      "loss/base_kto": 1.9277294874191284,
      "metrics/advantage_var": 13632.861328125,
      "rewards/chosen": 2.6894562502941453,
      "rewards/margins": 5.044658665066873,
      "rewards/rejected": -2.355202414772727,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 8.654077529907227,
      "kl": 20.20318603515625,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -48451214.73939394,
      "logits/rejected": -49571747.50967742,
      "logps/chosen": -453.94166666666666,
      "logps/rejected": -407.38770161290324,
      "loss": 0.2628,
      "loss/base_kto": 2.102159023284912,
      "metrics/advantage_var": 15591.119140625,
      "rewards/chosen": 2.485277210582386,
      "rewards/margins": 4.870609478097507,
      "rewards/rejected": -2.385332267515121,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 8.510467529296875,
      "kl": 17.95111656188965,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -48232398.61093248,
      "logits/rejected": -48532613.835866265,
      "logps/chosen": -474.1833802250804,
      "logps/rejected": -380.45203267477206,
      "loss": 0.2508,
      "loss/base_kto": 2.0065157413482666,
      "metrics/advantage_var": 14855.84375,
      "rewards/chosen": 2.3578408784038385,
      "rewards/margins": 4.872037767057904,
      "rewards/rejected": -2.5141968886540655,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 8.991572380065918,
      "kl": 15.149621963500977,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -49015907.62360447,
      "logits/rejected": -50663179.36906585,
      "logps/chosen": -475.938995215311,
      "logps/rejected": -412.8200612557427,
      "loss": 0.2499,
      "loss/base_kto": 1.9995607137680054,
      "metrics/advantage_var": 15994.5146484375,
      "rewards/chosen": 2.0744765188895533,
      "rewards/margins": 5.248389139726077,
      "rewards/rejected": -3.1739126208365236,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 7.84478235244751,
      "kl": 21.839262008666992,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -48684938.084848486,
      "logits/rejected": -50561935.69032258,
      "logps/chosen": -474.6498106060606,
      "logps/rejected": -399.6490927419355,
      "loss": 0.2454,
      "loss/base_kto": 1.9630470275878906,
      "metrics/advantage_var": 14052.4287109375,
      "rewards/chosen": 2.6722438003077653,
      "rewards/margins": 5.023397955423693,
      "rewards/rejected": -2.3511541551159274,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 9.335622787475586,
      "kl": 15.088180541992188,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -47908844.24437299,
      "logits/rejected": -50457746.28571428,
      "logps/chosen": -465.9122286977492,
      "logps/rejected": -383.3266052431611,
      "loss": 0.2499,
      "loss/base_kto": 1.9995310306549072,
      "metrics/advantage_var": 13934.4267578125,
      "rewards/chosen": 2.2671612803959005,
      "rewards/margins": 4.926837782936744,
      "rewards/rejected": -2.6596765025408433,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 6.023534297943115,
      "kl": 22.312055587768555,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -48825570.66248038,
      "logits/rejected": -49317209.58009331,
      "logps/chosen": -439.3760302197802,
      "logps/rejected": -393.66628596423016,
      "loss": 0.2703,
      "loss/base_kto": 2.162353277206421,
      "metrics/advantage_var": 14146.5986328125,
      "rewards/chosen": 2.341197446526688,
      "rewards/margins": 4.739072625740334,
      "rewards/rejected": -2.397875179213647,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 8.418098449707031,
      "kl": 28.46211814880371,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -49135524.76804916,
      "logits/rejected": -48735933.659777425,
      "logps/chosen": -460.48915130568355,
      "logps/rejected": -390.8551768680445,
      "loss": 0.2433,
      "loss/base_kto": 1.9464820623397827,
      "metrics/advantage_var": 12723.3603515625,
      "rewards/chosen": 2.8656859038978495,
      "rewards/margins": 4.710928111581176,
      "rewards/rejected": -1.8452422076833268,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 7.899602890014648,
      "kl": 18.49418067932129,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -51717907.0681458,
      "logits/rejected": -51133725.583975345,
      "logps/chosen": -465.9952456418383,
      "logps/rejected": -404.7318952234206,
      "loss": 0.2494,
      "loss/base_kto": 1.9950428009033203,
      "metrics/advantage_var": 14188.0537109375,
      "rewards/chosen": 2.3086831263619256,
      "rewards/margins": 5.01768047956156,
      "rewards/rejected": -2.708997353199634,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 10.677668571472168,
      "kl": 34.446258544921875,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -50640544.0,
      "logits/rejected": -50915161.6,
      "logps/chosen": -467.531201171875,
      "logps/rejected": -394.303173828125,
      "loss": 0.2485,
      "loss/base_kto": 1.9881908893585205,
      "metrics/advantage_var": 11588.8076171875,
      "rewards/chosen": 2.739125442504883,
      "rewards/margins": 4.502296447753906,
      "rewards/rejected": -1.7631710052490235,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 11.427660942077637,
      "kl": 19.136369705200195,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -51027703.54354354,
      "logits/rejected": -50779572.95114007,
      "logps/chosen": -447.70481418918916,
      "logps/rejected": -390.48531657166126,
      "loss": 0.262,
      "loss/base_kto": 2.096342086791992,
      "metrics/advantage_var": 12463.3466796875,
      "rewards/chosen": 2.589214336406719,
      "rewards/margins": 4.624389672318771,
      "rewards/rejected": -2.035175335912052,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 7.118358612060547,
      "kl": 29.556894302368164,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -50797718.87147336,
      "logits/rejected": -52695827.937694706,
      "logps/chosen": -459.33473746081506,
      "logps/rejected": -397.45083722741435,
      "loss": 0.2511,
      "loss/base_kto": 2.0086159706115723,
      "metrics/advantage_var": 13190.4873046875,
      "rewards/chosen": 2.7182460294621866,
      "rewards/margins": 4.6446517014345385,
      "rewards/rejected": -1.9264056719723521,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 7.0251383781433105,
      "kl": 21.34518814086914,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -49025282.761171035,
      "logits/rejected": -50635216.12678289,
      "logps/chosen": -459.6283224191063,
      "logps/rejected": -376.3868116085578,
      "loss": 0.2538,
      "loss/base_kto": 2.0301125049591064,
      "metrics/advantage_var": 12765.095703125,
      "rewards/chosen": 2.7072889330941834,
      "rewards/margins": 4.747839298028316,
      "rewards/rejected": -2.0405503649341323,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 7.253071308135986,
      "kl": 15.496813774108887,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -49935774.09404389,
      "logits/rejected": -49520655.950155765,
      "logps/chosen": -451.2373628526646,
      "logps/rejected": -389.37940517912773,
      "loss": 0.2642,
      "loss/base_kto": 2.1133599281311035,
      "metrics/advantage_var": 14677.015625,
      "rewards/chosen": 2.3958952613758817,
      "rewards/margins": 4.980968381873351,
      "rewards/rejected": -2.585073120497469,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 10.220635414123535,
      "kl": 18.263525009155273,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -50836060.110210694,
      "logits/rejected": -50945405.49019608,
      "logps/chosen": -478.1290012155592,
      "logps/rejected": -402.84605957767724,
      "loss": 0.2563,
      "loss/base_kto": 2.0507876873016357,
      "metrics/advantage_var": 13821.056640625,
      "rewards/chosen": 2.3436550344408427,
      "rewards/margins": 4.7163410379287765,
      "rewards/rejected": -2.372686003487934,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 10.060528755187988,
      "kl": 17.743131637573242,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -52369721.56937799,
      "logits/rejected": -50915011.23430321,
      "logps/chosen": -481.9745813397129,
      "logps/rejected": -410.11581163859114,
      "loss": 0.2446,
      "loss/base_kto": 1.9567173719406128,
      "metrics/advantage_var": 14335.0966796875,
      "rewards/chosen": 2.371380332934609,
      "rewards/margins": 5.096604732939395,
      "rewards/rejected": -2.7252244000047856,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 8.237143516540527,
      "kl": 16.771970748901367,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -49415204.51472868,
      "logits/rejected": -49890866.79685039,
      "logps/chosen": -462.5661821705426,
      "logps/rejected": -418.36003937007877,
      "loss": 0.2713,
      "loss/base_kto": 2.170379400253296,
      "metrics/advantage_var": 13692.6533203125,
      "rewards/chosen": 1.8580134901889536,
      "rewards/margins": 4.5056849268892485,
      "rewards/rejected": -2.6476714367002954,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 7.028975963592529,
      "kl": 17.3207950592041,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -50510410.51643193,
      "logits/rejected": -49549935.026521064,
      "logps/chosen": -456.42703442879497,
      "logps/rejected": -386.7444910296412,
      "loss": 0.2477,
      "loss/base_kto": 1.9812153577804565,
      "metrics/advantage_var": 14027.7607421875,
      "rewards/chosen": 2.5905593609399453,
      "rewards/margins": 4.8752099998951905,
      "rewards/rejected": -2.2846506389552457,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 11.530803680419922,
      "kl": 31.405414581298828,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -51228454.4,
      "logits/rejected": -50197628.8,
      "logps/chosen": -466.95244140625,
      "logps/rejected": -384.535595703125,
      "loss": 0.2648,
      "loss/base_kto": 2.11802339553833,
      "metrics/advantage_var": 13980.767578125,
      "rewards/chosen": 2.5955698013305666,
      "rewards/margins": 4.664482307434082,
      "rewards/rejected": -2.0689125061035156,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 7.988217353820801,
      "kl": 19.561134338378906,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -51357387.52795031,
      "logits/rejected": -50171961.96226415,
      "logps/chosen": -456.13480201863354,
      "logps/rejected": -402.56245086477986,
      "loss": 0.2583,
      "loss/base_kto": 2.0667009353637695,
      "metrics/advantage_var": 12997.6875,
      "rewards/chosen": 2.3391037461180124,
      "rewards/margins": 4.620794811492423,
      "rewards/rejected": -2.2816910653744102,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 9.072982788085938,
      "kl": 10.457106590270996,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -49079869.44,
      "logits/rejected": -49365005.0031746,
      "logps/chosen": -479.4330769230769,
      "logps/rejected": -389.29305555555555,
      "loss": 0.2524,
      "loss/base_kto": 2.019576072692871,
      "metrics/advantage_var": 13046.0380859375,
      "rewards/chosen": 2.1333349609375,
      "rewards/margins": 4.809057500930059,
      "rewards/rejected": -2.6757225399925595,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 6.771179676055908,
      "kl": 18.836105346679688,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -51580368.56230032,
      "logits/rejected": -51782458.71559633,
      "logps/chosen": -458.86077276357827,
      "logps/rejected": -387.5057339449541,
      "loss": 0.2481,
      "loss/base_kto": 1.9848469495773315,
      "metrics/advantage_var": 13515.59375,
      "rewards/chosen": 2.6073875549121404,
      "rewards/margins": 4.905324230657554,
      "rewards/rejected": -2.297936675745413,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 9.2738037109375,
      "kl": 20.875213623046875,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -49941718.99222395,
      "logits/rejected": -49673548.75981162,
      "logps/chosen": -477.6462869362364,
      "logps/rejected": -393.92337127158555,
      "loss": 0.2624,
      "loss/base_kto": 2.0992538928985596,
      "metrics/advantage_var": 13856.025390625,
      "rewards/chosen": 2.3467542965712482,
      "rewards/margins": 4.769467883207826,
      "rewards/rejected": -2.422713586636578,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 9.066880226135254,
      "kl": 15.743685722351074,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -49764477.83717358,
      "logits/rejected": -50455547.10047847,
      "logps/chosen": -475.1092069892473,
      "logps/rejected": -391.1201405502392,
      "loss": 0.2159,
      "loss/base_kto": 1.7273807525634766,
      "metrics/advantage_var": 14440.111328125,
      "rewards/chosen": 2.4454221945204493,
      "rewards/margins": 5.453206464543874,
      "rewards/rejected": -3.007784270023425,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 9.389199256896973,
      "kl": 14.396817207336426,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -49846694.853620954,
      "logits/rejected": -49367530.09191759,
      "logps/chosen": -454.39897919876734,
      "logps/rejected": -431.447999207607,
      "loss": 0.1951,
      "loss/base_kto": 1.5605216026306152,
      "metrics/advantage_var": 14475.1787109375,
      "rewards/chosen": 2.7826467552243837,
      "rewards/margins": 5.7794783438485915,
      "rewards/rejected": -2.9968315886242074,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 9.054242134094238,
      "kl": 17.919559478759766,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -48817131.61408882,
      "logits/rejected": -49831112.880382776,
      "logps/chosen": -472.0823602603369,
      "logps/rejected": -417.2660735645933,
      "loss": 0.223,
      "loss/base_kto": 1.7838348150253296,
      "metrics/advantage_var": 16574.927734375,
      "rewards/chosen": 2.5876096576497893,
      "rewards/margins": 5.747501519317453,
      "rewards/rejected": -3.1598918616676634,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 6.2732720375061035,
      "kl": 25.5059814453125,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -51527312.93802345,
      "logits/rejected": -51728000.18740849,
      "logps/chosen": -459.5612960636516,
      "logps/rejected": -430.1048682284041,
      "loss": 0.207,
      "loss/base_kto": 1.6557525396347046,
      "metrics/advantage_var": 15799.3505859375,
      "rewards/chosen": 2.7100296403894473,
      "rewards/margins": 5.6499131391998425,
      "rewards/rejected": -2.9398834988103952,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 4.447214126586914,
      "kl": 14.574604988098145,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -49083068.97160883,
      "logits/rejected": -50236628.408668734,
      "logps/chosen": -458.88761829653,
      "logps/rejected": -392.84940982972137,
      "loss": 0.2058,
      "loss/base_kto": 1.6461609601974487,
      "metrics/advantage_var": 15221.9443359375,
      "rewards/chosen": 2.56582015846806,
      "rewards/margins": 5.746134260877115,
      "rewards/rejected": -3.1803141024090555,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 7.516998291015625,
      "kl": 12.84099292755127,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -51319764.45984252,
      "logits/rejected": -49929865.327131785,
      "logps/chosen": -460.7167322834646,
      "logps/rejected": -405.8479893410853,
      "loss": 0.2265,
      "loss/base_kto": 1.811827540397644,
      "metrics/advantage_var": 15491.5888671875,
      "rewards/chosen": 2.40770523191437,
      "rewards/margins": 5.415428781454098,
      "rewards/rejected": -3.0077235495397288,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 7.593645095825195,
      "kl": 11.170424461364746,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -51468312.56371814,
      "logits/rejected": -52534046.48613377,
      "logps/chosen": -462.5133526986507,
      "logps/rejected": -403.46454424959217,
      "loss": 0.2075,
      "loss/base_kto": 1.6599674224853516,
      "metrics/advantage_var": 16793.154296875,
      "rewards/chosen": 2.6445632774433094,
      "rewards/margins": 6.016063264698611,
      "rewards/rejected": -3.371499987255302,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 8.261808395385742,
      "kl": 21.838741302490234,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -51434695.60697306,
      "logits/rejected": -50746618.87211094,
      "logps/chosen": -457.355883518225,
      "logps/rejected": -381.01047284283516,
      "loss": 0.2137,
      "loss/base_kto": 1.709884524345398,
      "metrics/advantage_var": 14293.1083984375,
      "rewards/chosen": 2.6543518163133912,
      "rewards/margins": 5.4538469079206715,
      "rewards/rejected": -2.7994950916072803,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 6.802206516265869,
      "kl": 22.52808380126953,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -49329979.88069074,
      "logits/rejected": -49380143.377916016,
      "logps/chosen": -430.70692700156985,
      "logps/rejected": -378.99448386469675,
      "loss": 0.2146,
      "loss/base_kto": 1.7167484760284424,
      "metrics/advantage_var": 14634.306640625,
      "rewards/chosen": 2.905923457123234,
      "rewards/margins": 5.455109060192344,
      "rewards/rejected": -2.54918560306911,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 8.124829292297363,
      "kl": 18.092741012573242,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -49389036.679245286,
      "logits/rejected": -52162470.95652174,
      "logps/chosen": -451.3704304245283,
      "logps/rejected": -398.70661878881987,
      "loss": 0.2241,
      "loss/base_kto": 1.7931416034698486,
      "metrics/advantage_var": 14365.892578125,
      "rewards/chosen": 2.5793134581367925,
      "rewards/margins": 5.32185048865892,
      "rewards/rejected": -2.7425370305221275,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 6.912888526916504,
      "kl": 18.40091896057129,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -50879499.39586645,
      "logits/rejected": -51614092.38709678,
      "logps/chosen": -469.5434220985692,
      "logps/rejected": -401.07716493855605,
      "loss": 0.1912,
      "loss/base_kto": 1.5298614501953125,
      "metrics/advantage_var": 15943.1455078125,
      "rewards/chosen": 2.826442409206081,
      "rewards/margins": 5.9957837470517035,
      "rewards/rejected": -3.1693413378456223,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.5973076820373535,
      "kl": 16.810462951660156,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -52416382.018575855,
      "logits/rejected": -50373166.83911672,
      "logps/chosen": -448.53125,
      "logps/rejected": -416.87638012618294,
      "loss": 0.2206,
      "loss/base_kto": 1.7645885944366455,
      "metrics/advantage_var": 15707.8125,
      "rewards/chosen": 2.6600234087775734,
      "rewards/margins": 5.533431488677021,
      "rewards/rejected": -2.873408079899448,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 8.159750938415527,
      "kl": 15.953948974609375,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -50439779.90243903,
      "logits/rejected": -52401309.538461536,
      "logps/chosen": -433.3176448170732,
      "logps/rejected": -398.44851762820514,
      "loss": 0.2066,
      "loss/base_kto": 1.653062105178833,
      "metrics/advantage_var": 14817.1083984375,
      "rewards/chosen": 2.8851549102038874,
      "rewards/margins": 5.672613281097316,
      "rewards/rejected": -2.7874583708934293,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 8.426179885864258,
      "kl": 21.919878005981445,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -53184847.127272725,
      "logits/rejected": -53741422.658064514,
      "logps/chosen": -479.77836174242424,
      "logps/rejected": -384.43377016129034,
      "loss": 0.2202,
      "loss/base_kto": 1.7617501020431519,
      "metrics/advantage_var": 13528.0146484375,
      "rewards/chosen": 2.8051628343986743,
      "rewards/margins": 5.240481516669339,
      "rewards/rejected": -2.4353186822706654,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 8.882457733154297,
      "kl": 14.964949607849121,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -51817944.85626911,
      "logits/rejected": -53017380.80511182,
      "logps/chosen": -478.57282110091745,
      "logps/rejected": -419.58541333865816,
      "loss": 0.2156,
      "loss/base_kto": 1.7244430780410767,
      "metrics/advantage_var": 15768.9912109375,
      "rewards/chosen": 2.7158899336415327,
      "rewards/margins": 5.506087336546884,
      "rewards/rejected": -2.7901974029053513,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 9.163458824157715,
      "kl": 15.542861938476562,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -52963602.73170732,
      "logits/rejected": -52902058.666666664,
      "logps/chosen": -471.7673399390244,
      "logps/rejected": -374.76407251602564,
      "loss": 0.2272,
      "loss/base_kto": 1.8178703784942627,
      "metrics/advantage_var": 13268.478515625,
      "rewards/chosen": 2.2558530481850227,
      "rewards/margins": 5.160528262903572,
      "rewards/rejected": -2.9046752147185497,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 8.934484481811523,
      "kl": 20.741098403930664,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -51061180.37735849,
      "logits/rejected": -51750101.06832298,
      "logps/chosen": -447.9060534591195,
      "logps/rejected": -389.9256599378882,
      "loss": 0.1976,
      "loss/base_kto": 1.580649733543396,
      "metrics/advantage_var": 14590.390625,
      "rewards/chosen": 2.9136214346255898,
      "rewards/margins": 5.722851883662372,
      "rewards/rejected": -2.809230449036782,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 8.323333740234375,
      "kl": 18.002378463745117,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -51800554.764976956,
      "logits/rejected": -51589393.50079491,
      "logps/chosen": -459.7103974654378,
      "logps/rejected": -413.1946045310016,
      "loss": 0.2154,
      "loss/base_kto": 1.723293662071228,
      "metrics/advantage_var": 14929.03125,
      "rewards/chosen": 2.601546561479935,
      "rewards/margins": 5.695510427114871,
      "rewards/rejected": -3.093963865634936,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 7.7876386642456055,
      "kl": 8.304442405700684,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -49807206.24624625,
      "logits/rejected": -52539025.094462544,
      "logps/chosen": -478.9786974474475,
      "logps/rejected": -405.7814026872964,
      "loss": 0.2193,
      "loss/base_kto": 1.7541526556015015,
      "metrics/advantage_var": 15291.994140625,
      "rewards/chosen": 2.579624118747654,
      "rewards/margins": 5.542451521663981,
      "rewards/rejected": -2.9628274029163273,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 8.908504486083984,
      "kl": 14.945642471313477,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -50482227.52201258,
      "logits/rejected": -52607968.19875777,
      "logps/chosen": -443.0347877358491,
      "logps/rejected": -399.8195603649068,
      "loss": 0.2169,
      "loss/base_kto": 1.735451579093933,
      "metrics/advantage_var": 14073.5625,
      "rewards/chosen": 2.4197395372690647,
      "rewards/margins": 5.358329261223354,
      "rewards/rejected": -2.9385897239542897,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 6.341398239135742,
      "kl": 17.427776336669922,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -49973613.24840765,
      "logits/rejected": -49779624.04907975,
      "logps/chosen": -439.78841560509557,
      "logps/rejected": -381.7407496165644,
      "loss": 0.2208,
      "loss/base_kto": 1.7662585973739624,
      "metrics/advantage_var": 14243.8115234375,
      "rewards/chosen": 2.6234202779782048,
      "rewards/margins": 5.432613520525176,
      "rewards/rejected": -2.809193242546971,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 7.262148857116699,
      "kl": 14.746039390563965,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -53409282.403755866,
      "logits/rejected": -53459969.5975039,
      "logps/chosen": -476.1471048513302,
      "logps/rejected": -378.63133775351014,
      "loss": 0.2166,
      "loss/base_kto": 1.7325859069824219,
      "metrics/advantage_var": 13408.611328125,
      "rewards/chosen": 2.5891344431607,
      "rewards/margins": 5.4351666438237265,
      "rewards/rejected": -2.8460322006630263,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 7.136597633361816,
      "kl": 17.677648544311523,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -52757400.216216214,
      "logits/rejected": -53190644.093023255,
      "logps/chosen": -452.89711782094594,
      "logps/rejected": -439.1735555959302,
      "loss": 0.2207,
      "loss/base_kto": 1.765488862991333,
      "metrics/advantage_var": 15374.1005859375,
      "rewards/chosen": 2.442764694626267,
      "rewards/margins": 5.516843631535787,
      "rewards/rejected": -3.0740789369095203,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 9.145492553710938,
      "kl": 17.41663932800293,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -52410463.403726704,
      "logits/rejected": -52548765.786163524,
      "logps/chosen": -464.3349184782609,
      "logps/rejected": -403.373034591195,
      "loss": 0.2086,
      "loss/base_kto": 1.6685141324996948,
      "metrics/advantage_var": 15616.8876953125,
      "rewards/chosen": 2.643056549640916,
      "rewards/margins": 5.672259376758644,
      "rewards/rejected": -3.029202827117728,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 7.5655131340026855,
      "kl": 15.2339448928833,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -51871648.66773676,
      "logits/rejected": -52101949.175038055,
      "logps/chosen": -466.7584269662921,
      "logps/rejected": -413.64906773211567,
      "loss": 0.2043,
      "loss/base_kto": 1.634189248085022,
      "metrics/advantage_var": 14743.259765625,
      "rewards/chosen": 2.6696169931280096,
      "rewards/margins": 5.812133804272416,
      "rewards/rejected": -3.1425168111444064,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 6.279407978057861,
      "kl": 14.870948791503906,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -53818698.7820711,
      "logits/rejected": -53071774.93838862,
      "logps/chosen": -491.1116692426584,
      "logps/rejected": -396.6323805292259,
      "loss": 0.2063,
      "loss/base_kto": 1.650800108909607,
      "metrics/advantage_var": 17707.349609375,
      "rewards/chosen": 2.7372101631931027,
      "rewards/margins": 5.961800446627641,
      "rewards/rejected": -3.224590283434538,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 8.242992401123047,
      "kl": 23.719873428344727,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -54485708.8,
      "logits/rejected": -52339644.8,
      "logps/chosen": -465.237744140625,
      "logps/rejected": -403.1852294921875,
      "loss": 0.2125,
      "loss/base_kto": 1.6999969482421875,
      "metrics/advantage_var": 15859.7861328125,
      "rewards/chosen": 2.6175846099853515,
      "rewards/margins": 5.560161018371582,
      "rewards/rejected": -2.9425764083862305,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.003545761108398,
      "kl": 20.206554412841797,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -51759656.01275917,
      "logits/rejected": -52200769.47013783,
      "logps/chosen": -468.85930023923447,
      "logps/rejected": -387.87416251914243,
      "loss": 0.2183,
      "loss/base_kto": 1.7463781833648682,
      "metrics/advantage_var": 14242.609375,
      "rewards/chosen": 2.620401239471192,
      "rewards/margins": 5.27650524113419,
      "rewards/rejected": -2.6561040016629978,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 8.930207252502441,
      "kl": 14.447230339050293,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -49582002.47318612,
      "logits/rejected": -51979628.58204334,
      "logps/chosen": -437.5366226340694,
      "logps/rejected": -408.49970975232196,
      "loss": 0.2252,
      "loss/base_kto": 1.801300048828125,
      "metrics/advantage_var": 14933.4375,
      "rewards/chosen": 2.586171628548896,
      "rewards/margins": 5.44742983711604,
      "rewards/rejected": -2.861258208567144,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 8.271646499633789,
      "kl": 15.560394287109375,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -52691174.4,
      "logits/rejected": -52273539.2,
      "logps/chosen": -449.1541015625,
      "logps/rejected": -425.691455078125,
      "loss": 0.2191,
      "loss/base_kto": 1.7531509399414062,
      "metrics/advantage_var": 14178.3193359375,
      "rewards/chosen": 2.5278388977050783,
      "rewards/margins": 5.39668254852295,
      "rewards/rejected": -2.868843650817871,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 8.030889511108398,
      "kl": 16.55352210998535,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -51556539.07692308,
      "logits/rejected": -52023102.43902439,
      "logps/chosen": -468.71634615384613,
      "logps/rejected": -404.0106707317073,
      "loss": 0.2067,
      "loss/base_kto": 1.6536568403244019,
      "metrics/advantage_var": 15274.6171875,
      "rewards/chosen": 2.796620099972456,
      "rewards/margins": 5.835180734678534,
      "rewards/rejected": -3.0385606347060783,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 8.587590217590332,
      "kl": 16.787067413330078,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -50933763.22012579,
      "logits/rejected": -51906292.86956522,
      "logps/chosen": -450.91283411949684,
      "logps/rejected": -395.3308909161491,
      "loss": 0.2252,
      "loss/base_kto": 1.801880121231079,
      "metrics/advantage_var": 13920.283203125,
      "rewards/chosen": 2.509590964647209,
      "rewards/margins": 5.2165346424665024,
      "rewards/rejected": -2.7069436778192935,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 6.320584774017334,
      "kl": 20.709028244018555,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -51578837.93996841,
      "logits/rejected": -50881885.774343126,
      "logps/chosen": -459.2451125592417,
      "logps/rejected": -404.6876207496136,
      "loss": 0.2161,
      "loss/base_kto": 1.7288354635238647,
      "metrics/advantage_var": 13952.947265625,
      "rewards/chosen": 2.7227630856289493,
      "rewards/margins": 5.459844009001245,
      "rewards/rejected": -2.7370809233722952,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 8.064732551574707,
      "kl": 10.20324993133545,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -52426484.32826748,
      "logits/rejected": -50288877.06752411,
      "logps/chosen": -456.69006458966567,
      "logps/rejected": -402.67941117363347,
      "loss": 0.218,
      "loss/base_kto": 1.7438405752182007,
      "metrics/advantage_var": 14851.8125,
      "rewards/chosen": 2.6717870648508737,
      "rewards/margins": 5.595717242202159,
      "rewards/rejected": -2.923930177351286,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 7.492672920227051,
      "kl": 13.49139404296875,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -53041129.494505495,
      "logits/rejected": -53300661.947122864,
      "logps/chosen": -470.7699175824176,
      "logps/rejected": -392.91937208398133,
      "loss": 0.1995,
      "loss/base_kto": 1.5957868099212646,
      "metrics/advantage_var": 14841.4296875,
      "rewards/chosen": 2.8295043753679354,
      "rewards/margins": 5.935396099949973,
      "rewards/rejected": -3.105891724582037,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.252459526062012,
      "kl": 13.560362815856934,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -52936039.783783786,
      "logits/rejected": -52383545.80645161,
      "logps/chosen": -459.2036963434022,
      "logps/rejected": -415.0323540706605,
      "loss": 0.2088,
      "loss/base_kto": 1.6702415943145752,
      "metrics/advantage_var": 14908.0498046875,
      "rewards/chosen": 2.6849551542006163,
      "rewards/margins": 5.633000079075808,
      "rewards/rejected": -2.948044924875192,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 7.345427513122559,
      "kl": 17.02194595336914,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -52286718.32679739,
      "logits/rejected": -52460586.922155686,
      "logps/chosen": -440.9954044117647,
      "logps/rejected": -395.601188248503,
      "loss": 0.2281,
      "loss/base_kto": 1.8249582052230835,
      "metrics/advantage_var": 14355.4287109375,
      "rewards/chosen": 2.5418543597452,
      "rewards/margins": 5.323924862294788,
      "rewards/rejected": -2.782070502549588,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 6.615070343017578,
      "kl": 12.28957462310791,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -52443794.28571428,
      "logits/rejected": -52651681.684210524,
      "logps/chosen": -479.6156994047619,
      "logps/rejected": -385.69053248355266,
      "loss": 0.2065,
      "loss/base_kto": 1.6521881818771362,
      "metrics/advantage_var": 15773.1162109375,
      "rewards/chosen": 2.806511470249721,
      "rewards/margins": 5.816383505226078,
      "rewards/rejected": -3.009872034976357,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.969850461966828e+17,
      "train_loss": 0.2751038508917589,
      "train_runtime": 11048.6004,
      "train_samples_per_second": 13.415,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.969850461966828e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}