{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 2316,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025906735751295335,
      "grad_norm": 10.65859317779541,
      "kl": 44.20431137084961,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -36499456.0,
      "logits/rejected": -37668092.71794872,
      "logps/chosen": -490.6620617378049,
      "logps/rejected": -359.51189403044873,
      "loss": 0.4506,
      "loss/base_kto": 3.6045689582824707,
      "metrics/advantage_var": 861.9415283203125,
      "rewards/chosen": 0.9872177868354611,
      "rewards/margins": 0.4244383358075665,
      "rewards/rejected": 0.5627794510278946,
      "step": 20
    },
    {
      "epoch": 0.05181347150259067,
      "grad_norm": 10.56235408782959,
      "kl": 44.744598388671875,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -38137711.73047473,
      "logits/rejected": -37998056.31897926,
      "logps/chosen": -475.3998372894334,
      "logps/rejected": -398.91088516746413,
      "loss": 0.4484,
      "loss/base_kto": 3.587329149246216,
      "metrics/advantage_var": 992.3124389648438,
      "rewards/chosen": 0.9791349495716883,
      "rewards/margins": 0.3998072552248483,
      "rewards/rejected": 0.5793276943468401,
      "step": 40
    },
    {
      "epoch": 0.07772020725388601,
      "grad_norm": 9.909646034240723,
      "kl": 27.297826766967773,
      "learning_rate": 5.9e-07,
      "logits/chosen": -36635869.108728945,
      "logits/rejected": -37625764.54226475,
      "logps/chosen": -487.3697358346095,
      "logps/rejected": -386.92222388357254,
      "loss": 0.4302,
      "loss/base_kto": 3.441992998123169,
      "metrics/advantage_var": 1028.0374755859375,
      "rewards/chosen": 0.8436054971645292,
      "rewards/margins": 0.5919641917247489,
      "rewards/rejected": 0.2516413054397802,
      "step": 60
    },
    {
      "epoch": 0.10362694300518134,
      "grad_norm": 11.076920509338379,
      "kl": 6.907355785369873,
      "learning_rate": 7.9e-07,
      "logits/chosen": -38943921.03050847,
      "logits/rejected": -39010942.14492754,
      "logps/chosen": -488.87748940677966,
      "logps/rejected": -398.9723731884058,
      "loss": 0.4244,
      "loss/base_kto": 3.3955726623535156,
      "metrics/advantage_var": 1177.5146484375,
      "rewards/chosen": 0.043001853813559324,
      "rewards/margins": 0.745197527217092,
      "rewards/rejected": -0.7021956734035326,
      "step": 80
    },
    {
      "epoch": 0.12953367875647667,
      "grad_norm": 7.774325370788574,
      "kl": 15.655855178833008,
      "learning_rate": 9.9e-07,
      "logits/chosen": -37001158.757763974,
      "logits/rejected": -38360018.91823899,
      "logps/chosen": -479.5358598602484,
      "logps/rejected": -358.2423840408805,
      "loss": 0.428,
      "loss/base_kto": 3.424107313156128,
      "metrics/advantage_var": 1501.2918701171875,
      "rewards/chosen": 0.3111519191576087,
      "rewards/margins": 0.7818387297290513,
      "rewards/rejected": -0.4706868105714426,
      "step": 100
    },
    {
      "epoch": 0.15544041450777202,
      "grad_norm": 8.679657936096191,
      "kl": 29.89056396484375,
      "learning_rate": 9.998186234298189e-07,
      "logits/chosen": -40178530.96485623,
      "logits/rejected": -39543200.48929664,
      "logps/chosen": -461.1728234824281,
      "logps/rejected": -375.8446339831804,
      "loss": 0.407,
      "loss/base_kto": 3.2559025287628174,
      "metrics/advantage_var": 2275.13330078125,
      "rewards/chosen": 1.0524772668418032,
      "rewards/margins": 1.0377021831424456,
      "rewards/rejected": 0.014775083699357618,
      "step": 120
    },
    {
      "epoch": 0.18134715025906736,
      "grad_norm": 7.838543891906738,
      "kl": 35.66352462768555,
      "learning_rate": 9.992359552107714e-07,
      "logits/chosen": -41033721.895678096,
      "logits/rejected": -41711303.25123153,
      "logps/chosen": -484.4836997019374,
      "logps/rejected": -369.20089285714283,
      "loss": 0.3773,
      "loss/base_kto": 3.0183520317077637,
      "metrics/advantage_var": 2924.481689453125,
      "rewards/chosen": 1.3930760481673807,
      "rewards/margins": 1.472008429783231,
      "rewards/rejected": -0.07893238161585014,
      "step": 140
    },
    {
      "epoch": 0.20725388601036268,
      "grad_norm": 6.404437065124512,
      "kl": 35.48348617553711,
      "learning_rate": 9.982519612796161e-07,
      "logits/chosen": -40603175.384615384,
      "logits/rejected": -40007502.048780486,
      "logps/chosen": -464.8622796474359,
      "logps/rejected": -349.6068502286585,
      "loss": 0.3852,
      "loss/base_kto": 3.0814366340637207,
      "metrics/advantage_var": 3211.025634765625,
      "rewards/chosen": 1.32285641401242,
      "rewards/margins": 1.3913330502477268,
      "rewards/rejected": -0.0684766362353069,
      "step": 160
    },
    {
      "epoch": 0.23316062176165803,
      "grad_norm": 8.489002227783203,
      "kl": 15.075944900512695,
      "learning_rate": 9.968674326492213e-07,
      "logits/chosen": -39859590.171073094,
      "logits/rejected": -39911282.53689168,
      "logps/chosen": -446.71646578538105,
      "logps/rejected": -366.1639030612245,
      "loss": 0.3852,
      "loss/base_kto": 3.081433057785034,
      "metrics/advantage_var": 3389.28564453125,
      "rewards/chosen": 0.8286895040034506,
      "rewards/margins": 1.4900147161796673,
      "rewards/rejected": -0.6613252121762166,
      "step": 180
    },
    {
      "epoch": 0.25906735751295334,
      "grad_norm": 8.60866641998291,
      "kl": 37.68467712402344,
      "learning_rate": 9.950834823142198e-07,
      "logits/chosen": -39526004.03169572,
      "logits/rejected": -41538766.693374425,
      "logps/chosen": -486.00425911251983,
      "logps/rejected": -376.5807251540832,
      "loss": 0.3715,
      "loss/base_kto": 2.971728801727295,
      "metrics/advantage_var": 4009.591796875,
      "rewards/chosen": 1.279915932052298,
      "rewards/margins": 1.6364233612251842,
      "rewards/rejected": -0.3565074291728862,
      "step": 200
    },
    {
      "epoch": 0.2849740932642487,
      "grad_norm": 9.43969440460205,
      "kl": 11.000213623046875,
      "learning_rate": 9.929015443562942e-07,
      "logits/chosen": -41568631.304897316,
      "logits/rejected": -40817902.98608965,
      "logps/chosen": -473.59414494470775,
      "logps/rejected": -385.6153641808346,
      "loss": 0.3744,
      "loss/base_kto": 2.9952468872070312,
      "metrics/advantage_var": 5182.689453125,
      "rewards/chosen": 0.8071200354216035,
      "rewards/margins": 1.9288033831442657,
      "rewards/rejected": -1.1216833477226622,
      "step": 220
    },
    {
      "epoch": 0.31088082901554404,
      "grad_norm": 7.523622989654541,
      "kl": 20.298025131225586,
      "learning_rate": 9.90323372791347e-07,
      "logits/chosen": -42630571.063565895,
      "logits/rejected": -41810284.44724409,
      "logps/chosen": -463.9735465116279,
      "logps/rejected": -392.01377952755905,
      "loss": 0.3711,
      "loss/base_kto": 2.9685542583465576,
      "metrics/advantage_var": 4850.92333984375,
      "rewards/chosen": 1.0982537321342054,
      "rewards/margins": 1.809265627741981,
      "rewards/rejected": -0.7110118956077756,
      "step": 240
    },
    {
      "epoch": 0.33678756476683935,
      "grad_norm": 8.435532569885254,
      "kl": 42.418304443359375,
      "learning_rate": 9.87351040159484e-07,
      "logits/chosen": -43215823.382789314,
      "logits/rejected": -43751947.828382835,
      "logps/chosen": -462.843471810089,
      "logps/rejected": -373.2071988448845,
      "loss": 0.3355,
      "loss/base_kto": 2.683739423751831,
      "metrics/advantage_var": 4713.56689453125,
      "rewards/chosen": 1.9778451438473665,
      "rewards/margins": 2.1419684272084174,
      "rewards/rejected": -0.16412328336105095,
      "step": 260
    },
    {
      "epoch": 0.3626943005181347,
      "grad_norm": 9.643255233764648,
      "kl": 23.864294052124023,
      "learning_rate": 9.839869358589396e-07,
      "logits/chosen": -45868187.99396682,
      "logits/rejected": -46083100.213938415,
      "logps/chosen": -479.51649698340873,
      "logps/rejected": -361.66473358995137,
      "loss": 0.3686,
      "loss/base_kto": 2.949044704437256,
      "metrics/advantage_var": 5534.42333984375,
      "rewards/chosen": 1.4372264004760558,
      "rewards/margins": 2.038752002399425,
      "rewards/rejected": -0.6015256019233691,
      "step": 280
    },
    {
      "epoch": 0.38860103626943004,
      "grad_norm": 7.352804660797119,
      "kl": 19.898496627807617,
      "learning_rate": 9.80233764225289e-07,
      "logits/chosen": -45201262.40758294,
      "logits/rejected": -44915165.972179286,
      "logps/chosen": -481.7296603475513,
      "logps/rejected": -392.0678129829985,
      "loss": 0.3648,
      "loss/base_kto": 2.918400287628174,
      "metrics/advantage_var": 7212.60888671875,
      "rewards/chosen": 1.2794505717441746,
      "rewards/margins": 2.153510517135162,
      "rewards/rejected": -0.8740599453909873,
      "step": 300
    },
    {
      "epoch": 0.41450777202072536,
      "grad_norm": 9.16907024383545,
      "kl": 43.78324890136719,
      "learning_rate": 9.760945423574868e-07,
      "logits/chosen": -45530638.006079026,
      "logits/rejected": -44813861.86495177,
      "logps/chosen": -477.1079502279635,
      "logps/rejected": -363.2409314710611,
      "loss": 0.3463,
      "loss/base_kto": 2.7703616619110107,
      "metrics/advantage_var": 7601.89990234375,
      "rewards/chosen": 1.9827135080262157,
      "rewards/margins": 2.358483512642122,
      "rewards/rejected": -0.3757700046159064,
      "step": 320
    },
    {
      "epoch": 0.44041450777202074,
      "grad_norm": 7.767402648925781,
      "kl": 31.018543243408203,
      "learning_rate": 9.71572597692482e-07,
      "logits/chosen": -43233508.2700157,
      "logits/rejected": -43803146.35147745,
      "logps/chosen": -458.922193877551,
      "logps/rejected": -372.84705482115083,
      "loss": 0.3386,
      "loss/base_kto": 2.709152936935425,
      "metrics/advantage_var": 9589.0205078125,
      "rewards/chosen": 1.8369244106897566,
      "rewards/margins": 2.9127549444426726,
      "rewards/rejected": -1.075830533752916,
      "step": 340
    },
    {
      "epoch": 0.46632124352331605,
      "grad_norm": 7.376603603363037,
      "kl": 21.673208236694336,
      "learning_rate": 9.666715653303588e-07,
      "logits/chosen": -44333835.43913713,
      "logits/rejected": -43826385.34389857,
      "logps/chosen": -473.8385496918336,
      "logps/rejected": -398.9099643423138,
      "loss": 0.3491,
      "loss/base_kto": 2.79294490814209,
      "metrics/advantage_var": 8912.041015625,
      "rewards/chosen": 1.4503943115610554,
      "rewards/margins": 2.665896970411095,
      "rewards/rejected": -1.2155026588500397,
      "step": 360
    },
    {
      "epoch": 0.49222797927461137,
      "grad_norm": 7.592734336853027,
      "kl": 19.095108032226562,
      "learning_rate": 9.613953851121528e-07,
      "logits/chosen": -43271092.090771556,
      "logits/rejected": -45212693.50565428,
      "logps/chosen": -486.91579992435703,
      "logps/rejected": -382.0510147415186,
      "loss": 0.3425,
      "loss/base_kto": 2.739999532699585,
      "metrics/advantage_var": 8978.3515625,
      "rewards/chosen": 1.8050278564083775,
      "rewards/margins": 2.751250756404591,
      "rewards/rejected": -0.9462228999962137,
      "step": 380
    },
    {
      "epoch": 0.5181347150259067,
      "grad_norm": 6.854775428771973,
      "kl": 45.80805206298828,
      "learning_rate": 9.557482984526924e-07,
      "logits/chosen": -43894309.42457232,
      "logits/rejected": -44310335.09576138,
      "logps/chosen": -443.4170392690513,
      "logps/rejected": -366.22024627158555,
      "loss": 0.3496,
      "loss/base_kto": 2.7964887619018555,
      "metrics/advantage_var": 7448.00390625,
      "rewards/chosen": 1.9952160967267691,
      "rewards/margins": 2.3970563785501353,
      "rewards/rejected": -0.40184028182336634,
      "step": 400
    },
    {
      "epoch": 0.5440414507772021,
      "grad_norm": 10.735581398010254,
      "kl": 19.892004013061523,
      "learning_rate": 9.497348449310107e-07,
      "logits/chosen": -44185031.67159277,
      "logits/rejected": -44101271.84500745,
      "logps/chosen": -462.24266215106735,
      "logps/rejected": -365.46805141579733,
      "loss": 0.3534,
      "loss/base_kto": 2.827474355697632,
      "metrics/advantage_var": 8355.3662109375,
      "rewards/chosen": 1.2435715648732553,
      "rewards/margins": 2.6411173694385033,
      "rewards/rejected": -1.3975458045652478,
      "step": 420
    },
    {
      "epoch": 0.5699481865284974,
      "grad_norm": 8.053645133972168,
      "kl": 14.910491943359375,
      "learning_rate": 9.433598586410701e-07,
      "logits/chosen": -46035449.7939394,
      "logits/rejected": -45075168.61935484,
      "logps/chosen": -498.8833333333333,
      "logps/rejected": -387.8259072580645,
      "loss": 0.3487,
      "loss/base_kto": 2.789438247680664,
      "metrics/advantage_var": 8716.130859375,
      "rewards/chosen": 1.1289883700284091,
      "rewards/margins": 2.8263150301846593,
      "rewards/rejected": -1.69732666015625,
      "step": 440
    },
    {
      "epoch": 0.5958549222797928,
      "grad_norm": 7.885801315307617,
      "kl": 33.040191650390625,
      "learning_rate": 9.366284643057313e-07,
      "logits/chosen": -45832788.90452261,
      "logits/rejected": -46052329.51098096,
      "logps/chosen": -454.71492881072027,
      "logps/rejected": -410.90794289897514,
      "loss": 0.3342,
      "loss/base_kto": 2.6736834049224854,
      "metrics/advantage_var": 9005.0234375,
      "rewards/chosen": 1.7976493388164783,
      "rewards/margins": 2.8960478406580776,
      "rewards/rejected": -1.0983985018415996,
      "step": 460
    },
    {
      "epoch": 0.6217616580310881,
      "grad_norm": 6.836333274841309,
      "kl": 28.7034854888916,
      "learning_rate": 9.295460731570917e-07,
      "logits/chosen": -46365180.05546995,
      "logits/rejected": -45693645.28684627,
      "logps/chosen": -479.06798921417567,
      "logps/rejected": -382.8000198098257,
      "loss": 0.3247,
      "loss/base_kto": 2.5976855754852295,
      "metrics/advantage_var": 10056.6845703125,
      "rewards/chosen": 1.8589578167132126,
      "rewards/margins": 3.031334252925574,
      "rewards/rejected": -1.1723764362123614,
      "step": 480
    },
    {
      "epoch": 0.6476683937823834,
      "grad_norm": 8.679668426513672,
      "kl": 32.58065414428711,
      "learning_rate": 9.221183785865056e-07,
      "logits/chosen": -44614913.69536424,
      "logits/rejected": -45147469.25443787,
      "logps/chosen": -472.30805049668874,
      "logps/rejected": -384.6820451183432,
      "loss": 0.3347,
      "loss/base_kto": 2.677523612976074,
      "metrics/advantage_var": 9521.0166015625,
      "rewards/chosen": 1.824923685844371,
      "rewards/margins": 3.0118727442416526,
      "rewards/rejected": -1.1869490583972817,
      "step": 500
    },
    {
      "epoch": 0.6735751295336787,
      "grad_norm": 7.148499488830566,
      "kl": 25.07105827331543,
      "learning_rate": 9.143513515677817e-07,
      "logits/chosen": -46456619.18072289,
      "logits/rejected": -44846964.36363637,
      "logps/chosen": -492.8112763554217,
      "logps/rejected": -390.60830965909093,
      "loss": 0.3248,
      "loss/base_kto": 2.598315954208374,
      "metrics/advantage_var": 11530.103515625,
      "rewards/chosen": 1.8040299013436558,
      "rewards/margins": 3.2287691940333882,
      "rewards/rejected": -1.4247392926897322,
      "step": 520
    },
    {
      "epoch": 0.6994818652849741,
      "grad_norm": 10.257616996765137,
      "kl": 41.308380126953125,
      "learning_rate": 9.062512358572373e-07,
      "logits/chosen": -46776827.9300477,
      "logits/rejected": -45388855.05376344,
      "logps/chosen": -477.1953994435612,
      "logps/rejected": -375.51804915514595,
      "loss": 0.3344,
      "loss/base_kto": 2.674875259399414,
      "metrics/advantage_var": 9839.3037109375,
      "rewards/chosen": 2.125871958527921,
      "rewards/margins": 2.839637327011504,
      "rewards/rejected": -0.713765368483583,
      "step": 540
    },
    {
      "epoch": 0.7253886010362695,
      "grad_norm": 8.11175537109375,
      "kl": 27.59626579284668,
      "learning_rate": 8.978245429744691e-07,
      "logits/chosen": -45277500.83572568,
      "logits/rejected": -44860392.477794796,
      "logps/chosen": -462.4535486443381,
      "logps/rejected": -370.9771248085758,
      "loss": 0.3345,
      "loss/base_kto": 2.676085948944092,
      "metrics/advantage_var": 10163.2373046875,
      "rewards/chosen": 1.8515385531922848,
      "rewards/margins": 2.9131068557382265,
      "rewards/rejected": -1.0615683025459417,
      "step": 560
    },
    {
      "epoch": 0.7512953367875648,
      "grad_norm": 8.053217887878418,
      "kl": 17.627090454101562,
      "learning_rate": 8.890780469678738e-07,
      "logits/chosen": -44548563.93114241,
      "logits/rejected": -46489897.13572543,
      "logps/chosen": -457.03032081377154,
      "logps/rejected": -408.0214996099844,
      "loss": 0.3287,
      "loss/base_kto": 2.6296844482421875,
      "metrics/advantage_var": 12007.1591796875,
      "rewards/chosen": 1.6501634863434076,
      "rewards/margins": 3.347522425622854,
      "rewards/rejected": -1.6973589392794461,
      "step": 580
    },
    {
      "epoch": 0.7772020725388601,
      "grad_norm": 7.483168125152588,
      "kl": 32.44775390625,
      "learning_rate": 8.800187789691269e-07,
      "logits/chosen": -48252049.8176,
      "logits/rejected": -47718249.917557254,
      "logps/chosen": -448.42165,
      "logps/rejected": -407.2638358778626,
      "loss": 0.3299,
      "loss/base_kto": 2.6391899585723877,
      "metrics/advantage_var": 10226.193359375,
      "rewards/chosen": 2.0992955078125,
      "rewards/margins": 3.1892319753399336,
      "rewards/rejected": -1.0899364675274332,
      "step": 600
    },
    {
      "epoch": 0.8031088082901554,
      "grad_norm": 6.4975266456604,
      "kl": 20.634342193603516,
      "learning_rate": 8.706540215409981e-07,
      "logits/chosen": -45601083.33225284,
      "logits/rejected": -46161734.66063348,
      "logps/chosen": -477.0887358184765,
      "logps/rejected": -379.0682032428356,
      "loss": 0.3278,
      "loss/base_kto": 2.62276554107666,
      "metrics/advantage_var": 10482.455078125,
      "rewards/chosen": 1.7784174543532212,
      "rewards/margins": 3.2988043933084907,
      "rewards/rejected": -1.5203869389552696,
      "step": 620
    },
    {
      "epoch": 0.8290155440414507,
      "grad_norm": 9.11014461517334,
      "kl": 18.934978485107422,
      "learning_rate": 8.609913028230462e-07,
      "logits/chosen": -44933326.67889908,
      "logits/rejected": -45986891.24600639,
      "logps/chosen": -472.6990634556575,
      "logps/rejected": -383.70162739616615,
      "loss": 0.3266,
      "loss/base_kto": 2.6129066944122314,
      "metrics/advantage_var": 10181.88671875,
      "rewards/chosen": 1.4984698281011086,
      "rewards/margins": 3.1533712046486326,
      "rewards/rejected": -1.654901376547524,
      "step": 640
    },
    {
      "epoch": 0.8549222797927462,
      "grad_norm": 10.547191619873047,
      "kl": 19.205251693725586,
      "learning_rate": 8.510383904798994e-07,
      "logits/chosen": -46755092.93081761,
      "logits/rejected": -44693739.32919255,
      "logps/chosen": -464.08451257861634,
      "logps/rejected": -399.30221273291926,
      "loss": 0.3362,
      "loss/base_kto": 2.6896512508392334,
      "metrics/advantage_var": 9943.228515625,
      "rewards/chosen": 1.3194805601108,
      "rewards/margins": 2.960457175684849,
      "rewards/rejected": -1.6409766155740488,
      "step": 660
    },
    {
      "epoch": 0.8808290155440415,
      "grad_norm": 7.576268196105957,
      "kl": 30.703176498413086,
      "learning_rate": 8.408032854569865e-07,
      "logits/chosen": -46373757.61490683,
      "logits/rejected": -46271948.47798742,
      "logps/chosen": -466.3391886645963,
      "logps/rejected": -374.7342521619497,
      "loss": 0.318,
      "loss/base_kto": 2.5438382625579834,
      "metrics/advantage_var": 10507.3154296875,
      "rewards/chosen": 2.4138117251188858,
      "rewards/margins": 3.4287190821968143,
      "rewards/rejected": -1.0149073570779286,
      "step": 680
    },
    {
      "epoch": 0.9067357512953368,
      "grad_norm": 7.023205757141113,
      "kl": 22.83217430114746,
      "learning_rate": 8.302942155487377e-07,
      "logits/chosen": -47770873.25271318,
      "logits/rejected": -47177956.988976374,
      "logps/chosen": -470.42790697674417,
      "logps/rejected": -380.4131151574803,
      "loss": 0.304,
      "loss/base_kto": 2.4317188262939453,
      "metrics/advantage_var": 10339.8466796875,
      "rewards/chosen": 2.1665962633236435,
      "rewards/margins": 3.6137248235475607,
      "rewards/rejected": -1.4471285602239172,
      "step": 700
    },
    {
      "epoch": 0.9326424870466321,
      "grad_norm": 6.914972305297852,
      "kl": 21.22562026977539,
      "learning_rate": 8.195196287844278e-07,
      "logits/chosen": -49408567.5980551,
      "logits/rejected": -49626354.48567119,
      "logps/chosen": -461.87180915721234,
      "logps/rejected": -400.6054864253394,
      "loss": 0.3195,
      "loss/base_kto": 2.55560040473938,
      "metrics/advantage_var": 12679.2861328125,
      "rewards/chosen": 1.997904822351094,
      "rewards/margins": 3.673362024462708,
      "rewards/rejected": -1.6754572021116139,
      "step": 720
    },
    {
      "epoch": 0.9585492227979274,
      "grad_norm": 9.571149826049805,
      "kl": 8.873680114746094,
      "learning_rate": 8.08488186636975e-07,
      "logits/chosen": -49232709.2327044,
      "logits/rejected": -48566799.900621116,
      "logps/chosen": -461.3564760220126,
      "logps/rejected": -389.0656541149068,
      "loss": 0.3329,
      "loss/base_kto": 2.6631648540496826,
      "metrics/advantage_var": 12759.3857421875,
      "rewards/chosen": 1.0083675744398586,
      "rewards/margins": 3.495386420276427,
      "rewards/rejected": -2.4870188458365683,
      "step": 740
    },
    {
      "epoch": 0.9844559585492227,
      "grad_norm": 7.072620868682861,
      "kl": 29.986997604370117,
      "learning_rate": 7.972087570601576e-07,
      "logits/chosen": -48027195.284210525,
      "logits/rejected": -48910156.17560975,
      "logps/chosen": -481.89906015037593,
      "logps/rejected": -372.8478912601626,
      "loss": 0.3108,
      "loss/base_kto": 2.4860572814941406,
      "metrics/advantage_var": 12530.525390625,
      "rewards/chosen": 2.4417236328125,
      "rewards/margins": 3.6977175828887194,
      "rewards/rejected": -1.2559939500762196,
      "step": 760
    },
    {
      "epoch": 1.0103626943005182,
      "grad_norm": 5.7229814529418945,
      "kl": 31.123401641845703,
      "learning_rate": 7.856904073598458e-07,
      "logits/chosen": -49082970.35294118,
      "logits/rejected": -49505685.06329114,
      "logps/chosen": -450.6265963622291,
      "logps/rejected": -376.8190763449367,
      "loss": 0.287,
      "loss/base_kto": 2.2956960201263428,
      "metrics/advantage_var": 11794.609375,
      "rewards/chosen": 2.3705805255901704,
      "rewards/margins": 3.957680204498893,
      "rewards/rejected": -1.5870996789087224,
      "step": 780
    },
    {
      "epoch": 1.0362694300518134,
      "grad_norm": 8.308485984802246,
      "kl": 21.34233283996582,
      "learning_rate": 7.739423969049761e-07,
      "logits/chosen": -48664785.53135314,
      "logits/rejected": -49527075.703264095,
      "logps/chosen": -452.4751443894389,
      "logps/rejected": -387.68386034866467,
      "loss": 0.2552,
      "loss/base_kto": 2.0418756008148193,
      "metrics/advantage_var": 15537.6943359375,
      "rewards/chosen": 2.2635937177702146,
      "rewards/margins": 4.9204337846517054,
      "rewards/rejected": -2.656840066881491,
      "step": 800
    },
    {
      "epoch": 1.0621761658031088,
      "grad_norm": 7.449312686920166,
      "kl": 19.720563888549805,
      "learning_rate": 7.619741696841322e-07,
      "logits/chosen": -46948384.35387046,
      "logits/rejected": -48699420.48840804,
      "logps/chosen": -446.41409952606637,
      "logps/rejected": -402.2554095826893,
      "loss": 0.2566,
      "loss/base_kto": 2.0527331829071045,
      "metrics/advantage_var": 14032.3798828125,
      "rewards/chosen": 2.2591982776708135,
      "rewards/margins": 4.754287164480125,
      "rewards/rejected": -2.495088886809312,
      "step": 820
    },
    {
      "epoch": 1.0880829015544042,
      "grad_norm": 6.976689338684082,
      "kl": 14.78875732421875,
      "learning_rate": 7.497953467137135e-07,
      "logits/chosen": -49828502.974358976,
      "logits/rejected": -51138731.70731708,
      "logps/chosen": -471.2696314102564,
      "logps/rejected": -404.3307450457317,
      "loss": 0.2542,
      "loss/base_kto": 2.033315420150757,
      "metrics/advantage_var": 14103.1162109375,
      "rewards/chosen": 1.9908480521960137,
      "rewards/margins": 4.918190536833018,
      "rewards/rejected": -2.9273424846370046,
      "step": 840
    },
    {
      "epoch": 1.1139896373056994,
      "grad_norm": 9.26498794555664,
      "kl": 37.292884826660156,
      "learning_rate": 7.37415718303793e-07,
      "logits/chosen": -48743260.09508716,
      "logits/rejected": -49833173.0046225,
      "logps/chosen": -453.1921057844691,
      "logps/rejected": -394.1962153312789,
      "loss": 0.2582,
      "loss/base_kto": 2.065633773803711,
      "metrics/advantage_var": 13541.900390625,
      "rewards/chosen": 2.3661943003417196,
      "rewards/margins": 4.667860513084882,
      "rewards/rejected": -2.3016662127431626,
      "step": 860
    },
    {
      "epoch": 1.1398963730569949,
      "grad_norm": 8.440954208374023,
      "kl": 24.506061553955078,
      "learning_rate": 7.248452361878855e-07,
      "logits/chosen": -47704310.278481014,
      "logits/rejected": -48767576.493827164,
      "logps/chosen": -438.6851265822785,
      "logps/rejected": -387.41307388117286,
      "loss": 0.2668,
      "loss/base_kto": 2.134312391281128,
      "metrics/advantage_var": 13320.8251953125,
      "rewards/chosen": 2.290668149537678,
      "rewards/margins": 4.493615318712061,
      "rewards/rejected": -2.202947169174383,
      "step": 880
    },
    {
      "epoch": 1.16580310880829,
      "grad_norm": 6.1074981689453125,
      "kl": 12.768942832946777,
      "learning_rate": 7.120940055229497e-07,
      "logits/chosen": -47911413.73065902,
      "logits/rejected": -47319152.604811,
      "logps/chosen": -444.51844555873924,
      "logps/rejected": -397.1567869415808,
      "loss": 0.2375,
      "loss/base_kto": 1.900206446647644,
      "metrics/advantage_var": 13858.2177734375,
      "rewards/chosen": 2.4379781378715975,
      "rewards/margins": 4.967347793624926,
      "rewards/rejected": -2.529369655753329,
      "step": 900
    },
    {
      "epoch": 1.1917098445595855,
      "grad_norm": 6.856977462768555,
      "kl": 25.13776969909668,
      "learning_rate": 6.991722767660556e-07,
      "logits/chosen": -48549149.80542265,
      "logits/rejected": -49921875.50382848,
      "logps/chosen": -450.0487440191388,
      "logps/rejected": -371.0899693721286,
      "loss": 0.2512,
      "loss/base_kto": 2.009587287902832,
      "metrics/advantage_var": 13175.6533203125,
      "rewards/chosen": 2.4123200290320974,
      "rewards/margins": 4.605071856534969,
      "rewards/rejected": -2.1927518275028715,
      "step": 920
    },
    {
      "epoch": 1.2176165803108807,
      "grad_norm": 5.254368305206299,
      "kl": 30.324390411376953,
      "learning_rate": 6.860904374342499e-07,
      "logits/chosen": -50275765.937106915,
      "logits/rejected": -50992865.78881988,
      "logps/chosen": -452.25776336477986,
      "logps/rejected": -388.415542507764,
      "loss": 0.2595,
      "loss/base_kto": 2.076131582260132,
      "metrics/advantage_var": 14928.7568359375,
      "rewards/chosen": 2.6799888370921776,
      "rewards/margins": 4.889126179083152,
      "rewards/rejected": -2.2091373419909743,
      "step": 940
    },
    {
      "epoch": 1.2435233160621761,
      "grad_norm": 5.639325141906738,
      "kl": 24.19855308532715,
      "learning_rate": 6.7285900375424e-07,
      "logits/chosen": -50279631.332302935,
      "logits/rejected": -49850833.8957346,
      "logps/chosen": -469.4557573415765,
      "logps/rejected": -370.7182316350711,
      "loss": 0.2498,
      "loss/base_kto": 1.9982547760009766,
      "metrics/advantage_var": 13618.6552734375,
      "rewards/chosen": 2.387874927550232,
      "rewards/margins": 4.803065565509355,
      "rewards/rejected": -2.4151906379591233,
      "step": 960
    },
    {
      "epoch": 1.2694300518134716,
      "grad_norm": 8.131012916564941,
      "kl": 11.774667739868164,
      "learning_rate": 6.594886122086123e-07,
      "logits/chosen": -50578138.77061469,
      "logits/rejected": -49652056.11745514,
      "logps/chosen": -481.5137275112444,
      "logps/rejected": -396.49551386623165,
      "loss": 0.2616,
      "loss/base_kto": 2.0926153659820557,
      "metrics/advantage_var": 14772.4716796875,
      "rewards/chosen": 2.2389902460878934,
      "rewards/margins": 4.830445802139076,
      "rewards/rejected": -2.5914555560511827,
      "step": 980
    },
    {
      "epoch": 1.2953367875647668,
      "grad_norm": 10.34538459777832,
      "kl": 21.447887420654297,
      "learning_rate": 6.459900109853766e-07,
      "logits/chosen": -50675736.78668684,
      "logits/rejected": -52042747.0371567,
      "logps/chosen": -479.4501701966717,
      "logps/rejected": -391.41013731825524,
      "loss": 0.2643,
      "loss/base_kto": 2.114217519760132,
      "metrics/advantage_var": 13216.7001953125,
      "rewards/chosen": 2.6053697640885023,
      "rewards/margins": 4.449835226275497,
      "rewards/rejected": -1.8444654621869951,
      "step": 1000
    },
    {
      "epoch": 1.3212435233160622,
      "grad_norm": 7.639214992523193,
      "kl": 25.33041763305664,
      "learning_rate": 6.323740513377171e-07,
      "logits/chosen": -50322896.25723473,
      "logits/rejected": -51459955.93920973,
      "logps/chosen": -436.558530948553,
      "logps/rejected": -399.7970649696049,
      "loss": 0.2517,
      "loss/base_kto": 2.0138232707977295,
      "metrics/advantage_var": 15641.556640625,
      "rewards/chosen": 2.574196769493569,
      "rewards/margins": 5.121921623751263,
      "rewards/rejected": -2.5477248542576936,
      "step": 1020
    },
    {
      "epoch": 1.3471502590673574,
      "grad_norm": 7.394321918487549,
      "kl": 20.35123062133789,
      "learning_rate": 6.186516788608865e-07,
      "logits/chosen": -50750205.15103339,
      "logits/rejected": -51357035.35483871,
      "logps/chosen": -474.0139109697933,
      "logps/rejected": -405.144777265745,
      "loss": 0.2626,
      "loss/base_kto": 2.1008141040802,
      "metrics/advantage_var": 13648.5361328125,
      "rewards/chosen": 2.109104465793919,
      "rewards/margins": 4.62314536441143,
      "rewards/rejected": -2.5140408986175116,
      "step": 1040
    },
    {
      "epoch": 1.3730569948186528,
      "grad_norm": 8.497376441955566,
      "kl": 23.546171188354492,
      "learning_rate": 6.048339246932652e-07,
      "logits/chosen": -48032839.29113924,
      "logits/rejected": -50251178.666666664,
      "logps/chosen": -456.9215783227848,
      "logps/rejected": -416.03303433641975,
      "loss": 0.2612,
      "loss/base_kto": 2.0899407863616943,
      "metrics/advantage_var": 14816.8515625,
      "rewards/chosen": 2.5009827432753164,
      "rewards/margins": 4.854381572605949,
      "rewards/rejected": -2.353398829330633,
      "step": 1060
    },
    {
      "epoch": 1.3989637305699483,
      "grad_norm": 6.526173114776611,
      "kl": 19.82684898376465,
      "learning_rate": 5.909318966486494e-07,
      "logits/chosen": -50905499.44227886,
      "logits/rejected": -50662217.918433934,
      "logps/chosen": -462.2083020989505,
      "logps/rejected": -396.3524418841762,
      "loss": 0.2414,
      "loss/base_kto": 1.930864691734314,
      "metrics/advantage_var": 13378.1474609375,
      "rewards/chosen": 2.5639007888633807,
      "rewards/margins": 4.943357848788952,
      "rewards/rejected": -2.379457059925571,
      "step": 1080
    },
    {
      "epoch": 1.4248704663212435,
      "grad_norm": 6.579440593719482,
      "kl": 25.88844871520996,
      "learning_rate": 5.769567702869052e-07,
      "logits/chosen": -52712109.75565611,
      "logits/rejected": -52538728.14262561,
      "logps/chosen": -444.0515177224736,
      "logps/rejected": -401.4771576175041,
      "loss": 0.2608,
      "loss/base_kto": 2.086622714996338,
      "metrics/advantage_var": 13585.439453125,
      "rewards/chosen": 2.560188397023473,
      "rewards/margins": 4.714666695810446,
      "rewards/rejected": -2.154478298786973,
      "step": 1100
    },
    {
      "epoch": 1.450777202072539,
      "grad_norm": 9.067955017089844,
      "kl": 21.861936569213867,
      "learning_rate": 5.629197799301614e-07,
      "logits/chosen": -52116110.97659907,
      "logits/rejected": -52317502.89827856,
      "logps/chosen": -483.269988299532,
      "logps/rejected": -408.1690385367762,
      "loss": 0.2603,
      "loss/base_kto": 2.082534074783325,
      "metrics/advantage_var": 14696.2470703125,
      "rewards/chosen": 2.6743337360447543,
      "rewards/margins": 4.77644622106774,
      "rewards/rejected": -2.1021124850229853,
      "step": 1120
    },
    {
      "epoch": 1.4766839378238341,
      "grad_norm": 6.779482841491699,
      "kl": 25.322052001953125,
      "learning_rate": 5.488322096317633e-07,
      "logits/chosen": -50702644.60273973,
      "logits/rejected": -50894502.83146068,
      "logps/chosen": -490.74714611872145,
      "logps/rejected": -384.28772070626,
      "loss": 0.2773,
      "loss/base_kto": 2.2181718349456787,
      "metrics/advantage_var": 14075.6591796875,
      "rewards/chosen": 2.5817913827649352,
      "rewards/margins": 4.610652880181669,
      "rewards/rejected": -2.0288614974167336,
      "step": 1140
    },
    {
      "epoch": 1.5025906735751295,
      "grad_norm": 7.117109298706055,
      "kl": 30.72871208190918,
      "learning_rate": 5.347053841052518e-07,
      "logits/chosen": -49487765.69160306,
      "logits/rejected": -51846499.5328,
      "logps/chosen": -436.91049618320613,
      "logps/rejected": -390.759775,
      "loss": 0.2602,
      "loss/base_kto": 2.081653356552124,
      "metrics/advantage_var": 13168.955078125,
      "rewards/chosen": 2.580529691972805,
      "rewards/margins": 4.646819145097805,
      "rewards/rejected": -2.066289453125,
      "step": 1160
    },
    {
      "epoch": 1.528497409326425,
      "grad_norm": 7.7086100578308105,
      "kl": 28.140439987182617,
      "learning_rate": 5.205506596206531e-07,
      "logits/chosen": -50617514.1510574,
      "logits/rejected": -51723751.14563107,
      "logps/chosen": -461.4189010574018,
      "logps/rejected": -406.2813764158576,
      "loss": 0.2693,
      "loss/base_kto": 2.154078960418701,
      "metrics/advantage_var": 13133.0810546875,
      "rewards/chosen": 2.545702608690521,
      "rewards/margins": 4.44313510262256,
      "rewards/rejected": -1.8974324939320388,
      "step": 1180
    },
    {
      "epoch": 1.5544041450777202,
      "grad_norm": 8.802536010742188,
      "kl": 37.46778106689453,
      "learning_rate": 5.063794148754032e-07,
      "logits/chosen": -51498450.944,
      "logits/rejected": -51648984.13435114,
      "logps/chosen": -447.47215,
      "logps/rejected": -403.54608778625953,
      "loss": 0.2703,
      "loss/base_kto": 2.1621289253234863,
      "metrics/advantage_var": 13803.234375,
      "rewards/chosen": 2.386833984375,
      "rewards/margins": 4.364916679091126,
      "rewards/rejected": -1.9780826947161259,
      "step": 1200
    },
    {
      "epoch": 1.5803108808290154,
      "grad_norm": 7.323568344116211,
      "kl": 26.67828369140625,
      "learning_rate": 4.922030418472422e-07,
      "logits/chosen": -49500446.38699187,
      "logits/rejected": -52240106.827067666,
      "logps/chosen": -481.9415142276423,
      "logps/rejected": -424.76339285714283,
      "loss": 0.2387,
      "loss/base_kto": 1.9094852209091187,
      "metrics/advantage_var": 14641.3720703125,
      "rewards/chosen": 2.3854688690929877,
      "rewards/margins": 5.121942927774849,
      "rewards/rejected": -2.736474058681861,
      "step": 1220
    },
    {
      "epoch": 1.6062176165803108,
      "grad_norm": 8.504008293151855,
      "kl": 18.42952537536621,
      "learning_rate": 4.780329366364336e-07,
      "logits/chosen": -50177989.48571429,
      "logits/rejected": -50049803.23902439,
      "logps/chosen": -471.387969924812,
      "logps/rejected": -393.54270833333334,
      "loss": 0.2588,
      "loss/base_kto": 2.0701587200164795,
      "metrics/advantage_var": 13637.3095703125,
      "rewards/chosen": 2.5208492422462405,
      "rewards/margins": 4.819602735640549,
      "rewards/rejected": -2.298753493394309,
      "step": 1240
    },
    {
      "epoch": 1.6321243523316062,
      "grad_norm": 7.224009037017822,
      "kl": 29.492969512939453,
      "learning_rate": 4.638804903046684e-07,
      "logits/chosen": -52442332.92598425,
      "logits/rejected": -53556908.254263565,
      "logps/chosen": -471.96633858267717,
      "logps/rejected": -400.7843023255814,
      "loss": 0.2597,
      "loss/base_kto": 2.0774409770965576,
      "metrics/advantage_var": 12654.884765625,
      "rewards/chosen": 2.6672249861589568,
      "rewards/margins": 4.531793557501011,
      "rewards/rejected": -1.8645685713420543,
      "step": 1260
    },
    {
      "epoch": 1.6580310880829017,
      "grad_norm": 9.018906593322754,
      "kl": 24.7161922454834,
      "learning_rate": 4.497570797180217e-07,
      "logits/chosen": -52677000.036753446,
      "logits/rejected": -54384068.3891547,
      "logps/chosen": -497.3668166156202,
      "logps/rejected": -371.66113437001593,
      "loss": 0.2578,
      "loss/base_kto": 2.062281608581543,
      "metrics/advantage_var": 12963.482421875,
      "rewards/chosen": 2.509745248791635,
      "rewards/margins": 4.59229086843378,
      "rewards/rejected": -2.0825456196421452,
      "step": 1280
    },
    {
      "epoch": 1.6839378238341969,
      "grad_norm": 7.554812431335449,
      "kl": 22.770795822143555,
      "learning_rate": 4.3567405840131853e-07,
      "logits/chosen": -52157474.628019325,
      "logits/rejected": -54097755.289833084,
      "logps/chosen": -455.2374194847021,
      "logps/rejected": -387.06060318664646,
      "loss": 0.2616,
      "loss/base_kto": 2.0927064418792725,
      "metrics/advantage_var": 13465.212890625,
      "rewards/chosen": 2.3146233550976247,
      "rewards/margins": 4.661450860669514,
      "rewards/rejected": -2.346827505571889,
      "step": 1300
    },
    {
      "epoch": 1.709844559585492,
      "grad_norm": 8.2971830368042,
      "kl": 27.238021850585938,
      "learning_rate": 4.2164274741126506e-07,
      "logits/chosen": -52593679.950155765,
      "logits/rejected": -52541517.04075235,
      "logps/chosen": -462.36896417445485,
      "logps/rejected": -404.9313773510972,
      "loss": 0.2527,
      "loss/base_kto": 2.021425247192383,
      "metrics/advantage_var": 14735.775390625,
      "rewards/chosen": 2.5553978655568534,
      "rewards/margins": 5.0356779614375355,
      "rewards/rejected": -2.4802800958806817,
      "step": 1320
    },
    {
      "epoch": 1.7357512953367875,
      "grad_norm": 8.200288772583008,
      "kl": 31.184919357299805,
      "learning_rate": 4.0767442623567856e-07,
      "logits/chosen": -51745873.85318108,
      "logits/rejected": -53775051.41829085,
      "logps/chosen": -449.0927814029364,
      "logps/rejected": -386.02633058470764,
      "loss": 0.2547,
      "loss/base_kto": 2.0375049114227295,
      "metrics/advantage_var": 14179.8154296875,
      "rewards/chosen": 2.6527774680108074,
      "rewards/margins": 4.816153482640211,
      "rewards/rejected": -2.163376014629404,
      "step": 1340
    },
    {
      "epoch": 1.761658031088083,
      "grad_norm": 6.618141174316406,
      "kl": 16.11330223083496,
      "learning_rate": 3.937803237261357e-07,
      "logits/chosen": -50715854.67889908,
      "logits/rejected": -52869191.974440895,
      "logps/chosen": -455.9623470948012,
      "logps/rejected": -380.09070487220447,
      "loss": 0.2821,
      "loss/base_kto": 2.2565581798553467,
      "metrics/advantage_var": 12346.3212890625,
      "rewards/chosen": 2.182855728569381,
      "rewards/margins": 4.179773940680004,
      "rewards/rejected": -1.996918212110623,
      "step": 1360
    },
    {
      "epoch": 1.7875647668393784,
      "grad_norm": 7.924007892608643,
      "kl": 35.323726654052734,
      "learning_rate": 3.7997160907132456e-07,
      "logits/chosen": -53506547.67255217,
      "logits/rejected": -52517515.49467275,
      "logps/chosen": -477.71960272873196,
      "logps/rejected": -414.9009703196347,
      "loss": 0.252,
      "loss/base_kto": 2.0156280994415283,
      "metrics/advantage_var": 15051.2763671875,
      "rewards/chosen": 2.7909207902788924,
      "rewards/margins": 5.021091889379919,
      "rewards/rejected": -2.2301710991010273,
      "step": 1380
    },
    {
      "epoch": 1.8134715025906736,
      "grad_norm": 7.5072784423828125,
      "kl": 23.88299560546875,
      "learning_rate": 3.662593828183601e-07,
      "logits/chosen": -51244745.6,
      "logits/rejected": -52956176.0,
      "logps/chosen": -453.29130859375,
      "logps/rejected": -397.8761962890625,
      "loss": 0.2651,
      "loss/base_kto": 2.1209917068481445,
      "metrics/advantage_var": 12839.2919921875,
      "rewards/chosen": 2.2394113540649414,
      "rewards/margins": 4.529068565368652,
      "rewards/rejected": -2.289657211303711,
      "step": 1400
    },
    {
      "epoch": 1.8393782383419688,
      "grad_norm": 8.157623291015625,
      "kl": 18.68647003173828,
      "learning_rate": 3.5265466794927725e-07,
      "logits/chosen": -53356375.12443778,
      "logits/rejected": -53121465.004893966,
      "logps/chosen": -451.46158170914543,
      "logps/rejected": -380.848338091354,
      "loss": 0.2588,
      "loss/base_kto": 2.070202350616455,
      "metrics/advantage_var": 13694.220703125,
      "rewards/chosen": 2.3687469619682346,
      "rewards/margins": 4.725921510168684,
      "rewards/rejected": -2.3571745482004487,
      "step": 1420
    },
    {
      "epoch": 1.8652849740932642,
      "grad_norm": 5.826056957244873,
      "kl": 33.48683547973633,
      "learning_rate": 3.3916840101987887e-07,
      "logits/chosen": -52616473.64069952,
      "logits/rejected": -53620673.08141321,
      "logps/chosen": -474.6203795707472,
      "logps/rejected": -400.85143049155147,
      "loss": 0.2594,
      "loss/base_kto": 2.0748069286346436,
      "metrics/advantage_var": 12404.947265625,
      "rewards/chosen": 2.778156322970489,
      "rewards/margins": 4.539449045704744,
      "rewards/rejected": -1.761292722734255,
      "step": 1440
    },
    {
      "epoch": 1.8911917098445596,
      "grad_norm": 7.233984470367432,
      "kl": 21.10243034362793,
      "learning_rate": 3.258114233680583e-07,
      "logits/chosen": -54232816.42433697,
      "logits/rejected": -53901193.414710484,
      "logps/chosen": -460.75594773790954,
      "logps/rejected": -382.471904342723,
      "loss": 0.257,
      "loss/base_kto": 2.055638313293457,
      "metrics/advantage_var": 12931.291015625,
      "rewards/chosen": 2.364379311500585,
      "rewards/margins": 4.625557055045186,
      "rewards/rejected": -2.261177743544601,
      "step": 1460
    },
    {
      "epoch": 1.917098445595855,
      "grad_norm": 7.463532447814941,
      "kl": 27.48748779296875,
      "learning_rate": 3.1259447239866796e-07,
      "logits/chosen": -52548510.1656051,
      "logits/rejected": -54142184.44171779,
      "logps/chosen": -445.79443670382165,
      "logps/rejected": -411.83301380368096,
      "loss": 0.2539,
      "loss/base_kto": 2.031330108642578,
      "metrics/advantage_var": 13952.5751953125,
      "rewards/chosen": 2.413046478465864,
      "rewards/margins": 4.821361563660457,
      "rewards/rejected": -2.4083150851945936,
      "step": 1480
    },
    {
      "epoch": 1.9430051813471503,
      "grad_norm": 6.730622291564941,
      "kl": 26.386274337768555,
      "learning_rate": 2.9952817295193435e-07,
      "logits/chosen": -53502204.405616224,
      "logits/rejected": -55983384.43818466,
      "logps/chosen": -465.86388455538224,
      "logps/rejected": -407.58355340375584,
      "loss": 0.2547,
      "loss/base_kto": 2.0376741886138916,
      "metrics/advantage_var": 12988.326171875,
      "rewards/chosen": 2.668120338094774,
      "rewards/margins": 4.6582057070795155,
      "rewards/rejected": -1.9900853689847418,
      "step": 1500
    },
    {
      "epoch": 1.9689119170984455,
      "grad_norm": 6.946061134338379,
      "kl": 21.508319854736328,
      "learning_rate": 2.866230287623651e-07,
      "logits/chosen": -51112526.58604651,
      "logits/rejected": -52070383.87401575,
      "logps/chosen": -486.25213178294575,
      "logps/rejected": -384.9942421259843,
      "loss": 0.2729,
      "loss/base_kto": 2.1833534240722656,
      "metrics/advantage_var": 14524.7392578125,
      "rewards/chosen": 2.373459832243217,
      "rewards/margins": 4.678766334457784,
      "rewards/rejected": -2.305306502214567,
      "step": 1520
    },
    {
      "epoch": 1.994818652849741,
      "grad_norm": 6.844990253448486,
      "kl": 30.158191680908203,
      "learning_rate": 2.738894140150075e-07,
      "logits/chosen": -50484298.10526316,
      "logits/rejected": -51932629.333333336,
      "logps/chosen": -471.68832236842104,
      "logps/rejected": -386.06856863839283,
      "loss": 0.2574,
      "loss/base_kto": 2.0595574378967285,
      "metrics/advantage_var": 12867.0029296875,
      "rewards/chosen": 2.412080865157278,
      "rewards/margins": 4.429532378538509,
      "rewards/rejected": -2.017451513381231,
      "step": 1540
    },
    {
      "epoch": 2.0207253886010363,
      "grad_norm": 9.334595680236816,
      "kl": 16.50066566467285,
      "learning_rate": 2.6133756500585156e-07,
      "logits/chosen": -54804304.73534073,
      "logits/rejected": -54237997.50231839,
      "logps/chosen": -480.82334587955626,
      "logps/rejected": -396.0348000386399,
      "loss": 0.2379,
      "loss/base_kto": 1.9035505056381226,
      "metrics/advantage_var": 15041.2568359375,
      "rewards/chosen": 2.3609980902089935,
      "rewards/margins": 5.105607751989326,
      "rewards/rejected": -2.7446096617803324,
      "step": 1560
    },
    {
      "epoch": 2.0466321243523318,
      "grad_norm": 8.898676872253418,
      "kl": 15.84814739227295,
      "learning_rate": 2.489775719130767e-07,
      "logits/chosen": -53248006.4503937,
      "logits/rejected": -53730565.95348837,
      "logps/chosen": -462.44576771653544,
      "logps/rejected": -418.21555232558137,
      "loss": 0.205,
      "loss/base_kto": 1.6402209997177124,
      "metrics/advantage_var": 15167.4091796875,
      "rewards/chosen": 2.560914431594488,
      "rewards/margins": 5.7559937678347985,
      "rewards/rejected": -3.19507933624031,
      "step": 1580
    },
    {
      "epoch": 2.0725388601036268,
      "grad_norm": 6.704883098602295,
      "kl": 25.383604049682617,
      "learning_rate": 2.3681937068576303e-07,
      "logits/chosen": -53025779.587878786,
      "logits/rejected": -52647830.29677419,
      "logps/chosen": -457.3313446969697,
      "logps/rejected": -406.0405745967742,
      "loss": 0.2133,
      "loss/base_kto": 1.706233263015747,
      "metrics/advantage_var": 14251.7578125,
      "rewards/chosen": 2.7049793590198865,
      "rewards/margins": 5.467889869667568,
      "rewards/rejected": -2.7629105106476817,
      "step": 1600
    },
    {
      "epoch": 2.098445595854922,
      "grad_norm": 7.268764495849609,
      "kl": 25.373451232910156,
      "learning_rate": 2.2487273505658e-07,
      "logits/chosen": -54058547.035369776,
      "logits/rejected": -55866875.331306994,
      "logps/chosen": -457.23467644694534,
      "logps/rejected": -430.41479863221883,
      "loss": 0.2082,
      "loss/base_kto": 1.6655216217041016,
      "metrics/advantage_var": 14516.7158203125,
      "rewards/chosen": 2.768206531978497,
      "rewards/margins": 5.502744416073291,
      "rewards/rejected": -2.7345378840947947,
      "step": 1620
    },
    {
      "epoch": 2.1243523316062176,
      "grad_norm": 7.603555679321289,
      "kl": 11.966200828552246,
      "learning_rate": 2.131472686848817e-07,
      "logits/chosen": -53182169.139280125,
      "logits/rejected": -54440051.02028081,
      "logps/chosen": -436.7267214397496,
      "logps/rejected": -405.76048166926677,
      "loss": 0.2122,
      "loss/base_kto": 1.697871446609497,
      "metrics/advantage_var": 15479.9716796875,
      "rewards/chosen": 2.381063016554186,
      "rewards/margins": 5.622166547414171,
      "rewards/rejected": -3.2411035308599843,
      "step": 1640
    },
    {
      "epoch": 2.150259067357513,
      "grad_norm": 6.021089553833008,
      "kl": 17.05512046813965,
      "learning_rate": 2.016523974365188e-07,
      "logits/chosen": -55499307.18072289,
      "logits/rejected": -55416984.935064934,
      "logps/chosen": -454.51336596385545,
      "logps/rejected": -400.203226461039,
      "loss": 0.2146,
      "loss/base_kto": 1.7166544198989868,
      "metrics/advantage_var": 13250.1689453125,
      "rewards/chosen": 2.676328911838761,
      "rewards/margins": 5.372626099148522,
      "rewards/rejected": -2.6962971873097605,
      "step": 1660
    },
    {
      "epoch": 2.1761658031088085,
      "grad_norm": 6.856086730957031,
      "kl": 15.850852966308594,
      "learning_rate": 1.9039736180657372e-07,
      "logits/chosen": -53884351.502332814,
      "logits/rejected": -55031528.288854,
      "logps/chosen": -477.06454121306376,
      "logps/rejected": -386.7890502354788,
      "loss": 0.222,
      "loss/base_kto": 1.7756065130233765,
      "metrics/advantage_var": 14796.390625,
      "rewards/chosen": 2.727618417695373,
      "rewards/margins": 5.3221108044938426,
      "rewards/rejected": -2.5944923867984695,
      "step": 1680
    },
    {
      "epoch": 2.2020725388601035,
      "grad_norm": 7.76173210144043,
      "kl": 22.50533103942871,
      "learning_rate": 1.7939120949111498e-07,
      "logits/chosen": -54061751.54716981,
      "logits/rejected": -55115057.29192547,
      "logps/chosen": -459.26827830188677,
      "logps/rejected": -395.9180900621118,
      "loss": 0.215,
      "loss/base_kto": 1.7201648950576782,
      "metrics/advantage_var": 13973.0341796875,
      "rewards/chosen": 2.4738017148191824,
      "rewards/margins": 5.320175657887408,
      "rewards/rejected": -2.846373943068226,
      "step": 1700
    },
    {
      "epoch": 2.227979274611399,
      "grad_norm": 7.984103202819824,
      "kl": 22.196212768554688,
      "learning_rate": 1.6864278811393523e-07,
      "logits/chosen": -53185831.822222225,
      "logits/rejected": -52781361.624615386,
      "logps/chosen": -445.7280753968254,
      "logps/rejected": -405.18557692307695,
      "loss": 0.2246,
      "loss/base_kto": 1.7964165210723877,
      "metrics/advantage_var": 14180.470703125,
      "rewards/chosen": 2.587810794890873,
      "rewards/margins": 5.1655412260807765,
      "rewards/rejected": -2.5777304311899036,
      "step": 1720
    },
    {
      "epoch": 2.2538860103626943,
      "grad_norm": 7.907989978790283,
      "kl": 15.732986450195312,
      "learning_rate": 1.5816073811412584e-07,
      "logits/chosen": -55130164.51282051,
      "logits/rejected": -55750868.29268292,
      "logps/chosen": -454.47455929487177,
      "logps/rejected": -396.7212271341463,
      "loss": 0.2038,
      "loss/base_kto": 1.6303322315216064,
      "metrics/advantage_var": 17776.615234375,
      "rewards/chosen": 2.779758942432893,
      "rewards/margins": 5.993346261411551,
      "rewards/rejected": -3.2135873189786586,
      "step": 1740
    },
    {
      "epoch": 2.2797927461139897,
      "grad_norm": 5.375637054443359,
      "kl": 16.633441925048828,
      "learning_rate": 1.4795348580020207e-07,
      "logits/chosen": -53817220.25377644,
      "logits/rejected": -54321791.58576052,
      "logps/chosen": -465.3703266616314,
      "logps/rejected": -395.2126314724919,
      "loss": 0.2098,
      "loss/base_kto": 1.6782522201538086,
      "metrics/advantage_var": 14872.291015625,
      "rewards/chosen": 2.733623216519543,
      "rewards/margins": 5.759970849066569,
      "rewards/rejected": -3.0263476325470267,
      "step": 1760
    },
    {
      "epoch": 2.305699481865285,
      "grad_norm": 7.098501682281494,
      "kl": 21.81011390686035,
      "learning_rate": 1.3802923657636355e-07,
      "logits/chosen": -53584120.7318612,
      "logits/rejected": -53781951.00928792,
      "logps/chosen": -464.52646884858046,
      "logps/rejected": -413.0716911764706,
      "loss": 0.2132,
      "loss/base_kto": 1.7055721282958984,
      "metrics/advantage_var": 14210.4912109375,
      "rewards/chosen": 2.736393588574527,
      "rewards/margins": 5.367398843266864,
      "rewards/rejected": -2.6310052546923375,
      "step": 1780
    },
    {
      "epoch": 2.33160621761658,
      "grad_norm": 6.580950736999512,
      "kl": 17.816463470458984,
      "learning_rate": 1.28395968346336e-07,
      "logits/chosen": -53349533.055214725,
      "logits/rejected": -54086770.14012739,
      "logps/chosen": -444.88899539877303,
      "logps/rejected": -396.07538813694265,
      "loss": 0.2174,
      "loss/base_kto": 1.7395038604736328,
      "metrics/advantage_var": 14362.5595703125,
      "rewards/chosen": 2.6289107433857364,
      "rewards/margins": 5.474019455940473,
      "rewards/rejected": -2.845108712554737,
      "step": 1800
    },
    {
      "epoch": 2.3575129533678756,
      "grad_norm": 4.578265190124512,
      "kl": 14.631937980651855,
      "learning_rate": 1.1906142510009415e-07,
      "logits/chosen": -54949289.25937031,
      "logits/rejected": -55131558.62969005,
      "logps/chosen": -502.6547976011994,
      "logps/rejected": -386.00341557911906,
      "loss": 0.1927,
      "loss/base_kto": 1.54173743724823,
      "metrics/advantage_var": 15154.0849609375,
      "rewards/chosen": 2.852893194516023,
      "rewards/margins": 5.928785482699139,
      "rewards/rejected": -3.0758922881831157,
      "step": 1820
    },
    {
      "epoch": 2.383419689119171,
      "grad_norm": 7.377775192260742,
      "kl": 21.765613555908203,
      "learning_rate": 1.1003311068862547e-07,
      "logits/chosen": -52769526.046656296,
      "logits/rejected": -54357302.25431711,
      "logps/chosen": -469.86314152410574,
      "logps/rejected": -392.0960312009419,
      "loss": 0.2113,
      "loss/base_kto": 1.6907843351364136,
      "metrics/advantage_var": 14896.6064453125,
      "rewards/chosen": 2.711991329400758,
      "rewards/margins": 5.5244031781546825,
      "rewards/rejected": -2.8124118487539245,
      "step": 1840
    },
    {
      "epoch": 2.4093264248704664,
      "grad_norm": 8.828693389892578,
      "kl": 18.875425338745117,
      "learning_rate": 1.0131828279173588e-07,
      "logits/chosen": -53972524.9122807,
      "logits/rejected": -54421036.692189895,
      "logps/chosen": -430.7370414673046,
      "logps/rejected": -411.7427258805513,
      "loss": 0.2136,
      "loss/base_kto": 1.7088441848754883,
      "metrics/advantage_var": 14549.0263671875,
      "rewards/chosen": 2.5342544482655502,
      "rewards/margins": 5.400527449079103,
      "rewards/rejected": -2.866273000813553,
      "step": 1860
    },
    {
      "epoch": 2.4352331606217614,
      "grad_norm": 7.736706256866455,
      "kl": 14.055915832519531,
      "learning_rate": 9.292394708374596e-08,
      "logits/chosen": -52026667.323076926,
      "logits/rejected": -54300707.75873016,
      "logps/chosen": -454.6047115384615,
      "logps/rejected": -396.18685515873017,
      "loss": 0.2123,
      "loss/base_kto": 1.6984790563583374,
      "metrics/advantage_var": 14236.0537109375,
      "rewards/chosen": 2.6927809495192307,
      "rewards/margins": 5.523264425461691,
      "rewards/rejected": -2.83048347594246,
      "step": 1880
    },
    {
      "epoch": 2.461139896373057,
      "grad_norm": 5.858903408050537,
      "kl": 15.589258193969727,
      "learning_rate": 8.48568516017727e-08,
      "logits/chosen": -53257451.78947368,
      "logits/rejected": -55407152.76190476,
      "logps/chosen": -455.32797080592104,
      "logps/rejected": -401.0511532738095,
      "loss": 0.2187,
      "loss/base_kto": 1.749979853630066,
      "metrics/advantage_var": 14435.7890625,
      "rewards/chosen": 2.5456470690275492,
      "rewards/margins": 5.29852553059284,
      "rewards/rejected": -2.7528784615652904,
      "step": 1900
    },
    {
      "epoch": 2.4870466321243523,
      "grad_norm": 6.895484924316406,
      "kl": 13.590128898620605,
      "learning_rate": 7.71234813211169e-08,
      "logits/chosen": -53473166.57230769,
      "logits/rejected": -55175941.68888889,
      "logps/chosen": -457.99591346153846,
      "logps/rejected": -389.46001984126985,
      "loss": 0.2023,
      "loss/base_kto": 1.6187559366226196,
      "metrics/advantage_var": 14734.931640625,
      "rewards/chosen": 2.576056941105769,
      "rewards/margins": 5.7244058916886065,
      "rewards/rejected": -3.1483489505828373,
      "step": 1920
    },
    {
      "epoch": 2.5129533678756477,
      "grad_norm": 8.741658210754395,
      "kl": 15.451461791992188,
      "learning_rate": 6.973005294212353e-08,
      "logits/chosen": -53555690.22992126,
      "logits/rejected": -54169911.16899225,
      "logps/chosen": -488.6531496062992,
      "logps/rejected": -409.95174418604654,
      "loss": 0.2197,
      "loss/base_kto": 1.7576106786727905,
      "metrics/advantage_var": 14217.2880859375,
      "rewards/chosen": 2.5698707400344487,
      "rewards/margins": 5.545861307500534,
      "rewards/rejected": -2.975990567466085,
      "step": 1940
    },
    {
      "epoch": 2.538860103626943,
      "grad_norm": 9.848816871643066,
      "kl": 16.752267837524414,
      "learning_rate": 6.268250989270102e-08,
      "logits/chosen": -54066898.5344,
      "logits/rejected": -55513400.67175572,
      "logps/chosen": -439.4038,
      "logps/rejected": -405.7951812977099,
      "loss": 0.212,
      "loss/base_kto": 1.6957625150680542,
      "metrics/advantage_var": 15750.0146484375,
      "rewards/chosen": 2.6572337890625,
      "rewards/margins": 5.661433753280058,
      "rewards/rejected": -3.004199964217557,
      "step": 1960
    },
    {
      "epoch": 2.5647668393782386,
      "grad_norm": 6.366123199462891,
      "kl": 13.643994331359863,
      "learning_rate": 5.598651755051975e-08,
      "logits/chosen": -55293618.01846154,
      "logits/rejected": -55183756.5968254,
      "logps/chosen": -461.94384615384615,
      "logps/rejected": -422.662251984127,
      "loss": 0.1961,
      "loss/base_kto": 1.5690070390701294,
      "metrics/advantage_var": 15086.9375,
      "rewards/chosen": 2.77778076171875,
      "rewards/margins": 5.87708011687748,
      "rewards/rejected": -3.0992993551587302,
      "step": 1980
    },
    {
      "epoch": 2.5906735751295336,
      "grad_norm": 6.5458807945251465,
      "kl": 23.304786682128906,
      "learning_rate": 4.964745868872933e-08,
      "logits/chosen": -54190597.03606557,
      "logits/rejected": -54947399.83283582,
      "logps/chosen": -454.877356557377,
      "logps/rejected": -427.23903917910445,
      "loss": 0.2231,
      "loss/base_kto": 1.7850319147109985,
      "metrics/advantage_var": 15889.9716796875,
      "rewards/chosen": 2.544925477074795,
      "rewards/margins": 5.412163627144757,
      "rewards/rejected": -2.8672381500699626,
      "step": 2000
    },
    {
      "epoch": 2.616580310880829,
      "grad_norm": 6.88004732131958,
      "kl": 19.396982192993164,
      "learning_rate": 4.3670429148855493e-08,
      "logits/chosen": -54500910.39745628,
      "logits/rejected": -53831865.60983103,
      "logps/chosen": -486.3404709856916,
      "logps/rejected": -385.31322004608296,
      "loss": 0.2178,
      "loss/base_kto": 1.7427517175674438,
      "metrics/advantage_var": 14418.283203125,
      "rewards/chosen": 2.622706863573132,
      "rewards/margins": 5.490551405623823,
      "rewards/rejected": -2.8678445420506913,
      "step": 2020
    },
    {
      "epoch": 2.6424870466321244,
      "grad_norm": 8.664511680603027,
      "kl": 23.54598045349121,
      "learning_rate": 3.806023374435663e-08,
      "logits/chosen": -54896742.4,
      "logits/rejected": -54253043.2,
      "logps/chosen": -476.05732421875,
      "logps/rejected": -433.92314453125,
      "loss": 0.2018,
      "loss/base_kto": 1.6147011518478394,
      "metrics/advantage_var": 15689.046875,
      "rewards/chosen": 2.820555877685547,
      "rewards/margins": 5.827705192565919,
      "rewards/rejected": -3.007149314880371,
      "step": 2040
    },
    {
      "epoch": 2.66839378238342,
      "grad_norm": 7.058914661407471,
      "kl": 18.342493057250977,
      "learning_rate": 3.282138239813037e-08,
      "logits/chosen": -54555392.0,
      "logits/rejected": -55395878.4,
      "logps/chosen": -477.841845703125,
      "logps/rejected": -403.7883544921875,
      "loss": 0.2113,
      "loss/base_kto": 1.6905354261398315,
      "metrics/advantage_var": 14639.556640625,
      "rewards/chosen": 2.585844802856445,
      "rewards/margins": 5.570068550109863,
      "rewards/rejected": -2.984223747253418,
      "step": 2060
    },
    {
      "epoch": 2.694300518134715,
      "grad_norm": 7.7522735595703125,
      "kl": 13.34247875213623,
      "learning_rate": 2.795808651707793e-08,
      "logits/chosen": -53318083.101321585,
      "logits/rejected": -54890116.487479135,
      "logps/chosen": -482.61926395007345,
      "logps/rejected": -379.9926179048414,
      "loss": 0.2016,
      "loss/base_kto": 1.6129086017608643,
      "metrics/advantage_var": 15127.2607421875,
      "rewards/chosen": 2.6659618495319384,
      "rewards/margins": 5.789327248569396,
      "rewards/rejected": -3.1233653990374584,
      "step": 2080
    },
    {
      "epoch": 2.7202072538860103,
      "grad_norm": 7.7964935302734375,
      "kl": 14.25732421875,
      "learning_rate": 2.3474255606639293e-08,
      "logits/chosen": -51992039.77287066,
      "logits/rejected": -54597689.06501548,
      "logps/chosen": -454.4478509463722,
      "logps/rejected": -378.53347523219816,
      "loss": 0.2157,
      "loss/base_kto": 1.7255557775497437,
      "metrics/advantage_var": 13727.2841796875,
      "rewards/chosen": 2.5626330450882295,
      "rewards/margins": 5.474771520441364,
      "rewards/rejected": -2.9121384753531347,
      "step": 2100
    },
    {
      "epoch": 2.7461139896373057,
      "grad_norm": 7.031707763671875,
      "kl": 17.868247985839844,
      "learning_rate": 1.9373494128020195e-08,
      "logits/chosen": -53826867.03924647,
      "logits/rejected": -56076074.60031104,
      "logps/chosen": -471.02634419152275,
      "logps/rejected": -403.050933125972,
      "loss": 0.2042,
      "loss/base_kto": 1.6338428258895874,
      "metrics/advantage_var": 14567.953125,
      "rewards/chosen": 2.7185585584895016,
      "rewards/margins": 5.788678176005053,
      "rewards/rejected": -3.070119617515552,
      "step": 2120
    },
    {
      "epoch": 2.772020725388601,
      "grad_norm": 8.422375679016113,
      "kl": 18.8111572265625,
      "learning_rate": 1.5659098600638798e-08,
      "logits/chosen": -55571169.28,
      "logits/rejected": -56779636.21587302,
      "logps/chosen": -465.2566346153846,
      "logps/rejected": -405.76974206349206,
      "loss": 0.2138,
      "loss/base_kto": 1.7100785970687866,
      "metrics/advantage_var": 14109.947265625,
      "rewards/chosen": 2.5388807091346153,
      "rewards/margins": 5.32893333500267,
      "rewards/rejected": -2.7900526258680554,
      "step": 2140
    },
    {
      "epoch": 2.7979274611398965,
      "grad_norm": 8.012675285339355,
      "kl": 15.15075969696045,
      "learning_rate": 1.2334054952120143e-08,
      "logits/chosen": -54361836.55696203,
      "logits/rejected": -53801984.0,
      "logps/chosen": -461.8822191455696,
      "logps/rejected": -381.108699845679,
      "loss": 0.205,
      "loss/base_kto": 1.6402161121368408,
      "metrics/advantage_var": 14670.7783203125,
      "rewards/chosen": 2.6562598506106605,
      "rewards/margins": 5.770099459262281,
      "rewards/rejected": -3.1138396086516202,
      "step": 2160
    },
    {
      "epoch": 2.823834196891192,
      "grad_norm": 8.247160911560059,
      "kl": 25.481403350830078,
      "learning_rate": 9.401036117969108e-09,
      "logits/chosen": -55167100.71794872,
      "logits/rejected": -54453379.12195122,
      "logps/chosen": -440.9677483974359,
      "logps/rejected": -428.6783060213415,
      "loss": 0.2258,
      "loss/base_kto": 1.8066402673721313,
      "metrics/advantage_var": 15019.609375,
      "rewards/chosen": 2.4441407032502003,
      "rewards/margins": 5.257231702202182,
      "rewards/rejected": -2.813090998951982,
      "step": 2180
    },
    {
      "epoch": 2.849740932642487,
      "grad_norm": 5.7789692878723145,
      "kl": 25.09571647644043,
      "learning_rate": 6.8623998928517555e-09,
      "logits/chosen": -52957783.33646322,
      "logits/rejected": -53720056.0124805,
      "logps/chosen": -444.24002347417843,
      "logps/rejected": -400.2240639625585,
      "loss": 0.2127,
      "loss/base_kto": 1.7019115686416626,
      "metrics/advantage_var": 13801.8740234375,
      "rewards/chosen": 2.5932023073772497,
      "rewards/margins": 5.430687681792226,
      "rewards/rejected": -2.8374853744149764,
      "step": 2200
    },
    {
      "epoch": 2.8756476683937824,
      "grad_norm": 7.976244926452637,
      "kl": 13.27165412902832,
      "learning_rate": 4.72018703521132e-09,
      "logits/chosen": -56062677.06832298,
      "logits/rejected": -55704505.1572327,
      "logps/chosen": -476.4144992236025,
      "logps/rejected": -395.6385613207547,
      "loss": 0.2142,
      "loss/base_kto": 1.713976502418518,
      "metrics/advantage_var": 15611.8466796875,
      "rewards/chosen": 2.5529902677358307,
      "rewards/margins": 5.451443698295481,
      "rewards/rejected": -2.89845343055965,
      "step": 2220
    },
    {
      "epoch": 2.901554404145078,
      "grad_norm": 5.943566799163818,
      "kl": 22.686498641967773,
      "learning_rate": 2.976119626744267e-09,
      "logits/chosen": -54723577.559748426,
      "logits/rejected": -55772865.98757764,
      "logps/chosen": -451.77982507861634,
      "logps/rejected": -397.55330454192546,
      "loss": 0.2198,
      "loss/base_kto": 1.7582666873931885,
      "metrics/advantage_var": 14203.173828125,
      "rewards/chosen": 2.444792741499607,
      "rewards/margins": 5.329901452315795,
      "rewards/rejected": -2.885108710816188,
      "step": 2240
    },
    {
      "epoch": 2.927461139896373,
      "grad_norm": 9.51264762878418,
      "kl": 15.295659065246582,
      "learning_rate": 1.631599688052765e-09,
      "logits/chosen": -55017559.414634146,
      "logits/rejected": -54868306.05128205,
      "logps/chosen": -448.9042492378049,
      "logps/rejected": -414.9752604166667,
      "loss": 0.2162,
      "loss/base_kto": 1.7298539876937866,
      "metrics/advantage_var": 14221.525390625,
      "rewards/chosen": 2.6078794525890814,
      "rewards/margins": 5.469379923655287,
      "rewards/rejected": -2.861500471066206,
      "step": 2260
    },
    {
      "epoch": 2.9533678756476682,
      "grad_norm": 8.533525466918945,
      "kl": 15.003725051879883,
      "learning_rate": 6.877080515894085e-10,
      "logits/chosen": -54759786.36334913,
      "logits/rejected": -55902100.377125196,
      "logps/chosen": -458.4798578199052,
      "logps/rejected": -388.92358964451313,
      "loss": 0.2237,
      "loss/base_kto": 1.7892799377441406,
      "metrics/advantage_var": 14245.1484375,
      "rewards/chosen": 2.5022184784755135,
      "rewards/margins": 5.4406616461605015,
      "rewards/rejected": -2.9384431676849885,
      "step": 2280
    },
    {
      "epoch": 2.9792746113989637,
      "grad_norm": 8.932574272155762,
      "kl": 16.011707305908203,
      "learning_rate": 1.4520349279739663e-10,
      "logits/chosen": -52867014.30985916,
      "logits/rejected": -54209454.527301095,
      "logps/chosen": -454.9545187793427,
      "logps/rejected": -377.59572445397816,
      "loss": 0.207,
      "loss/base_kto": 1.6560556888580322,
      "metrics/advantage_var": 14029.0283203125,
      "rewards/chosen": 2.5530711668011543,
      "rewards/margins": 5.502070533025803,
      "rewards/rejected": -2.948999366224649,
      "step": 2300
    },
    {
      "epoch": 3.0,
      "step": 2316,
      "total_flos": 9.972016217523487e+17,
      "train_loss": 0.27589865288792087,
      "train_runtime": 11043.1422,
      "train_samples_per_second": 13.422,
      "train_steps_per_second": 0.21
    }
  ],
  "logging_steps": 20,
  "max_steps": 2316,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 9.972016217523487e+17,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}