{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8599068434252956,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "drift/chosen_abs": 0.24268421530723572,
      "drift/rejected_abs": 0.09689199179410934,
      "epoch": 0.004777260241251642,
      "grad_norm": 158.0,
      "learning_rate": 1.111111111111111e-08,
      "logits/chosen": -1.9987547397613525,
      "logits/rejected": -2.0148520469665527,
      "logps/chosen": -0.27876606583595276,
      "logps/rejected": -0.2827395498752594,
      "loss": 23.720317840576172,
      "loss/core": 23.703845977783203,
      "loss/preference_sft": 0.27876606583595276,
      "loss/reference_anchor": 0.19172723591327667,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4264075756073,
      "rewards/margins": 1.473706603050232,
      "rewards/rejected": 0.9527010917663574,
      "step": 5
    },
    {
      "drift/chosen_abs": 0.1691116988658905,
      "drift/rejected_abs": 0.1621948778629303,
      "epoch": 0.009554520482503284,
      "grad_norm": 1336.0,
      "learning_rate": 2.5e-08,
      "logits/chosen": -2.345815658569336,
      "logits/rejected": -2.392836093902588,
      "logps/chosen": -0.28935980796813965,
      "logps/rejected": -0.2881030738353729,
      "loss": 25.067256927490234,
      "loss/core": 25.026386260986328,
      "loss/preference_sft": 0.28935980796813965,
      "loss/reference_anchor": 0.5160242915153503,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6900745630264282,
      "rewards/margins": 0.07324112951755524,
      "rewards/rejected": 1.6168334484100342,
      "step": 10
    },
    {
      "drift/chosen_abs": 0.21343913674354553,
      "drift/rejected_abs": 0.11456535756587982,
      "epoch": 0.014331780723754926,
      "grad_norm": 118.0,
      "learning_rate": 3.888888888888889e-08,
      "logits/chosen": -2.2311110496520996,
      "logits/rejected": -2.1946258544921875,
      "logps/chosen": -0.277534544467926,
      "logps/rejected": -0.28368276357650757,
      "loss": 24.18115997314453,
      "loss/core": 24.16556167602539,
      "loss/preference_sft": 0.277534544467926,
      "loss/reference_anchor": 0.1802430897951126,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1329588890075684,
      "rewards/margins": 0.9891046285629272,
      "rewards/rejected": 1.1438543796539307,
      "step": 15
    },
    {
      "drift/chosen_abs": 0.19140727818012238,
      "drift/rejected_abs": 0.08797380328178406,
      "epoch": 0.01910904096500657,
      "grad_norm": 104.5,
      "learning_rate": 5.2777777777777776e-08,
      "logits/chosen": -2.1776981353759766,
      "logits/rejected": -2.2193965911865234,
      "logps/chosen": -0.30152779817581177,
      "logps/rejected": -0.2865992486476898,
      "loss": 24.163585662841797,
      "loss/core": 24.14507484436035,
      "loss/preference_sft": 0.30152779817581177,
      "loss/reference_anchor": 0.21664480865001678,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9140727519989014,
      "rewards/margins": 1.0348074436187744,
      "rewards/rejected": 0.8792654275894165,
      "step": 20
    },
    {
      "drift/chosen_abs": 0.3856329321861267,
      "drift/rejected_abs": 0.16464610397815704,
      "epoch": 0.02388630120625821,
      "grad_norm": 86.5,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -2.0732712745666504,
      "logits/rejected": -2.13415789604187,
      "logps/chosen": -0.2700391411781311,
      "logps/rejected": -0.2578909993171692,
      "loss": 24.2296142578125,
      "loss/core": 24.161888122558594,
      "loss/preference_sft": 0.2700391411781311,
      "loss/reference_anchor": 0.8760007619857788,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.8547744750976562,
      "rewards/margins": 2.2091641426086426,
      "rewards/rejected": 1.6456102132797241,
      "step": 25
    },
    {
      "drift/chosen_abs": 0.22750432789325714,
      "drift/rejected_abs": 0.11023910343647003,
      "epoch": 0.028663561447509853,
      "grad_norm": 77.5,
      "learning_rate": 8.055555555555555e-08,
      "logits/chosen": -2.3493716716766357,
      "logits/rejected": -2.3317930698394775,
      "logps/chosen": -0.3004586100578308,
      "logps/rejected": -0.3256753087043762,
      "loss": 24.168228149414062,
      "loss/core": 24.141149520874023,
      "loss/preference_sft": 0.3004586100578308,
      "loss/reference_anchor": 0.33102959394454956,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.2747082710266113,
      "rewards/margins": 1.2620917558670044,
      "rewards/rejected": 1.0126166343688965,
      "step": 30
    },
    {
      "drift/chosen_abs": 0.26583537459373474,
      "drift/rejected_abs": 0.11596353352069855,
      "epoch": 0.033440821688761495,
      "grad_norm": 50.25,
      "learning_rate": 9.444444444444444e-08,
      "logits/chosen": -2.203460693359375,
      "logits/rejected": -2.18483829498291,
      "logps/chosen": -0.2677518129348755,
      "logps/rejected": -0.26530081033706665,
      "loss": 23.941822052001953,
      "loss/core": 23.910053253173828,
      "loss/preference_sft": 0.2677518129348755,
      "loss/reference_anchor": 0.3968318998813629,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.6582398414611816,
      "rewards/margins": 1.4989526271820068,
      "rewards/rejected": 1.1592872142791748,
      "step": 35
    },
    {
      "drift/chosen_abs": 0.44815701246261597,
      "drift/rejected_abs": 0.16203448176383972,
      "epoch": 0.03821808193001314,
      "grad_norm": 151.0,
      "learning_rate": 1.0833333333333334e-07,
      "logits/chosen": -2.002789258956909,
      "logits/rejected": -2.062656879425049,
      "logps/chosen": -0.2923213839530945,
      "logps/rejected": -0.2835318148136139,
      "loss": 23.72269058227539,
      "loss/core": 23.646509170532227,
      "loss/preference_sft": 0.2923213839530945,
      "loss/reference_anchor": 0.9865428805351257,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.481132984161377,
      "rewards/margins": 2.8950064182281494,
      "rewards/rejected": 1.5861257314682007,
      "step": 40
    },
    {
      "drift/chosen_abs": 0.20982900261878967,
      "drift/rejected_abs": 0.08536230772733688,
      "epoch": 0.04299534217126478,
      "grad_norm": 65.5,
      "learning_rate": 1.2222222222222222e-07,
      "logits/chosen": -2.199631452560425,
      "logits/rejected": -2.363292694091797,
      "logps/chosen": -0.2886757254600525,
      "logps/rejected": -0.2883760929107666,
      "loss": 23.874040603637695,
      "loss/core": 23.861454010009766,
      "loss/preference_sft": 0.2886757254600525,
      "loss/reference_anchor": 0.13895340263843536,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.0968005657196045,
      "rewards/margins": 1.254421353340149,
      "rewards/rejected": 0.8423792123794556,
      "step": 45
    },
    {
      "drift/chosen_abs": 0.2884393632411957,
      "drift/rejected_abs": 0.1451389044523239,
      "epoch": 0.04777260241251642,
      "grad_norm": 402.0,
      "learning_rate": 1.3611111111111108e-07,
      "logits/chosen": -2.166757106781006,
      "logits/rejected": -2.1628246307373047,
      "logps/chosen": -0.28129029273986816,
      "logps/rejected": -0.2850467562675476,
      "loss": 24.05858612060547,
      "loss/core": 24.029706954956055,
      "loss/preference_sft": 0.28129029273986816,
      "loss/reference_anchor": 0.3569650650024414,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8835978507995605,
      "rewards/margins": 1.4343656301498413,
      "rewards/rejected": 1.4492321014404297,
      "step": 50
    },
    {
      "drift/chosen_abs": 0.22632427513599396,
      "drift/rejected_abs": 0.20729069411754608,
      "epoch": 0.05254986265376806,
      "grad_norm": 32.25,
      "learning_rate": 1.5e-07,
      "logits/chosen": -2.1963820457458496,
      "logits/rejected": -2.1848561763763428,
      "logps/chosen": -0.28860360383987427,
      "logps/rejected": -0.28664013743400574,
      "loss": 25.665725708007812,
      "loss/core": 25.60601806640625,
      "loss/preference_sft": 0.28860360383987427,
      "loss/reference_anchor": 0.7672308087348938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.262362003326416,
      "rewards/margins": 0.20232732594013214,
      "rewards/rejected": 2.0600345134735107,
      "step": 55
    },
    {
      "drift/chosen_abs": 0.21578149497509003,
      "drift/rejected_abs": 0.1569838523864746,
      "epoch": 0.057327122895019705,
      "grad_norm": 1848.0,
      "learning_rate": 1.6388888888888888e-07,
      "logits/chosen": -2.0918967723846436,
      "logits/rejected": -2.1219000816345215,
      "logps/chosen": -0.27454960346221924,
      "logps/rejected": -0.2835273742675781,
      "loss": 24.56880760192871,
      "loss/core": 24.553863525390625,
      "loss/preference_sft": 0.27454960346221924,
      "loss/reference_anchor": 0.1717853546142578,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.1553452014923096,
      "rewards/margins": 0.5968301892280579,
      "rewards/rejected": 1.5585150718688965,
      "step": 60
    },
    {
      "drift/chosen_abs": 0.29738813638687134,
      "drift/rejected_abs": 0.11811880022287369,
      "epoch": 0.06210438313627135,
      "grad_norm": 66.0,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -1.962087869644165,
      "logits/rejected": -2.0643563270568848,
      "logps/chosen": -0.30899226665496826,
      "logps/rejected": -0.3064674437046051,
      "loss": 23.558834075927734,
      "loss/core": 23.519794464111328,
      "loss/preference_sft": 0.30899226665496826,
      "loss/reference_anchor": 0.48962312936782837,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9719326496124268,
      "rewards/margins": 1.8016983270645142,
      "rewards/rejected": 1.170233964920044,
      "step": 65
    },
    {
      "drift/chosen_abs": 0.3609868586063385,
      "drift/rejected_abs": 0.228200763463974,
      "epoch": 0.06688164337752299,
      "grad_norm": 229.0,
      "learning_rate": 1.9166666666666668e-07,
      "logits/chosen": -1.9723434448242188,
      "logits/rejected": -1.9571367502212524,
      "logps/chosen": -0.3828391134738922,
      "logps/rejected": -0.297024130821228,
      "loss": 24.988039016723633,
      "loss/core": 24.914230346679688,
      "loss/preference_sft": 0.3828391134738922,
      "loss/reference_anchor": 0.9458276629447937,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.6087005138397217,
      "rewards/margins": 1.4257218837738037,
      "rewards/rejected": 2.182979106903076,
      "step": 70
    },
    {
      "drift/chosen_abs": 0.2607436776161194,
      "drift/rejected_abs": 0.1599571406841278,
      "epoch": 0.07165890361877464,
      "grad_norm": 56.5,
      "learning_rate": 2.0555555555555553e-07,
      "logits/chosen": -2.0862269401550293,
      "logits/rejected": -2.050337553024292,
      "logps/chosen": -0.31514787673950195,
      "logps/rejected": -0.28866785764694214,
      "loss": 24.35919189453125,
      "loss/core": 24.329198837280273,
      "loss/preference_sft": 0.31514787673950195,
      "loss/reference_anchor": 0.36842408776283264,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6053974628448486,
      "rewards/margins": 1.00749671459198,
      "rewards/rejected": 1.5979008674621582,
      "step": 75
    },
    {
      "drift/chosen_abs": 0.3240804970264435,
      "drift/rejected_abs": 0.12827441096305847,
      "epoch": 0.07643616386002627,
      "grad_norm": 55.5,
      "learning_rate": 2.1944444444444442e-07,
      "logits/chosen": -2.0698435306549072,
      "logits/rejected": -2.1897106170654297,
      "logps/chosen": -0.26063475012779236,
      "logps/rejected": -0.2740657925605774,
      "loss": 23.68960189819336,
      "loss/core": 23.653255462646484,
      "loss/preference_sft": 0.26063475012779236,
      "loss/reference_anchor": 0.45853161811828613,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.2406482696533203,
      "rewards/margins": 1.9891496896743774,
      "rewards/rejected": 1.2514985799789429,
      "step": 80
    },
    {
      "drift/chosen_abs": 0.3498837351799011,
      "drift/rejected_abs": 0.13262492418289185,
      "epoch": 0.08121342410127792,
      "grad_norm": 79.0,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -2.189903497695923,
      "logits/rejected": -2.2383172512054443,
      "logps/chosen": -0.3075157105922699,
      "logps/rejected": -0.2782510817050934,
      "loss": 23.684247970581055,
      "loss/core": 23.61785125732422,
      "loss/preference_sft": 0.3075157105922699,
      "loss/reference_anchor": 0.8545700311660767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.498150587081909,
      "rewards/margins": 2.172797918319702,
      "rewards/rejected": 1.3253527879714966,
      "step": 85
    },
    {
      "drift/chosen_abs": 0.17537356913089752,
      "drift/rejected_abs": 0.08442483097314835,
      "epoch": 0.08599068434252956,
      "grad_norm": 31.875,
      "learning_rate": 2.4722222222222224e-07,
      "logits/chosen": -2.4203712940216064,
      "logits/rejected": -2.4085536003112793,
      "logps/chosen": -0.28136521577835083,
      "logps/rejected": -0.296181857585907,
      "loss": 24.193899154663086,
      "loss/core": 24.180316925048828,
      "loss/preference_sft": 0.28136521577835083,
      "loss/reference_anchor": 0.15298345685005188,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7537355422973633,
      "rewards/margins": 0.9103192090988159,
      "rewards/rejected": 0.8434165716171265,
      "step": 90
    },
    {
      "drift/chosen_abs": 0.3204111158847809,
      "drift/rejected_abs": 0.21080467104911804,
      "epoch": 0.09076794458378121,
      "grad_norm": 118.0,
      "learning_rate": 2.6111111111111113e-07,
      "logits/chosen": -2.0531086921691895,
      "logits/rejected": -2.108947992324829,
      "logps/chosen": -0.267969012260437,
      "logps/rejected": -0.27856558561325073,
      "loss": 24.32411766052246,
      "loss/core": 24.288156509399414,
      "loss/preference_sft": 0.267969012260437,
      "loss/reference_anchor": 0.4526856541633606,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2040367126464844,
      "rewards/margins": 1.0959895849227905,
      "rewards/rejected": 2.1080470085144043,
      "step": 95
    },
    {
      "drift/chosen_abs": 0.2465881109237671,
      "drift/rejected_abs": 0.1286160796880722,
      "epoch": 0.09554520482503284,
      "grad_norm": 73.5,
      "learning_rate": 2.75e-07,
      "logits/chosen": -2.2330098152160645,
      "logits/rejected": -2.2795863151550293,
      "logps/chosen": -0.2569820284843445,
      "logps/rejected": -0.2466917484998703,
      "loss": 23.988306045532227,
      "loss/core": 23.972742080688477,
      "loss/preference_sft": 0.2569820284843445,
      "loss/reference_anchor": 0.18182215094566345,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.4647648334503174,
      "rewards/margins": 1.1787031888961792,
      "rewards/rejected": 1.2860618829727173,
      "step": 100
    },
    {
      "drift/chosen_abs": 0.2001236379146576,
      "drift/rejected_abs": 0.079637810587883,
      "epoch": 0.10032246506628449,
      "grad_norm": 50.25,
      "learning_rate": 2.8888888888888885e-07,
      "logits/chosen": -2.3382439613342285,
      "logits/rejected": -2.424198865890503,
      "logps/chosen": -0.2838006615638733,
      "logps/rejected": -0.28019005060195923,
      "loss": 24.08165168762207,
      "loss/core": 24.064653396606445,
      "loss/preference_sft": 0.2838006615638733,
      "loss/reference_anchor": 0.1982627958059311,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0012366771698,
      "rewards/margins": 1.207162618637085,
      "rewards/rejected": 0.7940739393234253,
      "step": 105
    },
    {
      "drift/chosen_abs": 0.4472831189632416,
      "drift/rejected_abs": 0.18904665112495422,
      "epoch": 0.10509972530753613,
      "grad_norm": 71.0,
      "learning_rate": 3.0277777777777773e-07,
      "logits/chosen": -1.9077485799789429,
      "logits/rejected": -1.8608009815216064,
      "logps/chosen": -0.3383413255214691,
      "logps/rejected": -0.2978266477584839,
      "loss": 23.436580657958984,
      "loss/core": 23.36850357055664,
      "loss/preference_sft": 0.3383413255214691,
      "loss/reference_anchor": 0.8738870620727539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.471864223480225,
      "rewards/margins": 2.584223508834839,
      "rewards/rejected": 1.8876409530639648,
      "step": 110
    },
    {
      "drift/chosen_abs": 0.17792600393295288,
      "drift/rejected_abs": 0.14627966284751892,
      "epoch": 0.10987698554878778,
      "grad_norm": 334.0,
      "learning_rate": 3.166666666666666e-07,
      "logits/chosen": -2.088899850845337,
      "logits/rejected": -2.032494068145752,
      "logps/chosen": -0.30003082752227783,
      "logps/rejected": -0.2972942888736725,
      "loss": 24.94771957397461,
      "loss/core": 24.928913116455078,
      "loss/preference_sft": 0.30003082752227783,
      "loss/reference_anchor": 0.22078463435173035,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7790625095367432,
      "rewards/margins": 0.31695252656936646,
      "rewards/rejected": 1.4621098041534424,
      "step": 115
    },
    {
      "drift/chosen_abs": 0.18117384612560272,
      "drift/rejected_abs": 0.11267610639333725,
      "epoch": 0.11465424579003941,
      "grad_norm": 768.0,
      "learning_rate": 3.3055555555555556e-07,
      "logits/chosen": -2.0376081466674805,
      "logits/rejected": -2.0747933387756348,
      "logps/chosen": -0.3019946813583374,
      "logps/rejected": -0.29592204093933105,
      "loss": 24.376644134521484,
      "loss/core": 24.367816925048828,
      "loss/preference_sft": 0.3019946813583374,
      "loss/reference_anchor": 0.08752311021089554,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8115949630737305,
      "rewards/margins": 0.6873809099197388,
      "rewards/rejected": 1.1242139339447021,
      "step": 120
    },
    {
      "drift/chosen_abs": 0.20648498833179474,
      "drift/rejected_abs": 0.10197492688894272,
      "epoch": 0.11943150603129106,
      "grad_norm": 50.75,
      "learning_rate": 3.4444444444444444e-07,
      "logits/chosen": -2.1082651615142822,
      "logits/rejected": -2.139118194580078,
      "logps/chosen": -0.29931074380874634,
      "logps/rejected": -0.30651599168777466,
      "loss": 24.086589813232422,
      "loss/core": 24.07431983947754,
      "loss/preference_sft": 0.29931074380874634,
      "loss/reference_anchor": 0.13366781175136566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.0623745918273926,
      "rewards/margins": 1.0441718101501465,
      "rewards/rejected": 1.018202543258667,
      "step": 125
    },
    {
      "drift/chosen_abs": 0.18083539605140686,
      "drift/rejected_abs": 0.08944060653448105,
      "epoch": 0.1242087662725427,
      "grad_norm": 100.5,
      "learning_rate": 3.583333333333333e-07,
      "logits/chosen": -2.0009732246398926,
      "logits/rejected": -2.0936858654022217,
      "logps/chosen": -0.27988457679748535,
      "logps/rejected": -0.278131902217865,
      "loss": 24.129959106445312,
      "loss/core": 24.122276306152344,
      "loss/preference_sft": 0.27988457679748535,
      "loss/reference_anchor": 0.07443337142467499,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8083540201187134,
      "rewards/margins": 0.9148699641227722,
      "rewards/rejected": 0.8934842348098755,
      "step": 130
    },
    {
      "drift/chosen_abs": 0.33803224563598633,
      "drift/rejected_abs": 0.13368059694766998,
      "epoch": 0.12898602651379434,
      "grad_norm": 134.0,
      "learning_rate": 3.722222222222222e-07,
      "logits/chosen": -2.0908658504486084,
      "logits/rejected": -2.1650338172912598,
      "logps/chosen": -0.3015690743923187,
      "logps/rejected": -0.2998407483100891,
      "loss": 23.31682777404785,
      "loss/core": 23.28109359741211,
      "loss/preference_sft": 0.3015690743923187,
      "loss/reference_anchor": 0.446286141872406,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.3794403076171875,
      "rewards/margins": 2.081610918045044,
      "rewards/rejected": 1.2978298664093018,
      "step": 135
    },
    {
      "drift/chosen_abs": 0.16290757060050964,
      "drift/rejected_abs": 0.07148541510105133,
      "epoch": 0.13376328675504598,
      "grad_norm": 60.25,
      "learning_rate": 3.861111111111111e-07,
      "logits/chosen": -2.4541397094726562,
      "logits/rejected": -2.5143344402313232,
      "logps/chosen": -0.28250986337661743,
      "logps/rejected": -0.2895656228065491,
      "loss": 24.135583877563477,
      "loss/core": 24.1291446685791,
      "loss/preference_sft": 0.28250986337661743,
      "loss/reference_anchor": 0.05761883780360222,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6281633377075195,
      "rewards/margins": 0.9143427610397339,
      "rewards/rejected": 0.7138205170631409,
      "step": 140
    },
    {
      "drift/chosen_abs": 0.3603426516056061,
      "drift/rejected_abs": 0.17863936722278595,
      "epoch": 0.13854054699629761,
      "grad_norm": 96.0,
      "learning_rate": 4e-07,
      "logits/chosen": -1.77964186668396,
      "logits/rejected": -1.8101189136505127,
      "logps/chosen": -0.30144834518432617,
      "logps/rejected": -0.30214372277259827,
      "loss": 23.7529239654541,
      "loss/core": 23.6727352142334,
      "loss/preference_sft": 0.30144834518432617,
      "loss/reference_anchor": 1.0389857292175293,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.6030845642089844,
      "rewards/margins": 1.8218176364898682,
      "rewards/rejected": 1.7812671661376953,
      "step": 145
    },
    {
      "drift/chosen_abs": 0.36440587043762207,
      "drift/rejected_abs": 0.1463252753019333,
      "epoch": 0.14331780723754928,
      "grad_norm": 98.5,
      "learning_rate": 4.1388888888888887e-07,
      "logits/chosen": -2.2338650226593018,
      "logits/rejected": -2.282841444015503,
      "logps/chosen": -0.31720206141471863,
      "logps/rejected": -0.3081175684928894,
      "loss": 23.425756454467773,
      "loss/core": 23.36142349243164,
      "loss/preference_sft": 0.31720206141471863,
      "loss/reference_anchor": 0.8260464668273926,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.640651226043701,
      "rewards/margins": 2.182239055633545,
      "rewards/rejected": 1.4584124088287354,
      "step": 150
    },
    {
      "drift/chosen_abs": 0.28685861825942993,
      "drift/rejected_abs": 0.07811577618122101,
      "epoch": 0.1480950674788009,
      "grad_norm": 77.5,
      "learning_rate": 4.2777777777777775e-07,
      "logits/chosen": -2.0995328426361084,
      "logits/rejected": -2.177826166152954,
      "logps/chosen": -0.2826915383338928,
      "logps/rejected": -0.29879477620124817,
      "loss": 23.43790054321289,
      "loss/core": 23.408916473388672,
      "loss/preference_sft": 0.2826915383338928,
      "loss/reference_anchor": 0.3581943213939667,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8685860633850098,
      "rewards/margins": 2.106802225112915,
      "rewards/rejected": 0.7617841958999634,
      "step": 155
    },
    {
      "drift/chosen_abs": 0.31482672691345215,
      "drift/rejected_abs": 0.1407286822795868,
      "epoch": 0.15287232772005255,
      "grad_norm": 200.0,
      "learning_rate": 4.4166666666666664e-07,
      "logits/chosen": -2.05705189704895,
      "logits/rejected": -2.0315186977386475,
      "logps/chosen": -0.31071725487709045,
      "logps/rejected": -0.31473129987716675,
      "loss": 24.094379425048828,
      "loss/core": 24.039676666259766,
      "loss/preference_sft": 0.31071725487709045,
      "loss/reference_anchor": 0.698303759098053,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.1482670307159424,
      "rewards/margins": 1.7411534786224365,
      "rewards/rejected": 1.4071136713027954,
      "step": 160
    },
    {
      "drift/chosen_abs": 0.1602366864681244,
      "drift/rejected_abs": 0.07925491034984589,
      "epoch": 0.15764958796130418,
      "grad_norm": 80.5,
      "learning_rate": 4.555555555555555e-07,
      "logits/chosen": -2.223989725112915,
      "logits/rejected": -2.1556718349456787,
      "logps/chosen": -0.3111506998538971,
      "logps/rejected": -0.30567336082458496,
      "loss": 24.250789642333984,
      "loss/core": 24.241710662841797,
      "loss/preference_sft": 0.3111506998538971,
      "loss/reference_anchor": 0.08993285894393921,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.601798415184021,
      "rewards/margins": 0.8125208020210266,
      "rewards/rejected": 0.7892775535583496,
      "step": 165
    },
    {
      "drift/chosen_abs": 0.2597414255142212,
      "drift/rejected_abs": 0.2784227728843689,
      "epoch": 0.16242684820255585,
      "grad_norm": 81.5,
      "learning_rate": 4.694444444444444e-07,
      "logits/chosen": -2.187836170196533,
      "logits/rejected": -2.132575511932373,
      "logps/chosen": -0.27488964796066284,
      "logps/rejected": -0.2585277855396271,
      "loss": 26.337051391601562,
      "loss/core": 26.263935089111328,
      "loss/preference_sft": 0.27488964796066284,
      "loss/reference_anchor": 0.9473751783370972,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.59535551071167,
      "rewards/margins": -0.186926007270813,
      "rewards/rejected": 2.7822816371917725,
      "step": 170
    },
    {
      "drift/chosen_abs": 0.2583083212375641,
      "drift/rejected_abs": 0.12779515981674194,
      "epoch": 0.16720410844380748,
      "grad_norm": 372.0,
      "learning_rate": 4.833333333333333e-07,
      "logits/chosen": -2.1770424842834473,
      "logits/rejected": -2.216740608215332,
      "logps/chosen": -0.2869277000427246,
      "logps/rejected": -0.30118730664253235,
      "loss": 23.88767433166504,
      "loss/core": 23.843341827392578,
      "loss/preference_sft": 0.2869277000427246,
      "loss/reference_anchor": 0.5624145865440369,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.583082914352417,
      "rewards/margins": 1.3182199001312256,
      "rewards/rejected": 1.2648632526397705,
      "step": 175
    },
    {
      "drift/chosen_abs": 0.3239268660545349,
      "drift/rejected_abs": 0.1492597758769989,
      "epoch": 0.17198136868505912,
      "grad_norm": 280.0,
      "learning_rate": 4.972222222222222e-07,
      "logits/chosen": -2.201796054840088,
      "logits/rejected": -2.111999988555908,
      "logps/chosen": -0.27172645926475525,
      "logps/rejected": -0.2618543803691864,
      "loss": 23.988094329833984,
      "loss/core": 23.94233512878418,
      "loss/preference_sft": 0.27172645926475525,
      "loss/reference_anchor": 0.5829019546508789,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.2370712757110596,
      "rewards/margins": 1.7461532354354858,
      "rewards/rejected": 1.4909178018569946,
      "step": 180
    },
    {
      "drift/chosen_abs": 0.10453528165817261,
      "drift/rejected_abs": 0.06307251751422882,
      "epoch": 0.17675862892631075,
      "grad_norm": 62.0,
      "learning_rate": 4.999619237890978e-07,
      "logits/chosen": -2.206681966781616,
      "logits/rejected": -2.1635756492614746,
      "logps/chosen": -0.2884823679924011,
      "logps/rejected": -0.29190367460250854,
      "loss": 24.60689926147461,
      "loss/core": 24.602405548095703,
      "loss/preference_sft": 0.2884823679924011,
      "loss/reference_anchor": 0.031054329127073288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0448436737060547,
      "rewards/margins": 0.41608095169067383,
      "rewards/rejected": 0.6287627816200256,
      "step": 185
    },
    {
      "drift/chosen_abs": 0.28438663482666016,
      "drift/rejected_abs": 0.12391813099384308,
      "epoch": 0.18153588916756241,
      "grad_norm": 125.0,
      "learning_rate": 4.998072590601808e-07,
      "logits/chosen": -2.2966725826263428,
      "logits/rejected": -2.2743964195251465,
      "logps/chosen": -0.28313106298446655,
      "logps/rejected": -0.28445905447006226,
      "loss": 23.94525718688965,
      "loss/core": 23.900707244873047,
      "loss/preference_sft": 0.28313106298446655,
      "loss/reference_anchor": 0.5657359957695007,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.842867374420166,
      "rewards/margins": 1.6057891845703125,
      "rewards/rejected": 1.2370779514312744,
      "step": 190
    },
    {
      "drift/chosen_abs": 0.26715975999832153,
      "drift/rejected_abs": 0.09772941470146179,
      "epoch": 0.18631314940881405,
      "grad_norm": 67.5,
      "learning_rate": 4.995336996054667e-07,
      "logits/chosen": -1.9860740900039673,
      "logits/rejected": -2.006164073944092,
      "logps/chosen": -0.29072457551956177,
      "logps/rejected": -0.28935790061950684,
      "loss": 24.14964485168457,
      "loss/core": 24.105886459350586,
      "loss/preference_sft": 0.29072457551956177,
      "loss/reference_anchor": 0.5543695092201233,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.6712136268615723,
      "rewards/margins": 1.701263427734375,
      "rewards/rejected": 0.9699504971504211,
      "step": 195
    },
    {
      "drift/chosen_abs": 0.26089510321617126,
      "drift/rejected_abs": 0.12243443727493286,
      "epoch": 0.19109040965006568,
      "grad_norm": 75.0,
      "learning_rate": 4.991413756244404e-07,
      "logits/chosen": -1.8967984914779663,
      "logits/rejected": -1.9376294612884521,
      "logps/chosen": -0.3242345452308655,
      "logps/rejected": -0.3163239061832428,
      "loss": 23.804271697998047,
      "loss/core": 23.7833309173584,
      "loss/preference_sft": 0.3242345452308655,
      "loss/reference_anchor": 0.2467818558216095,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6078362464904785,
      "rewards/margins": 1.3843178749084473,
      "rewards/rejected": 1.2235184907913208,
      "step": 200
    },
    {
      "drift/chosen_abs": 0.328901469707489,
      "drift/rejected_abs": 0.2119167149066925,
      "epoch": 0.19586766989131732,
      "grad_norm": 50.75,
      "learning_rate": 4.986304738420683e-07,
      "logits/chosen": -2.1179518699645996,
      "logits/rejected": -2.169663667678833,
      "logps/chosen": -0.25019803643226624,
      "logps/rejected": -0.2696530222892761,
      "loss": 24.55345344543457,
      "loss/core": 24.50376319885254,
      "loss/preference_sft": 0.25019803643226624,
      "loss/reference_anchor": 0.6374753713607788,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.288543224334717,
      "rewards/margins": 1.1873959302902222,
      "rewards/rejected": 2.101147413253784,
      "step": 205
    },
    {
      "drift/chosen_abs": 0.2828282415866852,
      "drift/rejected_abs": 0.1485763043165207,
      "epoch": 0.20064493013256898,
      "grad_norm": 121.0,
      "learning_rate": 4.980012374199287e-07,
      "logits/chosen": -2.1135048866271973,
      "logits/rejected": -2.1378347873687744,
      "logps/chosen": -0.3129284381866455,
      "logps/rejected": -0.32191547751426697,
      "loss": 23.932619094848633,
      "loss/core": 23.88772964477539,
      "loss/preference_sft": 0.3129284381866455,
      "loss/reference_anchor": 0.5672147870063782,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8268136978149414,
      "rewards/margins": 1.3436048030853271,
      "rewards/rejected": 1.4832088947296143,
      "step": 210
    },
    {
      "drift/chosen_abs": 0.3194449245929718,
      "drift/rejected_abs": 0.1719251424074173,
      "epoch": 0.20542219037382062,
      "grad_norm": 133.0,
      "learning_rate": 4.972539658404792e-07,
      "logits/chosen": -2.0891597270965576,
      "logits/rejected": -2.0813040733337402,
      "logps/chosen": -0.2754015326499939,
      "logps/rejected": -0.2838408946990967,
      "loss": 24.59800910949707,
      "loss/core": 24.53702735900879,
      "loss/preference_sft": 0.2754015326499939,
      "loss/reference_anchor": 0.7855226397514343,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.1921732425689697,
      "rewards/margins": 1.485547661781311,
      "rewards/rejected": 1.7066256999969482,
      "step": 215
    },
    {
      "drift/chosen_abs": 0.18876484036445618,
      "drift/rejected_abs": 0.10738082975149155,
      "epoch": 0.21019945061507225,
      "grad_norm": 138.0,
      "learning_rate": 4.963890147645194e-07,
      "logits/chosen": -2.1035685539245605,
      "logits/rejected": -2.017023801803589,
      "logps/chosen": -0.3024953007698059,
      "logps/rejected": -0.30150723457336426,
      "loss": 24.240825653076172,
      "loss/core": 24.230350494384766,
      "loss/preference_sft": 0.3024953007698059,
      "loss/reference_anchor": 0.1094045639038086,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8857002258300781,
      "rewards/margins": 0.813431441783905,
      "rewards/rejected": 1.0722688436508179,
      "step": 220
    },
    {
      "drift/chosen_abs": 0.36280855536460876,
      "drift/rejected_abs": 0.14171245694160461,
      "epoch": 0.2149767108563239,
      "grad_norm": 68.0,
      "learning_rate": 4.95406795861916e-07,
      "logits/chosen": -2.189195156097412,
      "logits/rejected": -2.1777374744415283,
      "logps/chosen": -0.29006287455558777,
      "logps/rejected": -0.2748886048793793,
      "loss": 23.25436782836914,
      "loss/core": 23.220571517944336,
      "loss/preference_sft": 0.29006287455558777,
      "loss/reference_anchor": 0.4215586185455322,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.6274166107177734,
      "rewards/margins": 2.212296485900879,
      "rewards/rejected": 1.4151195287704468,
      "step": 225
    },
    {
      "drift/chosen_abs": 0.18566292524337769,
      "drift/rejected_abs": 0.07588974386453629,
      "epoch": 0.21975397109757555,
      "grad_norm": 828.0,
      "learning_rate": 4.943077766156697e-07,
      "logits/chosen": -2.2765140533447266,
      "logits/rejected": -2.2845187187194824,
      "logps/chosen": -0.2816498279571533,
      "logps/rejected": -0.30764302611351013,
      "loss": 24.015369415283203,
      "loss/core": 24.00626564025879,
      "loss/preference_sft": 0.2816498279571533,
      "loss/reference_anchor": 0.09326620399951935,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8551431894302368,
      "rewards/margins": 1.0997159481048584,
      "rewards/rejected": 0.7554270625114441,
      "step": 230
    },
    {
      "drift/chosen_abs": 0.19052401185035706,
      "drift/rejected_abs": 0.13283082842826843,
      "epoch": 0.2245312313388272,
      "grad_norm": 75.0,
      "learning_rate": 4.930924800994191e-07,
      "logits/chosen": -2.0162127017974854,
      "logits/rejected": -2.050363302230835,
      "logps/chosen": -0.2928079068660736,
      "logps/rejected": -0.28698280453681946,
      "loss": 24.55010414123535,
      "loss/core": 24.53103256225586,
      "loss/preference_sft": 0.2928079068660736,
      "loss/reference_anchor": 0.22502753138542175,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9030735492706299,
      "rewards/margins": 0.5786088705062866,
      "rewards/rejected": 1.3244645595550537,
      "step": 235
    },
    {
      "drift/chosen_abs": 0.1768554449081421,
      "drift/rejected_abs": 0.10264088213443756,
      "epoch": 0.22930849158007882,
      "grad_norm": 88.0,
      "learning_rate": 4.917614847284858e-07,
      "logits/chosen": -2.0287160873413086,
      "logits/rejected": -2.0479371547698975,
      "logps/chosen": -0.29354214668273926,
      "logps/rejected": -0.29428401589393616,
      "loss": 24.292566299438477,
      "loss/core": 24.284984588623047,
      "loss/preference_sft": 0.29354214668273926,
      "loss/reference_anchor": 0.07169236242771149,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.767877221107483,
      "rewards/margins": 0.7428776621818542,
      "rewards/rejected": 1.0249996185302734,
      "step": 240
    },
    {
      "drift/chosen_abs": 0.35989120602607727,
      "drift/rejected_abs": 0.1157422810792923,
      "epoch": 0.23408575182133046,
      "grad_norm": 1056.0,
      "learning_rate": 4.903154239845797e-07,
      "logits/chosen": -2.0682170391082764,
      "logits/rejected": -2.0564630031585693,
      "logps/chosen": -0.2781009078025818,
      "logps/rejected": -0.2913815677165985,
      "loss": 23.59634017944336,
      "loss/core": 23.54959487915039,
      "loss/preference_sft": 0.2781009078025818,
      "loss/reference_anchor": 0.5954445600509644,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.5985991954803467,
      "rewards/margins": 2.4494547843933105,
      "rewards/rejected": 1.1491445302963257,
      "step": 245
    },
    {
      "drift/chosen_abs": 0.2339925318956375,
      "drift/rejected_abs": 0.10722045600414276,
      "epoch": 0.23886301206258212,
      "grad_norm": 101.0,
      "learning_rate": 4.887549861142967e-07,
      "logits/chosen": -1.9105064868927002,
      "logits/rejected": -1.9906113147735596,
      "logps/chosen": -0.2999528646469116,
      "logps/rejected": -0.29507893323898315,
      "loss": 23.889766693115234,
      "loss/core": 23.876558303833008,
      "loss/preference_sft": 0.2999528646469116,
      "loss/reference_anchor": 0.1460881531238556,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.337829351425171,
      "rewards/margins": 1.2744026184082031,
      "rewards/rejected": 1.0634267330169678,
      "step": 250
    },
    {
      "drift/chosen_abs": 0.24554595351219177,
      "drift/rejected_abs": 0.16090847551822662,
      "epoch": 0.24364027230383375,
      "grad_norm": 64.0,
      "learning_rate": 4.870809138015498e-07,
      "logits/chosen": -2.220195770263672,
      "logits/rejected": -2.2151570320129395,
      "logps/chosen": -0.2845707833766937,
      "logps/rejected": -0.28703150153160095,
      "loss": 24.28571128845215,
      "loss/core": 24.260061264038086,
      "loss/preference_sft": 0.2845707833766937,
      "loss/reference_anchor": 0.31353074312210083,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4550528526306152,
      "rewards/margins": 0.8476669192314148,
      "rewards/rejected": 1.6073859930038452,
      "step": 255
    },
    {
      "drift/chosen_abs": 0.2403588742017746,
      "drift/rejected_abs": 0.15879188477993011,
      "epoch": 0.2484175325450854,
      "grad_norm": 35.25,
      "learning_rate": 4.852940038140927e-07,
      "logits/chosen": -2.1549439430236816,
      "logits/rejected": -2.227227210998535,
      "logps/chosen": -0.28740060329437256,
      "logps/rejected": -0.27014413475990295,
      "loss": 24.529048919677734,
      "loss/core": 24.506467819213867,
      "loss/preference_sft": 0.28740060329437256,
      "loss/reference_anchor": 0.2723291516304016,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.403057336807251,
      "rewards/margins": 0.8171398043632507,
      "rewards/rejected": 1.5859174728393555,
      "step": 260
    },
    {
      "drift/chosen_abs": 0.12319445610046387,
      "drift/rejected_abs": 0.11670766770839691,
      "epoch": 0.25319479278633705,
      "grad_norm": 34.0,
      "learning_rate": 4.833951066243004e-07,
      "logits/chosen": -2.0384128093719482,
      "logits/rejected": -2.0463626384735107,
      "logps/chosen": -0.29939448833465576,
      "logps/rejected": -0.30272746086120605,
      "loss": 25.318052291870117,
      "loss/core": 25.299692153930664,
      "loss/preference_sft": 0.29939448833465576,
      "loss/reference_anchor": 0.21487000584602356,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.2312783002853394,
      "rewards/margins": 0.06888408958911896,
      "rewards/rejected": 1.1623942852020264,
      "step": 265
    },
    {
      "drift/chosen_abs": 0.32817989587783813,
      "drift/rejected_abs": 0.21317580342292786,
      "epoch": 0.2579720530275887,
      "grad_norm": 57.0,
      "learning_rate": 4.813851260043915e-07,
      "logits/chosen": -1.9417810440063477,
      "logits/rejected": -2.031541347503662,
      "logps/chosen": -0.28844502568244934,
      "logps/rejected": -0.29021137952804565,
      "loss": 24.536828994750977,
      "loss/core": 24.490381240844727,
      "loss/preference_sft": 0.28844502568244934,
      "loss/reference_anchor": 0.5904703140258789,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.2817986011505127,
      "rewards/margins": 1.1503736972808838,
      "rewards/rejected": 2.13142466545105,
      "step": 270
    },
    {
      "drift/chosen_abs": 0.23207350075244904,
      "drift/rejected_abs": 0.12252874672412872,
      "epoch": 0.2627493132688403,
      "grad_norm": 572.0,
      "learning_rate": 4.79265018596281e-07,
      "logits/chosen": -2.0927796363830566,
      "logits/rejected": -2.0458498001098633,
      "logps/chosen": -0.29707807302474976,
      "logps/rejected": -0.3014887869358063,
      "loss": 23.969472885131836,
      "loss/core": 23.95018768310547,
      "loss/preference_sft": 0.29707807302474976,
      "loss/reference_anchor": 0.22740456461906433,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.3190271854400635,
      "rewards/margins": 1.1700794696807861,
      "rewards/rejected": 1.1489474773406982,
      "step": 275
    },
    {
      "drift/chosen_abs": 0.2051713466644287,
      "drift/rejected_abs": 0.08534234762191772,
      "epoch": 0.26752657351009196,
      "grad_norm": 56.75,
      "learning_rate": 4.770357934562704e-07,
      "logits/chosen": -2.063467502593994,
      "logits/rejected": -1.9795265197753906,
      "logps/chosen": -0.2879626750946045,
      "logps/rejected": -0.3061186373233795,
      "loss": 23.862253189086914,
      "loss/core": 23.851999282836914,
      "loss/preference_sft": 0.2879626750946045,
      "loss/reference_anchor": 0.10792496055364609,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.0476605892181396,
      "rewards/margins": 1.2737023830413818,
      "rewards/rejected": 0.7739582061767578,
      "step": 280
    },
    {
      "drift/chosen_abs": 0.2606334090232849,
      "drift/rejected_abs": 0.16303770244121552,
      "epoch": 0.2723038337513436,
      "grad_norm": 168.0,
      "learning_rate": 4.746985115747917e-07,
      "logits/chosen": -2.0845389366149902,
      "logits/rejected": -2.0801751613616943,
      "logps/chosen": -0.29716962575912476,
      "logps/rejected": -0.29298409819602966,
      "loss": 24.26735496520996,
      "loss/core": 24.245363235473633,
      "loss/preference_sft": 0.29716962575912476,
      "loss/reference_anchor": 0.26353123784065247,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.605191707611084,
      "rewards/margins": 0.9785102605819702,
      "rewards/rejected": 1.6266816854476929,
      "step": 285
    },
    {
      "drift/chosen_abs": 0.14788036048412323,
      "drift/rejected_abs": 0.14346002042293549,
      "epoch": 0.27708109399259523,
      "grad_norm": 1488.0,
      "learning_rate": 4.722542853714341e-07,
      "logits/chosen": -2.1646649837493896,
      "logits/rejected": -2.176093578338623,
      "logps/chosen": -0.2869679927825928,
      "logps/rejected": -0.28978294134140015,
      "loss": 25.25179672241211,
      "loss/core": 25.234987258911133,
      "loss/preference_sft": 0.2869679927825928,
      "loss/reference_anchor": 0.19546569883823395,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.4774421453475952,
      "rewards/margins": 0.04469200596213341,
      "rewards/rejected": 1.4327499866485596,
      "step": 290
    },
    {
      "drift/chosen_abs": 0.1308569461107254,
      "drift/rejected_abs": 0.0676531195640564,
      "epoch": 0.28185835423384686,
      "grad_norm": 64.5,
      "learning_rate": 4.697042781654913e-07,
      "logits/chosen": -2.565634250640869,
      "logits/rejected": -2.5284571647644043,
      "logps/chosen": -0.2447257786989212,
      "logps/rejected": -0.2566220164299011,
      "loss": 24.383955001831055,
      "loss/core": 24.379867553710938,
      "loss/preference_sft": 0.2447257786989212,
      "loss/reference_anchor": 0.030016642063856125,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.308022379875183,
      "rewards/margins": 0.6341660022735596,
      "rewards/rejected": 0.6738564968109131,
      "step": 295
    },
    {
      "drift/chosen_abs": 0.3740275204181671,
      "drift/rejected_abs": 0.10665237903594971,
      "epoch": 0.28663561447509855,
      "grad_norm": 388.0,
      "learning_rate": 4.670497036222856e-07,
      "logits/chosen": -1.9813871383666992,
      "logits/rejected": -1.9485054016113281,
      "logps/chosen": -0.31088101863861084,
      "logps/rejected": -0.3178117573261261,
      "loss": 23.0051326751709,
      "loss/core": 22.965051651000977,
      "loss/preference_sft": 0.31088101863861084,
      "loss/reference_anchor": 0.5033376216888428,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.739037275314331,
      "rewards/margins": 2.6878647804260254,
      "rewards/rejected": 1.0511722564697266,
      "step": 300
    },
    {
      "drift/chosen_abs": 0.2159603089094162,
      "drift/rejected_abs": 0.0897536352276802,
      "epoch": 0.2914128747163502,
      "grad_norm": 76.0,
      "learning_rate": 4.642918251755281e-07,
      "logits/chosen": -2.125406265258789,
      "logits/rejected": -2.185771942138672,
      "logps/chosen": -0.26945987343788147,
      "logps/rejected": -0.26080164313316345,
      "loss": 23.85554313659668,
      "loss/core": 23.84500503540039,
      "loss/preference_sft": 0.26945987343788147,
      "loss/reference_anchor": 0.11357325315475464,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.159000873565674,
      "rewards/margins": 1.2621368169784546,
      "rewards/rejected": 0.896864116191864,
      "step": 305
    },
    {
      "drift/chosen_abs": 0.26018840074539185,
      "drift/rejected_abs": 0.2073136568069458,
      "epoch": 0.2961901349576018,
      "grad_norm": 976.0,
      "learning_rate": 4.614319554259933e-07,
      "logits/chosen": -2.12811017036438,
      "logits/rejected": -2.0370516777038574,
      "logps/chosen": -0.2741633355617523,
      "logps/rejected": -0.2802058160305023,
      "loss": 25.01598358154297,
      "loss/core": 24.98000144958496,
      "loss/preference_sft": 0.2741633355617523,
      "loss/reference_anchor": 0.452305406332016,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.6010148525238037,
      "rewards/margins": 0.5811925530433655,
      "rewards/rejected": 2.019822359085083,
      "step": 310
    },
    {
      "drift/chosen_abs": 0.29169219732284546,
      "drift/rejected_abs": 0.13470803201198578,
      "epoch": 0.30096739519885346,
      "grad_norm": 61.0,
      "learning_rate": 4.58471455516792e-07,
      "logits/chosen": -1.9604593515396118,
      "logits/rejected": -2.0688371658325195,
      "logps/chosen": -0.2688148617744446,
      "logps/rejected": -0.2757693827152252,
      "loss": 23.543533325195312,
      "loss/core": 23.524099349975586,
      "loss/preference_sft": 0.2688148617744446,
      "loss/reference_anchor": 0.23222799599170685,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9164979457855225,
      "rewards/margins": 1.5716060400009155,
      "rewards/rejected": 1.344891905784607,
      "step": 315
    },
    {
      "drift/chosen_abs": 0.1583484411239624,
      "drift/rejected_abs": 0.12316056340932846,
      "epoch": 0.3057446554401051,
      "grad_norm": 51.5,
      "learning_rate": 4.5541173448554095e-07,
      "logits/chosen": -2.0016112327575684,
      "logits/rejected": -2.0177295207977295,
      "logps/chosen": -0.2901010513305664,
      "logps/rejected": -0.2924564480781555,
      "loss": 24.739116668701172,
      "loss/core": 24.729549407958984,
      "loss/preference_sft": 0.2901010513305664,
      "loss/reference_anchor": 0.09852476418018341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.5812407732009888,
      "rewards/margins": 0.35098347067832947,
      "rewards/rejected": 1.2302576303482056,
      "step": 320
    },
    {
      "drift/chosen_abs": 0.36936116218566895,
      "drift/rejected_abs": 0.1534915715456009,
      "epoch": 0.31052191568135673,
      "grad_norm": 146.0,
      "learning_rate": 4.5225424859373684e-07,
      "logits/chosen": -2.257908344268799,
      "logits/rejected": -2.2035529613494873,
      "logps/chosen": -0.28698205947875977,
      "logps/rejected": -0.28376373648643494,
      "loss": 24.22592544555664,
      "loss/core": 24.155797958374023,
      "loss/preference_sft": 0.28698205947875977,
      "loss/reference_anchor": 0.9063272476196289,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.6920619010925293,
      "rewards/margins": 2.15855073928833,
      "rewards/rejected": 1.5335115194320679,
      "step": 325
    },
    {
      "drift/chosen_abs": 0.20329172909259796,
      "drift/rejected_abs": 0.11353343725204468,
      "epoch": 0.31529917592260837,
      "grad_norm": 40.75,
      "learning_rate": 4.4900050063365547e-07,
      "logits/chosen": -2.2280595302581787,
      "logits/rejected": -2.217438220977783,
      "logps/chosen": -0.27844610810279846,
      "logps/rejected": -0.270263135433197,
      "loss": 24.234060287475586,
      "loss/core": 24.221086502075195,
      "loss/preference_sft": 0.27844610810279846,
      "loss/reference_anchor": 0.14516863226890564,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.0322957038879395,
      "rewards/margins": 0.8984895944595337,
      "rewards/rejected": 1.1338062286376953,
      "step": 330
    },
    {
      "drift/chosen_abs": 0.2127622663974762,
      "drift/rejected_abs": 0.07780848443508148,
      "epoch": 0.32007643616386,
      "grad_norm": 165.0,
      "learning_rate": 4.4565203921310344e-07,
      "logits/chosen": -2.0511183738708496,
      "logits/rejected": -2.0350396633148193,
      "logps/chosen": -0.30186930298805237,
      "logps/rejected": -0.3197532296180725,
      "loss": 23.990903854370117,
      "loss/core": 23.97220230102539,
      "loss/preference_sft": 0.30186930298805237,
      "loss/reference_anchor": 0.21921315789222717,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.124210834503174,
      "rewards/margins": 1.4477331638336182,
      "rewards/rejected": 0.6764776110649109,
      "step": 335
    },
    {
      "drift/chosen_abs": 0.2031564712524414,
      "drift/rejected_abs": 0.087882399559021,
      "epoch": 0.3248536964051117,
      "grad_norm": 186.0,
      "learning_rate": 4.422104580183649e-07,
      "logits/chosen": -2.095062017440796,
      "logits/rejected": -2.2184524536132812,
      "logps/chosen": -0.28098320960998535,
      "logps/rejected": -0.28097662329673767,
      "loss": 24.006999969482422,
      "loss/core": 23.993459701538086,
      "loss/preference_sft": 0.28098320960998535,
      "loss/reference_anchor": 0.15243388712406158,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.031026840209961,
      "rewards/margins": 1.153816819190979,
      "rewards/rejected": 0.8772099614143372,
      "step": 340
    },
    {
      "drift/chosen_abs": 0.16465213894844055,
      "drift/rejected_abs": 0.0845918357372284,
      "epoch": 0.3296309566463633,
      "grad_norm": 98.5,
      "learning_rate": 4.3867739505569303e-07,
      "logits/chosen": -2.3218281269073486,
      "logits/rejected": -2.3827481269836426,
      "logps/chosen": -0.282081663608551,
      "logps/rejected": -0.29249638319015503,
      "loss": 24.309249877929688,
      "loss/core": 24.29976463317871,
      "loss/preference_sft": 0.282081663608551,
      "loss/reference_anchor": 0.09826230257749557,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.644649863243103,
      "rewards/margins": 0.799951434135437,
      "rewards/rejected": 0.8446983098983765,
      "step": 345
    },
    {
      "drift/chosen_abs": 0.20000576972961426,
      "drift/rejected_abs": 0.09551592916250229,
      "epoch": 0.33440821688761496,
      "grad_norm": 80.5,
      "learning_rate": 4.35054531871708e-07,
      "logits/chosen": -2.0598835945129395,
      "logits/rejected": -2.03928804397583,
      "logps/chosen": -0.26030445098876953,
      "logps/rejected": -0.2558044195175171,
      "loss": 24.042661666870117,
      "loss/core": 24.03305435180664,
      "loss/preference_sft": 0.26030445098876953,
      "loss/reference_anchor": 0.1020883321762085,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.9989124536514282,
      "rewards/margins": 1.0452065467834473,
      "rewards/rejected": 0.953705906867981,
      "step": 350
    },
    {
      "drift/chosen_abs": 0.18520495295524597,
      "drift/rejected_abs": 0.1412447839975357,
      "epoch": 0.3391854771288666,
      "grad_norm": 75.5,
      "learning_rate": 4.3134359275307185e-07,
      "logits/chosen": -2.106975555419922,
      "logits/rejected": -2.2341411113739014,
      "logps/chosen": -0.28064435720443726,
      "logps/rejected": -0.2879922389984131,
      "loss": 24.90737533569336,
      "loss/core": 24.887197494506836,
      "loss/preference_sft": 0.28064435720443726,
      "loss/reference_anchor": 0.2410183846950531,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.851447343826294,
      "rewards/margins": 0.45425519347190857,
      "rewards/rejected": 1.3971920013427734,
      "step": 355
    },
    {
      "drift/chosen_abs": 0.26272064447402954,
      "drift/rejected_abs": 0.1274389922618866,
      "epoch": 0.34396273737011823,
      "grad_norm": 472.0,
      "learning_rate": 4.2754634390582133e-07,
      "logits/chosen": -2.1305363178253174,
      "logits/rejected": -2.192857265472412,
      "logps/chosen": -0.28095608949661255,
      "logps/rejected": -0.28217315673828125,
      "loss": 24.006431579589844,
      "loss/core": 23.981016159057617,
      "loss/preference_sft": 0.28095608949661255,
      "loss/reference_anchor": 0.31079161167144775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.62534761428833,
      "rewards/margins": 1.353183627128601,
      "rewards/rejected": 1.272163987159729,
      "step": 360
    },
    {
      "drift/chosen_abs": 0.2852028012275696,
      "drift/rejected_abs": 0.08169830590486526,
      "epoch": 0.34873999761136987,
      "grad_norm": 205.0,
      "learning_rate": 4.236645926147493e-07,
      "logits/chosen": -2.069857120513916,
      "logits/rejected": -2.1774981021881104,
      "logps/chosen": -0.287311851978302,
      "logps/rejected": -0.2833508551120758,
      "loss": 23.85344123840332,
      "loss/core": 23.815753936767578,
      "loss/preference_sft": 0.287311851978302,
      "loss/reference_anchor": 0.4737482964992523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.851012945175171,
      "rewards/margins": 2.036228895187378,
      "rewards/rejected": 0.8147839307785034,
      "step": 365
    },
    {
      "drift/chosen_abs": 0.1644589602947235,
      "drift/rejected_abs": 0.10670790821313858,
      "epoch": 0.3535172578526215,
      "grad_norm": 188.0,
      "learning_rate": 4.1970018638323547e-07,
      "logits/chosen": -2.2686495780944824,
      "logits/rejected": -2.3140177726745605,
      "logps/chosen": -0.2849913239479065,
      "logps/rejected": -0.2848779857158661,
      "loss": 24.466651916503906,
      "loss/core": 24.457897186279297,
      "loss/preference_sft": 0.2849913239479065,
      "loss/reference_anchor": 0.08822296559810638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6427894830703735,
      "rewards/margins": 0.5768302083015442,
      "rewards/rejected": 1.0659592151641846,
      "step": 370
    },
    {
      "drift/chosen_abs": 0.3016364276409149,
      "drift/rejected_abs": 0.22407476603984833,
      "epoch": 0.35829451809387314,
      "grad_norm": 70.5,
      "learning_rate": 4.1565501205393436e-07,
      "logits/chosen": -2.0465548038482666,
      "logits/rejected": -2.024115562438965,
      "logps/chosen": -0.25850996375083923,
      "logps/rejected": -0.25814658403396606,
      "loss": 24.701723098754883,
      "loss/core": 24.66933822631836,
      "loss/preference_sft": 0.25850996375083923,
      "loss/reference_anchor": 0.4059620499610901,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.0161895751953125,
      "rewards/margins": 0.7962467670440674,
      "rewards/rejected": 2.2199432849884033,
      "step": 375
    },
    {
      "drift/chosen_abs": 0.20642168819904327,
      "drift/rejected_abs": 0.16034074127674103,
      "epoch": 0.36307177833512483,
      "grad_norm": 106.0,
      "learning_rate": 4.1153099491074093e-07,
      "logits/chosen": -2.122387409210205,
      "logits/rejected": -2.131068468093872,
      "logps/chosen": -0.27889400720596313,
      "logps/rejected": -0.2892562448978424,
      "loss": 25.11880874633789,
      "loss/core": 25.087757110595703,
      "loss/preference_sft": 0.27889400720596313,
      "loss/reference_anchor": 0.38615480065345764,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.06227445602417,
      "rewards/margins": 0.45964282751083374,
      "rewards/rejected": 1.6026318073272705,
      "step": 380
    },
    {
      "drift/chosen_abs": 0.21806609630584717,
      "drift/rejected_abs": 0.15287581086158752,
      "epoch": 0.36784903857637646,
      "grad_norm": 131.0,
      "learning_rate": 4.0733009776245937e-07,
      "logits/chosen": -2.115004062652588,
      "logits/rejected": -2.1338284015655518,
      "logps/chosen": -0.2700949013233185,
      "logps/rejected": -0.27447065711021423,
      "loss": 24.529088973999023,
      "loss/core": 24.512638092041016,
      "loss/preference_sft": 0.2700949013233185,
      "loss/reference_anchor": 0.19232222437858582,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.179309129714966,
      "rewards/margins": 0.6530361175537109,
      "rewards/rejected": 1.5262731313705444,
      "step": 385
    },
    {
      "drift/chosen_abs": 0.17654861509799957,
      "drift/rejected_abs": 0.09032897651195526,
      "epoch": 0.3726262988176281,
      "grad_norm": 42.75,
      "learning_rate": 4.0305432000861226e-07,
      "logits/chosen": -1.8726367950439453,
      "logits/rejected": -1.9506858587265015,
      "logps/chosen": -0.3072229325771332,
      "logps/rejected": -0.3017742931842804,
      "loss": 24.248693466186523,
      "loss/core": 24.23879051208496,
      "loss/preference_sft": 0.3072229325771332,
      "loss/reference_anchor": 0.10132338106632233,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.761505126953125,
      "rewards/margins": 0.8584615588188171,
      "rewards/rejected": 0.9030437469482422,
      "step": 390
    },
    {
      "drift/chosen_abs": 0.1685396432876587,
      "drift/rejected_abs": 0.08101732283830643,
      "epoch": 0.37740355905887973,
      "grad_norm": 101.0,
      "learning_rate": 3.9870569668783533e-07,
      "logits/chosen": -2.223672389984131,
      "logits/rejected": -2.3257579803466797,
      "logps/chosen": -0.29469752311706543,
      "logps/rejected": -0.28933185338974,
      "loss": 24.182411193847656,
      "loss/core": 24.174169540405273,
      "loss/preference_sft": 0.29469752311706543,
      "loss/reference_anchor": 0.08043049275875092,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.6851272583007812,
      "rewards/margins": 0.8753724098205566,
      "rewards/rejected": 0.8097550272941589,
      "step": 395
    },
    {
      "drift/chosen_abs": 0.29273721575737,
      "drift/rejected_abs": 0.0846409797668457,
      "epoch": 0.38218081930013137,
      "grad_norm": 85.0,
      "learning_rate": 3.942862975093084e-07,
      "logits/chosen": -2.1020679473876953,
      "logits/rejected": -2.1372578144073486,
      "logps/chosen": -0.29274052381515503,
      "logps/rejected": -0.2928157448768616,
      "loss": 23.699491500854492,
      "loss/core": 23.65845489501953,
      "loss/preference_sft": 0.29274052381515503,
      "loss/reference_anchor": 0.5178688168525696,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.925940990447998,
      "rewards/margins": 2.0813891887664795,
      "rewards/rejected": 0.8445518612861633,
      "step": 400
    },
    {
      "drift/chosen_abs": 0.2351594716310501,
      "drift/rejected_abs": 0.12229617685079575,
      "epoch": 0.386958079541383,
      "grad_norm": 65.0,
      "learning_rate": 3.8979822586768666e-07,
      "logits/chosen": -2.135050058364868,
      "logits/rejected": -2.189692735671997,
      "logps/chosen": -0.3113452196121216,
      "logps/rejected": -0.3050457835197449,
      "loss": 24.190441131591797,
      "loss/core": 24.1619873046875,
      "loss/preference_sft": 0.3113452196121216,
      "loss/reference_anchor": 0.3482899069786072,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3515946865081787,
      "rewards/margins": 1.1289160251617432,
      "rewards/rejected": 1.2226788997650146,
      "step": 405
    },
    {
      "drift/chosen_abs": 0.20916077494621277,
      "drift/rejected_abs": 0.11924557387828827,
      "epoch": 0.39173533978263464,
      "grad_norm": 86.0,
      "learning_rate": 3.8524361784199847e-07,
      "logits/chosen": -2.251237630844116,
      "logits/rejected": -2.2815263271331787,
      "logps/chosen": -0.27680549025535583,
      "logps/rejected": -0.2878721058368683,
      "loss": 24.154521942138672,
      "loss/core": 24.142080307006836,
      "loss/preference_sft": 0.27680549025535583,
      "loss/reference_anchor": 0.1381910741329193,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.090045690536499,
      "rewards/margins": 0.8996987342834473,
      "rewards/rejected": 1.1903469562530518,
      "step": 410
    },
    {
      "drift/chosen_abs": 0.09965378046035767,
      "drift/rejected_abs": 0.037877295166254044,
      "epoch": 0.3965126000238863,
      "grad_norm": 77.5,
      "learning_rate": 3.806246411789872e-07,
      "logits/chosen": -2.1194329261779785,
      "logits/rejected": -2.1981375217437744,
      "logps/chosen": -0.33038097620010376,
      "logps/rejected": -0.32079142332077026,
      "loss": 24.420391082763672,
      "loss/core": 24.41567039489746,
      "loss/preference_sft": 0.33038097620010376,
      "loss/reference_anchor": 0.02990715578198433,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.9942773580551147,
      "rewards/margins": 0.616710364818573,
      "rewards/rejected": 0.37756696343421936,
      "step": 415
    },
    {
      "drift/chosen_abs": 0.2046138346195221,
      "drift/rejected_abs": 0.07252134382724762,
      "epoch": 0.40128986026513797,
      "grad_norm": 129.0,
      "learning_rate": 3.7594349426138154e-07,
      "logits/chosen": -2.350454807281494,
      "logits/rejected": -2.316537618637085,
      "logps/chosen": -0.2832780182361603,
      "logps/rejected": -0.29880502820014954,
      "loss": 23.930326461791992,
      "loss/core": 23.915685653686523,
      "loss/preference_sft": 0.2832780182361603,
      "loss/reference_anchor": 0.1668982058763504,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.046138286590576,
      "rewards/margins": 1.3250911235809326,
      "rewards/rejected": 0.721047043800354,
      "step": 420
    },
    {
      "drift/chosen_abs": 0.2236243188381195,
      "drift/rejected_abs": 0.1645563691854477,
      "epoch": 0.4060671205063896,
      "grad_norm": 486.0,
      "learning_rate": 3.712024050615843e-07,
      "logits/chosen": -2.0146186351776123,
      "logits/rejected": -2.0732080936431885,
      "logps/chosen": -0.2916070222854614,
      "logps/rejected": -0.2652800679206848,
      "loss": 25.089502334594727,
      "loss/core": 25.058069229125977,
      "loss/preference_sft": 0.2916070222854614,
      "loss/reference_anchor": 0.3899894654750824,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.235166549682617,
      "rewards/margins": 0.5908761620521545,
      "rewards/rejected": 1.6442902088165283,
      "step": 425
    },
    {
      "drift/chosen_abs": 0.44021230936050415,
      "drift/rejected_abs": 0.1417957842350006,
      "epoch": 0.41084438074764124,
      "grad_norm": 274.0,
      "learning_rate": 3.664036300812778e-07,
      "logits/chosen": -2.0683376789093018,
      "logits/rejected": -2.091358184814453,
      "logps/chosen": -0.27976277470588684,
      "logps/rejected": -0.2667387127876282,
      "loss": 23.26540184020996,
      "loss/core": 23.194873809814453,
      "loss/preference_sft": 0.27976277470588684,
      "loss/reference_anchor": 0.9124003648757935,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.400694847106934,
      "rewards/margins": 2.9839932918548584,
      "rewards/rejected": 1.4167020320892334,
      "step": 430
    },
    {
      "drift/chosen_abs": 0.16288933157920837,
      "drift/rejected_abs": 0.10126354545354843,
      "epoch": 0.41562164098889287,
      "grad_norm": 99.5,
      "learning_rate": 3.615494532774522e-07,
      "logits/chosen": -2.3494062423706055,
      "logits/rejected": -2.3344578742980957,
      "logps/chosen": -0.2779366374015808,
      "logps/rejected": -0.29263314604759216,
      "loss": 24.44803810119629,
      "loss/core": 24.437061309814453,
      "loss/preference_sft": 0.2779366374015808,
      "loss/reference_anchor": 0.11851142346858978,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.628753900527954,
      "rewards/margins": 0.6175113916397095,
      "rewards/rejected": 1.0112426280975342,
      "step": 435
    },
    {
      "drift/chosen_abs": 0.30069416761398315,
      "drift/rejected_abs": 0.13946831226348877,
      "epoch": 0.4203989012301445,
      "grad_norm": 668.0,
      "learning_rate": 3.5664218497536456e-07,
      "logits/chosen": -2.05379581451416,
      "logits/rejected": -2.1259214878082275,
      "logps/chosen": -0.303558886051178,
      "logps/rejected": -0.3001348078250885,
      "loss": 24.09839630126953,
      "loss/core": 24.058101654052734,
      "loss/preference_sft": 0.303558886051178,
      "loss/reference_anchor": 0.5069498419761658,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.003722667694092,
      "rewards/margins": 1.6185331344604492,
      "rewards/rejected": 1.3851896524429321,
      "step": 440
    },
    {
      "drift/chosen_abs": 0.29670435190200806,
      "drift/rejected_abs": 0.16009071469306946,
      "epoch": 0.42517616147139614,
      "grad_norm": 130.0,
      "learning_rate": 3.516841607689501e-07,
      "logits/chosen": -1.9354779720306396,
      "logits/rejected": -1.9123506546020508,
      "logps/chosen": -0.29504531621932983,
      "logps/rejected": -0.2956696152687073,
      "loss": 23.822757720947266,
      "loss/core": 23.801040649414062,
      "loss/preference_sft": 0.29504531621932983,
      "loss/reference_anchor": 0.2600812315940857,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.9661905765533447,
      "rewards/margins": 1.388580560684204,
      "rewards/rejected": 1.577609896659851,
      "step": 445
    },
    {
      "drift/chosen_abs": 0.345471054315567,
      "drift/rejected_abs": 0.161865234375,
      "epoch": 0.4299534217126478,
      "grad_norm": 768.0,
      "learning_rate": 3.4667774040920515e-07,
      "logits/chosen": -1.9765275716781616,
      "logits/rejected": -1.9926903247833252,
      "logps/chosen": -0.29864731431007385,
      "logps/rejected": -0.2888573110103607,
      "loss": 23.699243545532227,
      "loss/core": 23.66305160522461,
      "loss/preference_sft": 0.29864731431007385,
      "loss/reference_anchor": 0.4526439309120178,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.4545040130615234,
      "rewards/margins": 1.8422412872314453,
      "rewards/rejected": 1.6122627258300781,
      "step": 450
    },
    {
      "drift/chosen_abs": 0.34665170311927795,
      "drift/rejected_abs": 0.14890094101428986,
      "epoch": 0.4347306819538994,
      "grad_norm": 246.0,
      "learning_rate": 3.416253066810743e-07,
      "logits/chosen": -1.936833143234253,
      "logits/rejected": -2.127368927001953,
      "logps/chosen": -0.32262879610061646,
      "logps/rejected": -0.31309956312179565,
      "loss": 23.719173431396484,
      "loss/core": 23.66214370727539,
      "loss/preference_sft": 0.32262879610061646,
      "loss/reference_anchor": 0.728130042552948,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.4665169715881348,
      "rewards/margins": 1.979360580444336,
      "rewards/rejected": 1.4871563911437988,
      "step": 455
    },
    {
      "drift/chosen_abs": 0.36950841546058655,
      "drift/rejected_abs": 0.18701323866844177,
      "epoch": 0.4395079421951511,
      "grad_norm": 141.0,
      "learning_rate": 3.3652926426937325e-07,
      "logits/chosen": -2.0627896785736084,
      "logits/rejected": -2.0499472618103027,
      "logps/chosen": -0.2930159270763397,
      "logps/rejected": -0.3065492510795593,
      "loss": 23.571096420288086,
      "loss/core": 23.52437973022461,
      "loss/preference_sft": 0.2930159270763397,
      "loss/reference_anchor": 0.5935932397842407,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.694608688354492,
      "rewards/margins": 1.8250821828842163,
      "rewards/rejected": 1.8695262670516968,
      "step": 460
    },
    {
      "drift/chosen_abs": 0.23000410199165344,
      "drift/rejected_abs": 0.09393614530563354,
      "epoch": 0.44428520243640274,
      "grad_norm": 60.25,
      "learning_rate": 3.3139203861428914e-07,
      "logits/chosen": -1.8922626972198486,
      "logits/rejected": -1.9913305044174194,
      "logps/chosen": -0.29730328917503357,
      "logps/rejected": -0.29092690348625183,
      "loss": 23.748533248901367,
      "loss/core": 23.73752212524414,
      "loss/preference_sft": 0.29730328917503357,
      "loss/reference_anchor": 0.11708039045333862,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.2995853424072266,
      "rewards/margins": 1.3625905513763428,
      "rewards/rejected": 0.936994731426239,
      "step": 465
    },
    {
      "drift/chosen_abs": 0.30310288071632385,
      "drift/rejected_abs": 0.12414111942052841,
      "epoch": 0.4490624626776544,
      "grad_norm": 208.0,
      "learning_rate": 3.262160747570027e-07,
      "logits/chosen": -2.1301331520080566,
      "logits/rejected": -2.198742628097534,
      "logps/chosen": -0.27701377868652344,
      "logps/rejected": -0.26015716791152954,
      "loss": 23.711040496826172,
      "loss/core": 23.664438247680664,
      "loss/preference_sft": 0.27701377868652344,
      "loss/reference_anchor": 0.5936552286148071,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.030449151992798,
      "rewards/margins": 1.86244797706604,
      "rewards/rejected": 1.168001413345337,
      "step": 470
    },
    {
      "drift/chosen_abs": 0.18387170135974884,
      "drift/rejected_abs": 0.15331368148326874,
      "epoch": 0.453839722918906,
      "grad_norm": 332.0,
      "learning_rate": 3.210038361759807e-07,
      "logits/chosen": -2.095447301864624,
      "logits/rejected": -2.033949613571167,
      "logps/chosen": -0.29470914602279663,
      "logps/rejected": -0.3144375681877136,
      "loss": 25.121068954467773,
      "loss/core": 25.099435806274414,
      "loss/preference_sft": 0.29470914602279663,
      "loss/reference_anchor": 0.2589879631996155,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.838440179824829,
      "rewards/margins": 0.307887464761734,
      "rewards/rejected": 1.530552625656128,
      "step": 475
    },
    {
      "drift/chosen_abs": 0.2358885258436203,
      "drift/rejected_abs": 0.10619956254959106,
      "epoch": 0.45861698316015764,
      "grad_norm": 70.0,
      "learning_rate": 3.1575780361449364e-07,
      "logits/chosen": -2.264855146408081,
      "logits/rejected": -2.3214104175567627,
      "logps/chosen": -0.2755188047885895,
      "logps/rejected": -0.27138248085975647,
      "loss": 24.032466888427734,
      "loss/core": 24.010181427001953,
      "loss/preference_sft": 0.2755188047885895,
      "loss/reference_anchor": 0.2696211636066437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3573050498962402,
      "rewards/margins": 1.3217544555664062,
      "rewards/rejected": 1.0355504751205444,
      "step": 480
    },
    {
      "drift/chosen_abs": 0.1745290607213974,
      "drift/rejected_abs": 0.08883767575025558,
      "epoch": 0.4633942434014093,
      "grad_norm": 74.0,
      "learning_rate": 3.104804738999169e-07,
      "logits/chosen": -2.205657482147217,
      "logits/rejected": -2.2127983570098877,
      "logps/chosen": -0.29136472940444946,
      "logps/rejected": -0.29586881399154663,
      "loss": 24.19198226928711,
      "loss/core": 24.183902740478516,
      "loss/preference_sft": 0.29136472940444946,
      "loss/reference_anchor": 0.07860977947711945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.74505615234375,
      "rewards/margins": 0.8590679168701172,
      "rewards/rejected": 0.8859883546829224,
      "step": 485
    },
    {
      "drift/chosen_abs": 0.20774760842323303,
      "drift/rejected_abs": 0.10662759840488434,
      "epoch": 0.4681715036426609,
      "grad_norm": 99.5,
      "learning_rate": 3.0517435875537536e-07,
      "logits/chosen": -2.1565749645233154,
      "logits/rejected": -2.198429822921753,
      "logps/chosen": -0.30721935629844666,
      "logps/rejected": -0.29325515031814575,
      "loss": 24.078283309936523,
      "loss/core": 24.06472396850586,
      "loss/preference_sft": 0.30721935629844666,
      "loss/reference_anchor": 0.15004561841487885,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.0768425464630127,
      "rewards/margins": 1.012283444404602,
      "rewards/rejected": 1.0645591020584106,
      "step": 490
    },
    {
      "drift/chosen_abs": 0.2635328471660614,
      "drift/rejected_abs": 0.08991514146327972,
      "epoch": 0.47294876388391255,
      "grad_norm": 133.0,
      "learning_rate": 2.998419836042993e-07,
      "logits/chosen": -2.2264866828918457,
      "logits/rejected": -2.3216514587402344,
      "logps/chosen": -0.29546287655830383,
      "logps/rejected": -0.2902601361274719,
      "loss": 23.78059959411621,
      "loss/core": 23.750764846801758,
      "loss/preference_sft": 0.29546287655830383,
      "loss/reference_anchor": 0.36827731132507324,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.6349475383758545,
      "rewards/margins": 1.7373361587524414,
      "rewards/rejected": 0.897611141204834,
      "step": 495
    },
    {
      "drift/chosen_abs": 0.4189828336238861,
      "drift/rejected_abs": 0.20560988783836365,
      "epoch": 0.47772602412516424,
      "grad_norm": 284.0,
      "learning_rate": 2.9448588636846043e-07,
      "logits/chosen": -2.023249387741089,
      "logits/rejected": -1.972783088684082,
      "logps/chosen": -0.2784698009490967,
      "logps/rejected": -0.2997981607913971,
      "loss": 24.244253158569336,
      "loss/core": 24.155033111572266,
      "loss/preference_sft": 0.2784698009490967,
      "loss/reference_anchor": 1.161747694015503,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.189828395843506,
      "rewards/margins": 2.1354339122772217,
      "rewards/rejected": 2.054394245147705,
      "step": 500
    },
    {
      "drift/chosen_abs": 0.3628165125846863,
      "drift/rejected_abs": 0.19208696484565735,
      "epoch": 0.4825032843664159,
      "grad_norm": 76.0,
      "learning_rate": 2.8910861626005773e-07,
      "logits/chosen": -2.1512835025787354,
      "logits/rejected": -2.1834206581115723,
      "logps/chosen": -0.2839333713054657,
      "logps/rejected": -0.278773695230484,
      "loss": 24.71078109741211,
      "loss/core": 24.65015983581543,
      "loss/preference_sft": 0.2839333713054657,
      "loss/reference_anchor": 0.7799011468887329,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.6272101402282715,
      "rewards/margins": 1.7839975357055664,
      "rewards/rejected": 1.8432128429412842,
      "step": 505
    },
    {
      "drift/chosen_abs": 0.2025258094072342,
      "drift/rejected_abs": 0.08022260665893555,
      "epoch": 0.4872805446076675,
      "grad_norm": 27.5,
      "learning_rate": 2.8371273256843074e-07,
      "logits/chosen": -2.1484246253967285,
      "logits/rejected": -2.282355546951294,
      "logps/chosen": -0.3147265315055847,
      "logps/rejected": -0.2912803888320923,
      "loss": 24.102039337158203,
      "loss/core": 24.08344078063965,
      "loss/preference_sft": 0.3147265315055847,
      "loss/reference_anchor": 0.21651633083820343,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.022645950317383,
      "rewards/margins": 1.2219388484954834,
      "rewards/rejected": 0.8007070422172546,
      "step": 510
    },
    {
      "drift/chosen_abs": 0.2555674910545349,
      "drift/rejected_abs": 0.16905184090137482,
      "epoch": 0.49205780484891914,
      "grad_norm": 129.0,
      "learning_rate": 2.783008034419767e-07,
      "logits/chosen": -2.032346487045288,
      "logits/rejected": -2.036229372024536,
      "logps/chosen": -0.2818189561367035,
      "logps/rejected": -0.30472418665885925,
      "loss": 24.25507164001465,
      "loss/core": 24.236268997192383,
      "loss/preference_sft": 0.2818189561367035,
      "loss/reference_anchor": 0.22251644730567932,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.553283214569092,
      "rewards/margins": 0.8822887539863586,
      "rewards/rejected": 1.670994758605957,
      "step": 515
    },
    {
      "drift/chosen_abs": 0.30994483828544617,
      "drift/rejected_abs": 0.09894011914730072,
      "epoch": 0.4968350650901708,
      "grad_norm": 111.0,
      "learning_rate": 2.7287540466585064e-07,
      "logits/chosen": -2.059782028198242,
      "logits/rejected": -2.228891372680664,
      "logps/chosen": -0.3199217915534973,
      "logps/rejected": -0.333048939704895,
      "loss": 23.464609146118164,
      "loss/core": 23.430858612060547,
      "loss/preference_sft": 0.3199217915534973,
      "loss/reference_anchor": 0.41807693243026733,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.0952601432800293,
      "rewards/margins": 2.12144136428833,
      "rewards/rejected": 0.9738188982009888,
      "step": 520
    },
    {
      "drift/chosen_abs": 0.43741607666015625,
      "drift/rejected_abs": 0.14899471402168274,
      "epoch": 0.5016123253314224,
      "grad_norm": 668.0,
      "learning_rate": 2.674391184360313e-07,
      "logits/chosen": -2.195627450942993,
      "logits/rejected": -2.1632399559020996,
      "logps/chosen": -0.2940196394920349,
      "logps/rejected": -0.2978762984275818,
      "loss": 23.785734176635742,
      "loss/core": 23.704212188720703,
      "loss/preference_sft": 0.2940196394920349,
      "loss/reference_anchor": 1.057555079460144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.371184349060059,
      "rewards/margins": 2.8844447135925293,
      "rewards/rejected": 1.4867396354675293,
      "step": 525
    },
    {
      "drift/chosen_abs": 0.3064503073692322,
      "drift/rejected_abs": 0.1510452926158905,
      "epoch": 0.5063895855726741,
      "grad_norm": 27.0,
      "learning_rate": 2.6199453213033593e-07,
      "logits/chosen": -2.0389158725738525,
      "logits/rejected": -2.002628803253174,
      "logps/chosen": -0.28475403785705566,
      "logps/rejected": -0.28787198662757874,
      "loss": 23.84844970703125,
      "loss/core": 23.816450119018555,
      "loss/preference_sft": 0.28475403785705566,
      "loss/reference_anchor": 0.3981722593307495,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.063196897506714,
      "rewards/margins": 1.5666277408599854,
      "rewards/rejected": 1.496569275856018,
      "step": 530
    },
    {
      "drift/chosen_abs": 0.3102155029773712,
      "drift/rejected_abs": 0.10850729793310165,
      "epoch": 0.5111668458139257,
      "grad_norm": 58.0,
      "learning_rate": 2.565442370769683e-07,
      "logits/chosen": -2.1870720386505127,
      "logits/rejected": -2.109102487564087,
      "logps/chosen": -0.2847766578197479,
      "logps/rejected": -0.27864956855773926,
      "loss": 23.705360412597656,
      "loss/core": 23.66876220703125,
      "loss/preference_sft": 0.2847766578197479,
      "loss/reference_anchor": 0.4595085680484772,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.1017348766326904,
      "rewards/margins": 2.018510580062866,
      "rewards/rejected": 1.0832241773605347,
      "step": 535
    },
    {
      "drift/chosen_abs": 0.15850523114204407,
      "drift/rejected_abs": 0.15349014103412628,
      "epoch": 0.5159441060551774,
      "grad_norm": 40.5,
      "learning_rate": 2.510908273211866e-07,
      "logits/chosen": -2.1092724800109863,
      "logits/rejected": -2.0874218940734863,
      "logps/chosen": -0.27907878160476685,
      "logps/rejected": -0.2851378917694092,
      "loss": 25.15981674194336,
      "loss/core": 25.145463943481445,
      "loss/preference_sft": 0.27907878160476685,
      "loss/reference_anchor": 0.163437157869339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.5850121974945068,
      "rewards/margins": 0.05249936506152153,
      "rewards/rejected": 1.532512903213501,
      "step": 540
    },
    {
      "drift/chosen_abs": 0.2638896703720093,
      "drift/rejected_abs": 0.09009301662445068,
      "epoch": 0.520721366296429,
      "grad_norm": 344.0,
      "learning_rate": 2.456368983906791e-07,
      "logits/chosen": -2.334498882293701,
      "logits/rejected": -2.1754422187805176,
      "logps/chosen": -0.28355711698532104,
      "logps/rejected": -0.2911638617515564,
      "loss": 23.94521141052246,
      "loss/core": 23.90779685974121,
      "loss/preference_sft": 0.28355711698532104,
      "loss/reference_anchor": 0.47052621841430664,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.637960433959961,
      "rewards/margins": 1.7409608364105225,
      "rewards/rejected": 0.8969996571540833,
      "step": 545
    },
    {
      "drift/chosen_abs": 0.2771431505680084,
      "drift/rejected_abs": 0.16234190762043,
      "epoch": 0.5254986265376806,
      "grad_norm": 94.0,
      "learning_rate": 2.401850460602329e-07,
      "logits/chosen": -1.901158332824707,
      "logits/rejected": -1.9195445775985718,
      "logps/chosen": -0.27585816383361816,
      "logps/rejected": -0.29788321256637573,
      "loss": 23.990930557250977,
      "loss/core": 23.959190368652344,
      "loss/preference_sft": 0.27585816383361816,
      "loss/reference_anchor": 0.3956103026866913,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.7710883617401123,
      "rewards/margins": 1.148720383644104,
      "rewards/rejected": 1.6223682165145874,
      "step": 550
    },
    {
      "drift/chosen_abs": 0.33259114623069763,
      "drift/rejected_abs": 0.10077937692403793,
      "epoch": 0.5302758867789323,
      "grad_norm": 55.5,
      "learning_rate": 2.3473786511628575e-07,
      "logits/chosen": -2.095702648162842,
      "logits/rejected": -2.1536788940429688,
      "logps/chosen": -0.308633029460907,
      "logps/rejected": -0.2737549841403961,
      "loss": 23.32325553894043,
      "loss/core": 23.289531707763672,
      "loss/preference_sft": 0.308633029460907,
      "loss/reference_anchor": 0.4187850058078766,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.3254246711730957,
      "rewards/margins": 2.317995548248291,
      "rewards/rejected": 1.0074293613433838,
      "step": 555
    },
    {
      "drift/chosen_abs": 0.2825181186199188,
      "drift/rejected_abs": 0.15401551127433777,
      "epoch": 0.5350531470201839,
      "grad_norm": 135.0,
      "learning_rate": 2.2929794812194894e-07,
      "logits/chosen": -2.1838390827178955,
      "logits/rejected": -2.171027660369873,
      "logps/chosen": -0.2748723030090332,
      "logps/rejected": -0.2799665927886963,
      "loss": 23.88632583618164,
      "loss/core": 23.858224868774414,
      "loss/preference_sft": 0.2748723030090332,
      "loss/reference_anchor": 0.3471968472003937,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.8238039016723633,
      "rewards/margins": 1.2850908041000366,
      "rewards/rejected": 1.5387130975723267,
      "step": 560
    },
    {
      "drift/chosen_abs": 0.280038982629776,
      "drift/rejected_abs": 0.16534456610679626,
      "epoch": 0.5398304072614356,
      "grad_norm": 140.0,
      "learning_rate": 2.2386788418308665e-07,
      "logits/chosen": -2.077315092086792,
      "logits/rejected": -2.1030631065368652,
      "logps/chosen": -0.28564533591270447,
      "logps/rejected": -0.3059332072734833,
      "loss": 24.126893997192383,
      "loss/core": 24.075992584228516,
      "loss/preference_sft": 0.28564533591270447,
      "loss/reference_anchor": 0.6501015424728394,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.7995214462280273,
      "rewards/margins": 1.1462886333465576,
      "rewards/rejected": 1.6532329320907593,
      "step": 565
    },
    {
      "drift/chosen_abs": 0.3362275958061218,
      "drift/rejected_abs": 0.1704963594675064,
      "epoch": 0.5446076675026872,
      "grad_norm": 316.0,
      "learning_rate": 2.184502577160426e-07,
      "logits/chosen": -2.07682466506958,
      "logits/rejected": -2.1273488998413086,
      "logps/chosen": -0.286944180727005,
      "logps/rejected": -0.2869289517402649,
      "loss": 23.644794464111328,
      "loss/core": 23.60843276977539,
      "loss/preference_sft": 0.286944180727005,
      "loss/reference_anchor": 0.4561174809932709,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.3612422943115234,
      "rewards/margins": 1.6572866439819336,
      "rewards/rejected": 1.703955888748169,
      "step": 570
    },
    {
      "drift/chosen_abs": 0.18439261615276337,
      "drift/rejected_abs": 0.0657983049750328,
      "epoch": 0.5493849277439389,
      "grad_norm": 43.0,
      "learning_rate": 2.1304764721759732e-07,
      "logits/chosen": -2.1571121215820312,
      "logits/rejected": -2.188605546951294,
      "logps/chosen": -0.3006740212440491,
      "logps/rejected": -0.32234296202659607,
      "loss": 24.048736572265625,
      "loss/core": 24.03409194946289,
      "loss/preference_sft": 0.3006740212440491,
      "loss/reference_anchor": 0.1651538759469986,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8426148891448975,
      "rewards/margins": 1.273414134979248,
      "rewards/rejected": 0.569200873374939,
      "step": 575
    },
    {
      "drift/chosen_abs": 0.14222335815429688,
      "drift/rejected_abs": 0.09565664827823639,
      "epoch": 0.5541621879851905,
      "grad_norm": 30.25,
      "learning_rate": 2.0766262403774385e-07,
      "logits/chosen": -2.4316046237945557,
      "logits/rejected": -2.4871370792388916,
      "logps/chosen": -0.26630157232284546,
      "logps/rejected": -0.2813344895839691,
      "loss": 24.60654067993164,
      "loss/core": 24.598276138305664,
      "loss/preference_sft": 0.26630157232284546,
      "loss/reference_anchor": 0.08357779681682587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.4222335815429688,
      "rewards/margins": 0.46712565422058105,
      "rewards/rejected": 0.9551078677177429,
      "step": 580
    },
    {
      "drift/chosen_abs": 0.22317564487457275,
      "drift/rejected_abs": 0.13778264820575714,
      "epoch": 0.5589394482264421,
      "grad_norm": 68.0,
      "learning_rate": 2.0229775115586378e-07,
      "logits/chosen": -2.1374573707580566,
      "logits/rejected": -2.1010804176330566,
      "logps/chosen": -0.26939648389816284,
      "logps/rejected": -0.2822672426700592,
      "loss": 24.302589416503906,
      "loss/core": 24.24777603149414,
      "loss/preference_sft": 0.26939648389816284,
      "loss/reference_anchor": 0.7039062976837158,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.230914354324341,
      "rewards/margins": 0.8574030995368958,
      "rewards/rejected": 1.3735109567642212,
      "step": 585
    },
    {
      "drift/chosen_abs": 0.3644371032714844,
      "drift/rejected_abs": 0.104056216776371,
      "epoch": 0.5637167084676937,
      "grad_norm": 51.0,
      "learning_rate": 1.9695558196088844e-07,
      "logits/chosen": -2.0276551246643066,
      "logits/rejected": -2.1235756874084473,
      "logps/chosen": -0.2852238714694977,
      "logps/rejected": -0.2887842059135437,
      "loss": 23.151445388793945,
      "loss/core": 23.115150451660156,
      "loss/preference_sft": 0.2852238714694977,
      "loss/reference_anchor": 0.45538654923439026,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.6441140174865723,
      "rewards/margins": 2.6299378871917725,
      "rewards/rejected": 1.0141762495040894,
      "step": 590
    },
    {
      "drift/chosen_abs": 0.1715865433216095,
      "drift/rejected_abs": 0.09020796418190002,
      "epoch": 0.5684939687089454,
      "grad_norm": 69.5,
      "learning_rate": 1.9163865903602372e-07,
      "logits/chosen": -2.0477294921875,
      "logits/rejected": -2.0416600704193115,
      "logps/chosen": -0.2748182415962219,
      "logps/rejected": -0.27176275849342346,
      "loss": 24.21047592163086,
      "loss/core": 24.204145431518555,
      "loss/preference_sft": 0.2748182415962219,
      "loss/reference_anchor": 0.05691518262028694,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7147114276885986,
      "rewards/margins": 0.8242376446723938,
      "rewards/rejected": 0.8904738426208496,
      "step": 595
    },
    {
      "drift/chosen_abs": 0.18755662441253662,
      "drift/rejected_abs": 0.10950811952352524,
      "epoch": 0.5732712289501971,
      "grad_norm": 103.0,
      "learning_rate": 1.8634951294861806e-07,
      "logits/chosen": -2.2788078784942627,
      "logits/rejected": -2.2681713104248047,
      "logps/chosen": -0.266190767288208,
      "logps/rejected": -0.2825302183628082,
      "loss": 24.342437744140625,
      "loss/core": 24.329063415527344,
      "loss/preference_sft": 0.266190767288208,
      "loss/reference_anchor": 0.15168902277946472,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8744417428970337,
      "rewards/margins": 0.7936787009239197,
      "rewards/rejected": 1.0807631015777588,
      "step": 600
    },
    {
      "drift/chosen_abs": 0.42088302969932556,
      "drift/rejected_abs": 0.1206652969121933,
      "epoch": 0.5780484891914487,
      "grad_norm": 122.5,
      "learning_rate": 1.810906610457502e-07,
      "logits/chosen": -2.1177077293395996,
      "logits/rejected": -2.1916587352752686,
      "logps/chosen": -0.2778300940990448,
      "logps/rejected": -0.275773286819458,
      "loss": 23.372730255126953,
      "loss/core": 23.302967071533203,
      "loss/preference_sft": 0.2778300940990448,
      "loss/reference_anchor": 0.9023693203926086,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 4.206851482391357,
      "rewards/margins": 3.001718759536743,
      "rewards/rejected": 1.2051331996917725,
      "step": 605
    },
    {
      "drift/chosen_abs": 0.4304037094116211,
      "drift/rejected_abs": 0.19502094388008118,
      "epoch": 0.5828257494327004,
      "grad_norm": 145.0,
      "learning_rate": 1.7586460625610726e-07,
      "logits/chosen": -2.118242025375366,
      "logits/rejected": -2.1343255043029785,
      "logps/chosen": -0.27321526408195496,
      "logps/rejected": -0.27437523007392883,
      "loss": 23.397014617919922,
      "loss/core": 23.323720932006836,
      "loss/preference_sft": 0.27321526408195496,
      "loss/reference_anchor": 0.9498865008354187,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.302402973175049,
      "rewards/margins": 2.3571293354034424,
      "rewards/rejected": 1.945273995399475,
      "step": 610
    },
    {
      "drift/chosen_abs": 0.27419549226760864,
      "drift/rejected_abs": 0.1494186818599701,
      "epoch": 0.587603009673952,
      "grad_norm": 69.0,
      "learning_rate": 1.70673835898727e-07,
      "logits/chosen": -1.967576026916504,
      "logits/rejected": -2.034571409225464,
      "logps/chosen": -0.3140396773815155,
      "logps/rejected": -0.3035440444946289,
      "loss": 23.98012351989746,
      "loss/core": 23.95610237121582,
      "loss/preference_sft": 0.3140396773815155,
      "loss/reference_anchor": 0.28889569640159607,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.739633321762085,
      "rewards/margins": 1.2492014169692993,
      "rewards/rejected": 1.4904316663742065,
      "step": 615
    },
    {
      "drift/chosen_abs": 0.16910451650619507,
      "drift/rejected_abs": 0.14680986106395721,
      "epoch": 0.5923802699152036,
      "grad_norm": 61.25,
      "learning_rate": 1.6552082049916824e-07,
      "logits/chosen": -2.143625259399414,
      "logits/rejected": -2.1699137687683105,
      "logps/chosen": -0.286045640707016,
      "logps/rejected": -0.2911421060562134,
      "loss": 25.047000885009766,
      "loss/core": 25.030277252197266,
      "loss/preference_sft": 0.286045640707016,
      "loss/reference_anchor": 0.19438524544239044,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.690314531326294,
      "rewards/margins": 0.22506046295166016,
      "rewards/rejected": 1.4652539491653442,
      "step": 620
    },
    {
      "drift/chosen_abs": 0.388736367225647,
      "drift/rejected_abs": 0.1826535165309906,
      "epoch": 0.5971575301564552,
      "grad_norm": 95.5,
      "learning_rate": 1.6040801261367493e-07,
      "logits/chosen": -1.9324958324432373,
      "logits/rejected": -1.961570143699646,
      "logps/chosen": -0.304928719997406,
      "logps/rejected": -0.33531856536865234,
      "loss": 24.033096313476562,
      "loss/core": 23.950403213500977,
      "loss/preference_sft": 0.304928719997406,
      "loss/reference_anchor": 1.0720922946929932,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.885380268096924,
      "rewards/margins": 2.3656325340270996,
      "rewards/rejected": 1.5197473764419556,
      "step": 625
    },
    {
      "drift/chosen_abs": 0.29494503140449524,
      "drift/rejected_abs": 0.12427574396133423,
      "epoch": 0.6019347903977069,
      "grad_norm": 57.0,
      "learning_rate": 1.5533784566189175e-07,
      "logits/chosen": -1.9347823858261108,
      "logits/rejected": -2.0498507022857666,
      "logps/chosen": -0.27183371782302856,
      "logps/rejected": -0.2836715281009674,
      "loss": 23.67662239074707,
      "loss/core": 23.64113998413086,
      "loss/preference_sft": 0.27183371782302856,
      "loss/reference_anchor": 0.4459313452243805,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.949350595474243,
      "rewards/margins": 1.7082860469818115,
      "rewards/rejected": 1.2410645484924316,
      "step": 630
    },
    {
      "drift/chosen_abs": 0.23291683197021484,
      "drift/rejected_abs": 0.10206238925457001,
      "epoch": 0.6067120506389586,
      "grad_norm": 222.0,
      "learning_rate": 1.5031273276868845e-07,
      "logits/chosen": -1.984095573425293,
      "logits/rejected": -2.0696752071380615,
      "logps/chosen": -0.3027849495410919,
      "logps/rejected": -0.2994707524776459,
      "loss": 23.96482276916504,
      "loss/core": 23.94552230834961,
      "loss/preference_sft": 0.3027849495410919,
      "loss/reference_anchor": 0.2270624190568924,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.327078342437744,
      "rewards/margins": 1.3064545392990112,
      "rewards/rejected": 1.0206239223480225,
      "step": 635
    },
    {
      "drift/chosen_abs": 0.1955791413784027,
      "drift/rejected_abs": 0.08006999641656876,
      "epoch": 0.6114893108802102,
      "grad_norm": 49.5,
      "learning_rate": 1.4533506561564305e-07,
      "logits/chosen": -2.2103934288024902,
      "logits/rejected": -2.2860031127929688,
      "logps/chosen": -0.3017186224460602,
      "logps/rejected": -0.2985337972640991,
      "loss": 24.067489624023438,
      "loss/core": 24.050811767578125,
      "loss/preference_sft": 0.3017186224460602,
      "loss/reference_anchor": 0.19217993319034576,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9552767276763916,
      "rewards/margins": 1.1555898189544678,
      "rewards/rejected": 0.7996870279312134,
      "step": 640
    },
    {
      "drift/chosen_abs": 0.16859070956707,
      "drift/rejected_abs": 0.12346317619085312,
      "epoch": 0.6162665711214619,
      "grad_norm": 117.5,
      "learning_rate": 1.404072133027306e-07,
      "logits/chosen": -2.1554760932922363,
      "logits/rejected": -2.13128662109375,
      "logps/chosen": -0.2895345687866211,
      "logps/rejected": -0.2951858341693878,
      "loss": 24.680572509765625,
      "loss/core": 24.669631958007812,
      "loss/preference_sft": 0.2895345687866211,
      "loss/reference_anchor": 0.11693717539310455,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.6848987340927124,
      "rewards/margins": 0.4532623291015625,
      "rewards/rejected": 1.2316365242004395,
      "step": 645
    },
    {
      "drift/chosen_abs": 0.13376560807228088,
      "drift/rejected_abs": 0.08641010522842407,
      "epoch": 0.6210438313627135,
      "grad_norm": 66.5,
      "learning_rate": 1.3553152122076078e-07,
      "logits/chosen": -2.2198781967163086,
      "logits/rejected": -2.170907974243164,
      "logps/chosen": -0.28950920701026917,
      "logps/rejected": -0.2986505925655365,
      "loss": 24.61799430847168,
      "loss/core": 24.610118865966797,
      "loss/preference_sft": 0.28950920701026917,
      "loss/reference_anchor": 0.07604795694351196,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.3332895040512085,
      "rewards/margins": 0.4888167381286621,
      "rewards/rejected": 0.8444727063179016,
      "step": 650
    },
    {
      "drift/chosen_abs": 0.2010502815246582,
      "drift/rejected_abs": 0.0992017537355423,
      "epoch": 0.6258210916039652,
      "grad_norm": 304.0,
      "learning_rate": 1.3071030993509787e-07,
      "logits/chosen": -2.059145450592041,
      "logits/rejected": -2.1219570636749268,
      "logps/chosen": -0.2817350924015045,
      "logps/rejected": -0.28943654894828796,
      "loss": 24.06000328063965,
      "loss/core": 24.04839515686035,
      "loss/preference_sft": 0.2817350924015045,
      "loss/reference_anchor": 0.12662085890769958,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.009782314300537,
      "rewards/margins": 1.0185507535934448,
      "rewards/rejected": 0.9912316203117371,
      "step": 655
    },
    {
      "drift/chosen_abs": 0.2789165675640106,
      "drift/rejected_abs": 0.1331566572189331,
      "epoch": 0.6305983518452167,
      "grad_norm": 45.75,
      "learning_rate": 1.2594587408119804e-07,
      "logits/chosen": -2.0298891067504883,
      "logits/rejected": -2.0245847702026367,
      "logps/chosen": -0.29279690980911255,
      "logps/rejected": -0.30883678793907166,
      "loss": 24.24443244934082,
      "loss/core": 24.194019317626953,
      "loss/preference_sft": 0.29279690980911255,
      "loss/reference_anchor": 0.6429013609886169,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.7881667613983154,
      "rewards/margins": 1.5086616277694702,
      "rewards/rejected": 1.2795050144195557,
      "step": 660
    },
    {
      "drift/chosen_abs": 0.23610401153564453,
      "drift/rejected_abs": 0.1503429114818573,
      "epoch": 0.6353756120864684,
      "grad_norm": 123.5,
      "learning_rate": 1.2124048127248644e-07,
      "logits/chosen": -2.188072681427002,
      "logits/rejected": -2.164827823638916,
      "logps/chosen": -0.28306499123573303,
      "logps/rejected": -0.27927109599113464,
      "loss": 24.441707611083984,
      "loss/core": 24.4201717376709,
      "loss/preference_sft": 0.28306499123573303,
      "loss/reference_anchor": 0.25882014632225037,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3602824211120605,
      "rewards/margins": 0.8587505221366882,
      "rewards/rejected": 1.5015321969985962,
      "step": 665
    },
    {
      "drift/chosen_abs": 0.3666432499885559,
      "drift/rejected_abs": 0.1319333016872406,
      "epoch": 0.64015287232772,
      "grad_norm": 44.0,
      "learning_rate": 1.1659637102109712e-07,
      "logits/chosen": -1.9824602603912354,
      "logits/rejected": -2.0293171405792236,
      "logps/chosen": -0.2721920907497406,
      "logps/rejected": -0.31627121567726135,
      "loss": 23.281818389892578,
      "loss/core": 23.24245834350586,
      "loss/preference_sft": 0.2721920907497406,
      "loss/reference_anchor": 0.49754709005355835,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.666236162185669,
      "rewards/margins": 2.3700363636016846,
      "rewards/rejected": 1.2962002754211426,
      "step": 670
    },
    {
      "drift/chosen_abs": 0.162507563829422,
      "drift/rejected_abs": 0.09013229608535767,
      "epoch": 0.6449301325689717,
      "grad_norm": 93.0,
      "learning_rate": 1.1201575367198546e-07,
      "logits/chosen": -2.038818359375,
      "logits/rejected": -2.0799050331115723,
      "logps/chosen": -0.28820866346359253,
      "logps/rejected": -0.28691327571868896,
      "loss": 24.332555770874023,
      "loss/core": 24.325624465942383,
      "loss/preference_sft": 0.28820866346359253,
      "loss/reference_anchor": 0.06361403316259384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6243938207626343,
      "rewards/margins": 0.730514645576477,
      "rewards/rejected": 0.893879234790802,
      "step": 675
    },
    {
      "drift/chosen_abs": 0.2139333039522171,
      "drift/rejected_abs": 0.13963082432746887,
      "epoch": 0.6497073928102234,
      "grad_norm": 43.75,
      "learning_rate": 1.0750080935092423e-07,
      "logits/chosen": -2.1613402366638184,
      "logits/rejected": -2.079357385635376,
      "logps/chosen": -0.28284314274787903,
      "logps/rejected": -0.2906249463558197,
      "loss": 24.423181533813477,
      "loss/core": 24.4051456451416,
      "loss/preference_sft": 0.28284314274787903,
      "loss/reference_anchor": 0.2122153490781784,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1380627155303955,
      "rewards/margins": 0.7430015802383423,
      "rewards/rejected": 1.3950612545013428,
      "step": 680
    },
    {
      "drift/chosen_abs": 0.28872284293174744,
      "drift/rejected_abs": 0.15741479396820068,
      "epoch": 0.654484653051475,
      "grad_norm": 57.25,
      "learning_rate": 1.0305368692688174e-07,
      "logits/chosen": -2.088265895843506,
      "logits/rejected": -2.072359561920166,
      "logps/chosen": -0.253498375415802,
      "logps/rejected": -0.2732449471950531,
      "loss": 24.38766098022461,
      "loss/core": 24.349506378173828,
      "loss/preference_sft": 0.253498375415802,
      "loss/reference_anchor": 0.4833889901638031,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.8872287273406982,
      "rewards/margins": 1.3145825862884521,
      "rewards/rejected": 1.5726460218429565,
      "step": 685
    },
    {
      "drift/chosen_abs": 0.17745468020439148,
      "drift/rejected_abs": 0.1100490465760231,
      "epoch": 0.6592619132927267,
      "grad_norm": 51.5,
      "learning_rate": 9.867650298927643e-08,
      "logits/chosen": -1.8866245746612549,
      "logits/rejected": -1.9085317850112915,
      "logps/chosen": -0.307719886302948,
      "logps/rejected": -0.32763782143592834,
      "loss": 24.435400009155273,
      "loss/core": 24.425445556640625,
      "loss/preference_sft": 0.307719886302948,
      "loss/reference_anchor": 0.10196495056152344,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7728378772735596,
      "rewards/margins": 0.6770357489585876,
      "rewards/rejected": 1.0958020687103271,
      "step": 690
    },
    {
      "drift/chosen_abs": 0.1973785161972046,
      "drift/rejected_abs": 0.06951657682657242,
      "epoch": 0.6640391735339782,
      "grad_norm": 65.5,
      "learning_rate": 9.437134084059515e-08,
      "logits/chosen": -2.272217273712158,
      "logits/rejected": -2.280867576599121,
      "logps/chosen": -0.3060583770275116,
      "logps/rejected": -0.29238781332969666,
      "loss": 24.02126121520996,
      "loss/core": 24.004791259765625,
      "loss/preference_sft": 0.3060583770275116,
      "loss/reference_anchor": 0.18900753557682037,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.972781777381897,
      "rewards/margins": 1.285192847251892,
      "rewards/rejected": 0.6875890493392944,
      "step": 695
    },
    {
      "drift/chosen_abs": 0.22317072749137878,
      "drift/rejected_abs": 0.16237227618694305,
      "epoch": 0.6688164337752299,
      "grad_norm": 1552.0,
      "learning_rate": 9.014024950485383e-08,
      "logits/chosen": -2.1077773571014404,
      "logits/rejected": -2.214353561401367,
      "logps/chosen": -0.29673534631729126,
      "logps/rejected": -0.3178013861179352,
      "loss": 24.56673240661621,
      "loss/core": 24.545368194580078,
      "loss/preference_sft": 0.29673534631729126,
      "loss/reference_anchor": 0.25515273213386536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.231229305267334,
      "rewards/margins": 0.6277844309806824,
      "rewards/rejected": 1.603445053100586,
      "step": 700
    },
    {
      "drift/chosen_abs": 0.341328501701355,
      "drift/rejected_abs": 0.15423105657100677,
      "epoch": 0.6735936940164815,
      "grad_norm": 132.0,
      "learning_rate": 8.598524275237321e-08,
      "logits/chosen": -2.1965928077697754,
      "logits/rejected": -2.154327630996704,
      "logps/chosen": -0.27952122688293457,
      "logps/rejected": -0.2664346396923065,
      "loss": 23.59442138671875,
      "loss/core": 23.561824798583984,
      "loss/preference_sft": 0.27952122688293457,
      "loss/reference_anchor": 0.40662726759910583,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.412951946258545,
      "rewards/margins": 1.871299386024475,
      "rewards/rejected": 1.5416524410247803,
      "step": 705
    },
    {
      "drift/chosen_abs": 0.3609875440597534,
      "drift/rejected_abs": 0.19022467732429504,
      "epoch": 0.6783709542577332,
      "grad_norm": 576.0,
      "learning_rate": 8.190829814133293e-08,
      "logits/chosen": -1.8862717151641846,
      "logits/rejected": -1.87451171875,
      "logps/chosen": -0.2760660648345947,
      "logps/rejected": -0.2974444329738617,
      "loss": 23.68022346496582,
      "loss/core": 23.627628326416016,
      "loss/preference_sft": 0.2760660648345947,
      "loss/reference_anchor": 0.6736692786216736,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.608205795288086,
      "rewards/margins": 1.707396149635315,
      "rewards/rejected": 1.90080988407135,
      "step": 710
    },
    {
      "drift/chosen_abs": 0.18668507039546967,
      "drift/rejected_abs": 0.07842984795570374,
      "epoch": 0.6831482144989849,
      "grad_norm": 57.25,
      "learning_rate": 7.791135607656146e-08,
      "logits/chosen": -2.2538669109344482,
      "logits/rejected": -2.2352232933044434,
      "logps/chosen": -0.28401559591293335,
      "logps/rejected": -0.3294680714607239,
      "loss": 24.112934112548828,
      "loss/core": 24.091781616210938,
      "loss/preference_sft": 0.28401559591293335,
      "loss/reference_anchor": 0.25365859270095825,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8665748834609985,
      "rewards/margins": 1.0951749086380005,
      "rewards/rejected": 0.771399974822998,
      "step": 715
    },
    {
      "drift/chosen_abs": 0.21051010489463806,
      "drift/rejected_abs": 0.1500566005706787,
      "epoch": 0.6879254747402365,
      "grad_norm": 136.0,
      "learning_rate": 7.399631888600796e-08,
      "logits/chosen": -1.9951555728912354,
      "logits/rejected": -2.00339674949646,
      "logps/chosen": -0.2962464392185211,
      "logps/rejected": -0.30031222105026245,
      "loss": 24.666812896728516,
      "loss/core": 24.645822525024414,
      "loss/preference_sft": 0.2962464392185211,
      "loss/reference_anchor": 0.25020235776901245,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.1030123233795166,
      "rewards/margins": 0.6037241816520691,
      "rewards/rejected": 1.4992883205413818,
      "step": 720
    },
    {
      "drift/chosen_abs": 0.15870442986488342,
      "drift/rejected_abs": 0.07348223030567169,
      "epoch": 0.6927027349814882,
      "grad_norm": 66.5,
      "learning_rate": 7.016504991533726e-08,
      "logits/chosen": -2.1241562366485596,
      "logits/rejected": -2.2400271892547607,
      "logps/chosen": -0.28562217950820923,
      "logps/rejected": -0.2771227955818176,
      "loss": 24.232282638549805,
      "loss/core": 24.223468780517578,
      "loss/preference_sft": 0.28562217950820923,
      "loss/reference_anchor": 0.08894944936037064,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.5855097770690918,
      "rewards/margins": 0.8525517582893372,
      "rewards/rejected": 0.7329580783843994,
      "step": 725
    },
    {
      "drift/chosen_abs": 0.102408766746521,
      "drift/rejected_abs": 0.08807560056447983,
      "epoch": 0.6974799952227397,
      "grad_norm": 956.0,
      "learning_rate": 6.641937264107867e-08,
      "logits/chosen": -2.2225677967071533,
      "logits/rejected": -2.271383285522461,
      "logps/chosen": -0.28616029024124146,
      "logps/rejected": -0.293459951877594,
      "loss": 24.99797821044922,
      "loss/core": 24.98850440979004,
      "loss/preference_sft": 0.28616029024124146,
      "loss/reference_anchor": 0.09770476073026657,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.022210717201233,
      "rewards/margins": 0.14174190163612366,
      "rewards/rejected": 0.8804686665534973,
      "step": 730
    },
    {
      "drift/chosen_abs": 0.1421697437763214,
      "drift/rejected_abs": 0.06707651913166046,
      "epoch": 0.7022572554639914,
      "grad_norm": 119.0,
      "learning_rate": 6.276106980274944e-08,
      "logits/chosen": -2.0496909618377686,
      "logits/rejected": -2.097294330596924,
      "logps/chosen": -0.31965214014053345,
      "logps/rejected": -0.31716209650039673,
      "loss": 24.297515869140625,
      "loss/core": 24.289684295654297,
      "loss/preference_sft": 0.31965214014053345,
      "loss/reference_anchor": 0.07245230674743652,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.4212677478790283,
      "rewards/margins": 0.7517526149749756,
      "rewards/rejected": 0.6695151329040527,
      "step": 735
    },
    {
      "drift/chosen_abs": 0.2564278841018677,
      "drift/rejected_abs": 0.18303130567073822,
      "epoch": 0.707034515705243,
      "grad_norm": 71.0,
      "learning_rate": 5.919188255436777e-08,
      "logits/chosen": -2.1012163162231445,
      "logits/rejected": -2.1193010807037354,
      "logps/chosen": -0.3130371868610382,
      "logps/rejected": -0.3160560727119446,
      "loss": 25.59921646118164,
      "loss/core": 25.540393829345703,
      "loss/preference_sft": 0.3130371868610382,
      "loss/reference_anchor": 0.7530360221862793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.564279079437256,
      "rewards/margins": 0.7340623736381531,
      "rewards/rejected": 1.830216646194458,
      "step": 740
    },
    {
      "drift/chosen_abs": 0.3141566514968872,
      "drift/rejected_abs": 0.15219292044639587,
      "epoch": 0.7118117759464947,
      "grad_norm": 126.0,
      "learning_rate": 5.571350963575727e-08,
      "logits/chosen": -2.0531277656555176,
      "logits/rejected": -2.09647798538208,
      "logps/chosen": -0.2884739339351654,
      "logps/rejected": -0.2817172408103943,
      "loss": 23.735965728759766,
      "loss/core": 23.71042823791504,
      "loss/preference_sft": 0.2884739339351654,
      "loss/reference_anchor": 0.3116436302661896,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.141024351119995,
      "rewards/margins": 1.6229345798492432,
      "rewards/rejected": 1.5180898904800415,
      "step": 745
    },
    {
      "drift/chosen_abs": 0.12072908878326416,
      "drift/rejected_abs": 0.12682561576366425,
      "epoch": 0.7165890361877463,
      "grad_norm": 340.0,
      "learning_rate": 5.232760656403923e-08,
      "logits/chosen": -2.1889564990997314,
      "logits/rejected": -2.1800601482391357,
      "logps/chosen": -0.3116965889930725,
      "logps/rejected": -0.3028327226638794,
      "loss": 25.507654190063477,
      "loss/core": 25.483829498291016,
      "loss/preference_sft": 0.3116965889930725,
      "loss/reference_anchor": 0.28652527928352356,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.2060551643371582,
      "rewards/margins": -0.05827074497938156,
      "rewards/rejected": 1.2643258571624756,
      "step": 750
    },
    {
      "drift/chosen_abs": 0.24006719887256622,
      "drift/rejected_abs": 0.17291317880153656,
      "epoch": 0.721366296428998,
      "grad_norm": 133.0,
      "learning_rate": 4.9035784845695675e-08,
      "logits/chosen": -1.9279276132583618,
      "logits/rejected": -1.8381156921386719,
      "logps/chosen": -0.3074774742126465,
      "logps/rejected": -0.29301613569259644,
      "loss": 24.52367401123047,
      "loss/core": 24.506526947021484,
      "loss/preference_sft": 0.3074774742126465,
      "loss/reference_anchor": 0.19787776470184326,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.3972103595733643,
      "rewards/margins": 0.6746050715446472,
      "rewards/rejected": 1.7226054668426514,
      "step": 755
    },
    {
      "drift/chosen_abs": 0.19889570772647858,
      "drift/rejected_abs": 0.08718482404947281,
      "epoch": 0.7261435566702497,
      "grad_norm": 64.0,
      "learning_rate": 4.583961120958027e-08,
      "logits/chosen": -2.186131715774536,
      "logits/rejected": -2.203399419784546,
      "logps/chosen": -0.30299732089042664,
      "logps/rejected": -0.29353067278862,
      "loss": 24.01272964477539,
      "loss/core": 24.000045776367188,
      "loss/preference_sft": 0.30299732089042664,
      "loss/reference_anchor": 0.13879242539405823,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9878475666046143,
      "rewards/margins": 1.1234432458877563,
      "rewards/rejected": 0.8644043803215027,
      "step": 760
    },
    {
      "drift/chosen_abs": 0.1797488033771515,
      "drift/rejected_abs": 0.1066058874130249,
      "epoch": 0.7309208169115012,
      "grad_norm": 106.5,
      "learning_rate": 4.2740606861239594e-08,
      "logits/chosen": -2.08768892288208,
      "logits/rejected": -2.158001184463501,
      "logps/chosen": -0.275774210691452,
      "logps/rejected": -0.27186232805252075,
      "loss": 24.34197425842285,
      "loss/core": 24.330942153930664,
      "loss/preference_sft": 0.275774210691452,
      "loss/reference_anchor": 0.11952831596136093,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7971556186676025,
      "rewards/margins": 0.7334049344062805,
      "rewards/rejected": 1.0637508630752563,
      "step": 765
    },
    {
      "drift/chosen_abs": 0.3747478127479553,
      "drift/rejected_abs": 0.2356497049331665,
      "epoch": 0.7356980771527529,
      "grad_norm": 222.0,
      "learning_rate": 3.974024675890189e-08,
      "logits/chosen": -1.9823691844940186,
      "logits/rejected": -1.9497228860855103,
      "logps/chosen": -0.3086339831352234,
      "logps/rejected": -0.28454282879829407,
      "loss": 24.525297164916992,
      "loss/core": 24.43834114074707,
      "loss/preference_sft": 0.3086339831352234,
      "loss/reference_anchor": 1.1285443305969238,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.744788408279419,
      "rewards/margins": 1.3908659219741821,
      "rewards/rejected": 2.3539223670959473,
      "step": 770
    },
    {
      "drift/chosen_abs": 0.15283748507499695,
      "drift/rejected_abs": 0.09471901506185532,
      "epoch": 0.7404753373940045,
      "grad_norm": 32.75,
      "learning_rate": 3.683995891147695e-08,
      "logits/chosen": -2.0964419841766357,
      "logits/rejected": -2.1264514923095703,
      "logps/chosen": -0.30610623955726624,
      "logps/rejected": -0.29977482557296753,
      "loss": 24.4835147857666,
      "loss/core": 24.475711822509766,
      "loss/preference_sft": 0.30610623955726624,
      "loss/reference_anchor": 0.0734710842370987,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5278440713882446,
      "rewards/margins": 0.5806539058685303,
      "rewards/rejected": 0.9471901059150696,
      "step": 775
    },
    {
      "drift/chosen_abs": 0.1896996945142746,
      "drift/rejected_abs": 0.13039933145046234,
      "epoch": 0.7452525976352562,
      "grad_norm": 102.5,
      "learning_rate": 3.404112369890108e-08,
      "logits/chosen": -2.1795246601104736,
      "logits/rejected": -2.1940202713012695,
      "logps/chosen": -0.3016170859336853,
      "logps/rejected": -0.30035489797592163,
      "loss": 24.48188018798828,
      "loss/core": 24.470382690429688,
      "loss/preference_sft": 0.3016170859336853,
      "loss/reference_anchor": 0.1231255754828453,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8962314128875732,
      "rewards/margins": 0.5943207144737244,
      "rewards/rejected": 1.3019107580184937,
      "step": 780
    },
    {
      "drift/chosen_abs": 0.3444829285144806,
      "drift/rejected_abs": 0.13935281336307526,
      "epoch": 0.7500298578765078,
      "grad_norm": 42.5,
      "learning_rate": 3.134507321515106e-08,
      "logits/chosen": -1.9970579147338867,
      "logits/rejected": -1.953643560409546,
      "logps/chosen": -0.30979782342910767,
      "logps/rejected": -0.2892361879348755,
      "loss": 24.037694931030273,
      "loss/core": 23.987783432006836,
      "loss/preference_sft": 0.30979782342910767,
      "loss/reference_anchor": 0.6345094442367554,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.4429678916931152,
      "rewards/margins": 2.0524790287017822,
      "rewards/rejected": 1.390488862991333,
      "step": 785
    },
    {
      "drift/chosen_abs": 0.26237040758132935,
      "drift/rejected_abs": 0.19609911739826202,
      "epoch": 0.7548071181177595,
      "grad_norm": 780.0,
      "learning_rate": 2.875309063423956e-08,
      "logits/chosen": -2.1686906814575195,
      "logits/rejected": -1.9951951503753662,
      "logps/chosen": -0.28583893179893494,
      "logps/rejected": -0.315361887216568,
      "loss": 24.721479415893555,
      "loss/core": 24.67513656616211,
      "loss/preference_sft": 0.28583893179893494,
      "loss/reference_anchor": 0.5893004536628723,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6218254566192627,
      "rewards/margins": 0.6629058122634888,
      "rewards/rejected": 1.9589197635650635,
      "step": 790
    },
    {
      "drift/chosen_abs": 0.24005870521068573,
      "drift/rejected_abs": 0.11813025176525116,
      "epoch": 0.7595843783590112,
      "grad_norm": 140.0,
      "learning_rate": 2.626640959949375e-08,
      "logits/chosen": -1.971401572227478,
      "logits/rejected": -1.991015076637268,
      "logps/chosen": -0.30549412965774536,
      "logps/rejected": -0.296720415353775,
      "loss": 24.04024887084961,
      "loss/core": 23.997535705566406,
      "loss/preference_sft": 0.30549412965774536,
      "loss/reference_anchor": 0.5389922857284546,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.399735927581787,
      "rewards/margins": 1.2197308540344238,
      "rewards/rejected": 1.1800049543380737,
      "step": 795
    },
    {
      "drift/chosen_abs": 0.2291211634874344,
      "drift/rejected_abs": 0.12003247439861298,
      "epoch": 0.7643616386002627,
      "grad_norm": 59.75,
      "learning_rate": 2.388621363640797e-08,
      "logits/chosen": -2.1559276580810547,
      "logits/rejected": -2.1329169273376465,
      "logps/chosen": -0.28992101550102234,
      "logps/rejected": -0.2854466736316681,
      "loss": 24.079273223876953,
      "loss/core": 24.065502166748047,
      "loss/preference_sft": 0.28992101550102234,
      "loss/reference_anchor": 0.15460558235645294,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.290127754211426,
      "rewards/margins": 1.0904570817947388,
      "rewards/rejected": 1.1996705532073975,
      "step": 800
    },
    {
      "drift/chosen_abs": 0.22606876492500305,
      "drift/rejected_abs": 0.2016010731458664,
      "epoch": 0.7691388988415144,
      "grad_norm": 104.0,
      "learning_rate": 2.1613635589349756e-08,
      "logits/chosen": -2.028775453567505,
      "logits/rejected": -2.0167717933654785,
      "logps/chosen": -0.2745590806007385,
      "logps/rejected": -0.2676575779914856,
      "loss": 24.971622467041016,
      "loss/core": 24.949968338012695,
      "loss/preference_sft": 0.2745590806007385,
      "loss/reference_anchor": 0.2612423002719879,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.2601115703582764,
      "rewards/margins": 0.24478550255298615,
      "rewards/rejected": 2.0153260231018066,
      "step": 805
    },
    {
      "drift/chosen_abs": 0.15367405116558075,
      "drift/rejected_abs": 0.14802715182304382,
      "epoch": 0.773916159082766,
      "grad_norm": 51.5,
      "learning_rate": 1.944975708238708e-08,
      "logits/chosen": -2.2288622856140137,
      "logits/rejected": -2.1330811977386475,
      "logps/chosen": -0.30788537859916687,
      "logps/rejected": -0.31650078296661377,
      "loss": 25.090599060058594,
      "loss/core": 25.078384399414062,
      "loss/preference_sft": 0.30788537859916687,
      "loss/reference_anchor": 0.13205906748771667,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.534415364265442,
      "rewards/margins": 0.05462510138750076,
      "rewards/rejected": 1.4797903299331665,
      "step": 810
    },
    {
      "drift/chosen_abs": 0.2130316197872162,
      "drift/rejected_abs": 0.11016812175512314,
      "epoch": 0.7786934193240177,
      "grad_norm": 145.0,
      "learning_rate": 1.7395608004493884e-08,
      "logits/chosen": -2.0962109565734863,
      "logits/rejected": -2.1853909492492676,
      "logps/chosen": -0.2830978333950043,
      "logps/rejected": -0.2838820815086365,
      "loss": 24.020706176757812,
      "loss/core": 24.00937843322754,
      "loss/preference_sft": 0.2830978333950043,
      "loss/reference_anchor": 0.12274503707885742,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1303162574768066,
      "rewards/margins": 1.032294750213623,
      "rewards/rejected": 1.0980216264724731,
      "step": 815
    },
    {
      "drift/chosen_abs": 0.1970573216676712,
      "drift/rejected_abs": 0.07308342307806015,
      "epoch": 0.7834706795652693,
      "grad_norm": 45.25,
      "learning_rate": 1.5452166019378987e-08,
      "logits/chosen": -2.099004030227661,
      "logits/rejected": -2.1342720985412598,
      "logps/chosen": -0.2875714898109436,
      "logps/rejected": -0.31725940108299255,
      "loss": 23.889062881469727,
      "loss/core": 23.877830505371094,
      "loss/preference_sft": 0.2875714898109436,
      "loss/reference_anchor": 0.12097527086734772,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.9672977924346924,
      "rewards/margins": 1.2818208932876587,
      "rewards/rejected": 0.685477077960968,
      "step": 820
    },
    {
      "drift/chosen_abs": 0.2467678040266037,
      "drift/rejected_abs": 0.12167330831289291,
      "epoch": 0.788247939806521,
      "grad_norm": 183.0,
      "learning_rate": 1.3620356100170788e-08,
      "logits/chosen": -2.1444103717803955,
      "logits/rejected": -2.247798442840576,
      "logps/chosen": -0.3017248809337616,
      "logps/rejected": -0.28473973274230957,
      "loss": 23.85074234008789,
      "loss/core": 23.836971282958984,
      "loss/preference_sft": 0.3017248809337616,
      "loss/reference_anchor": 0.15346962213516235,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.466996669769287,
      "rewards/margins": 1.2521899938583374,
      "rewards/rejected": 1.2148066759109497,
      "step": 825
    },
    {
      "drift/chosen_abs": 0.17280618846416473,
      "drift/rejected_abs": 0.16574548184871674,
      "epoch": 0.7930252000477725,
      "grad_norm": 91.0,
      "learning_rate": 1.190105008918041e-08,
      "logits/chosen": -2.1808249950408936,
      "logits/rejected": -2.1297428607940674,
      "logps/chosen": -0.2784227430820465,
      "logps/rejected": -0.30759522318840027,
      "loss": 25.10189437866211,
      "loss/core": 25.08694839477539,
      "loss/preference_sft": 0.2784227430820465,
      "loss/reference_anchor": 0.17144832015037537,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7274526357650757,
      "rewards/margins": 0.07077233493328094,
      "rewards/rejected": 1.6566804647445679,
      "step": 830
    },
    {
      "drift/chosen_abs": 0.24050016701221466,
      "drift/rejected_abs": 0.11438360065221786,
      "epoch": 0.7978024602890242,
      "grad_norm": 58.75,
      "learning_rate": 1.0295066282951737e-08,
      "logits/chosen": -2.096271276473999,
      "logits/rejected": -2.096055507659912,
      "logps/chosen": -0.2547900378704071,
      "logps/rejected": -0.3017795979976654,
      "loss": 23.816970825195312,
      "loss/core": 23.80051040649414,
      "loss/preference_sft": 0.2547900378704071,
      "loss/reference_anchor": 0.19398611783981323,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4034855365753174,
      "rewards/margins": 1.5796620845794678,
      "rewards/rejected": 0.8238231539726257,
      "step": 835
    },
    {
      "drift/chosen_abs": 0.28091713786125183,
      "drift/rejected_abs": 0.10494539886713028,
      "epoch": 0.8025797205302759,
      "grad_norm": 796.0,
      "learning_rate": 8.803169042796765e-09,
      "logits/chosen": -2.0359435081481934,
      "logits/rejected": -2.085045576095581,
      "logps/chosen": -0.3219825327396393,
      "logps/rejected": -0.3002759516239166,
      "loss": 23.931066513061523,
      "loss/core": 23.892078399658203,
      "loss/preference_sft": 0.3219825327396393,
      "loss/reference_anchor": 0.4876865744590759,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8080499172210693,
      "rewards/margins": 1.7595913410186768,
      "rewards/rejected": 1.0484585762023926,
      "step": 840
    },
    {
      "drift/chosen_abs": 0.141829252243042,
      "drift/rejected_abs": 0.06329073011875153,
      "epoch": 0.8073569807715275,
      "grad_norm": 54.5,
      "learning_rate": 7.4260684310008815e-09,
      "logits/chosen": -2.1739792823791504,
      "logits/rejected": -2.248871326446533,
      "logps/chosen": -0.29366642236709595,
      "logps/rejected": -0.3113446831703186,
      "loss": 24.187685012817383,
      "loss/core": 24.17953109741211,
      "loss/preference_sft": 0.29366642236709595,
      "loss/reference_anchor": 0.07929544895887375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.4174824953079224,
      "rewards/margins": 0.963509202003479,
      "rewards/rejected": 0.45397335290908813,
      "step": 845
    },
    {
      "drift/chosen_abs": 0.15600529313087463,
      "drift/rejected_abs": 0.09067943692207336,
      "epoch": 0.8121342410127792,
      "grad_norm": 314.0,
      "learning_rate": 6.164419872871835e-09,
      "logits/chosen": -2.2953596115112305,
      "logits/rejected": -2.3431358337402344,
      "logps/chosen": -0.2520250082015991,
      "logps/rejected": -0.24778322875499725,
      "loss": 24.37624168395996,
      "loss/core": 24.37070083618164,
      "loss/preference_sft": 0.2520250082015991,
      "loss/reference_anchor": 0.04869046062231064,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.5578166246414185,
      "rewards/margins": 0.6599420309066772,
      "rewards/rejected": 0.8978745341300964,
      "step": 850
    },
    {
      "drift/chosen_abs": 0.27003389596939087,
      "drift/rejected_abs": 0.12391098588705063,
      "epoch": 0.8169115012540308,
      "grad_norm": 143.0,
      "learning_rate": 5.018823844792602e-09,
      "logits/chosen": -2.0034213066101074,
      "logits/rejected": -2.040942668914795,
      "logps/chosen": -0.2998163104057312,
      "logps/rejected": -0.2942708730697632,
      "loss": 23.890283584594727,
      "loss/core": 23.85587501525879,
      "loss/preference_sft": 0.2998163104057312,
      "loss/reference_anchor": 0.42880433797836304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.6989176273345947,
      "rewards/margins": 1.4601688385009766,
      "rewards/rejected": 1.2387486696243286,
      "step": 855
    },
    {
      "drift/chosen_abs": 0.409585177898407,
      "drift/rejected_abs": 0.19676879048347473,
      "epoch": 0.8216887614952825,
      "grad_norm": 37.25,
      "learning_rate": 3.989825588427281e-09,
      "logits/chosen": -1.9499990940093994,
      "logits/rejected": -2.038193702697754,
      "logps/chosen": -0.2986469268798828,
      "logps/rejected": -0.2962108850479126,
      "loss": 23.53319549560547,
      "loss/core": 23.46843910217285,
      "loss/preference_sft": 0.2986469268798828,
      "loss/reference_anchor": 0.8335119485855103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.0953145027160645,
      "rewards/margins": 2.138956308364868,
      "rewards/rejected": 1.956358551979065,
      "step": 860
    },
    {
      "drift/chosen_abs": 0.24526743590831757,
      "drift/rejected_abs": 0.15863600373268127,
      "epoch": 0.826466021736534,
      "grad_norm": 85.5,
      "learning_rate": 3.077914851215585e-09,
      "logits/chosen": -1.95840585231781,
      "logits/rejected": -2.0312142372131348,
      "logps/chosen": -0.29089808464050293,
      "logps/rejected": -0.29388031363487244,
      "loss": 24.42396354675293,
      "loss/core": 24.399206161499023,
      "loss/preference_sft": 0.29089808464050293,
      "loss/reference_anchor": 0.30098140239715576,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.4520413875579834,
      "rewards/margins": 0.8663004040718079,
      "rewards/rejected": 1.5857408046722412,
      "step": 865
    },
    {
      "drift/chosen_abs": 0.3167889714241028,
      "drift/rejected_abs": 0.09579071402549744,
      "epoch": 0.8312432819777857,
      "grad_norm": 91.0,
      "learning_rate": 2.2835256532794387e-09,
      "logits/chosen": -2.1105716228485107,
      "logits/rejected": -2.119257926940918,
      "logps/chosen": -0.3130267262458801,
      "logps/rejected": -0.2952987551689148,
      "loss": 23.687124252319336,
      "loss/core": 23.64649772644043,
      "loss/preference_sft": 0.3130267262458801,
      "loss/reference_anchor": 0.5103899240493774,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.165898561477661,
      "rewards/margins": 2.2113327980041504,
      "rewards/rejected": 0.9545663595199585,
      "step": 870
    },
    {
      "drift/chosen_abs": 0.1863785833120346,
      "drift/rejected_abs": 0.10340656340122223,
      "epoch": 0.8360205422190374,
      "grad_norm": 66.0,
      "learning_rate": 1.6070360808531359e-09,
      "logits/chosen": -2.1814465522766113,
      "logits/rejected": -2.22865891456604,
      "logps/chosen": -0.3077881932258606,
      "logps/rejected": -0.3306838274002075,
      "loss": 24.516956329345703,
      "loss/core": 24.487125396728516,
      "loss/preference_sft": 0.3077881932258606,
      "loss/reference_anchor": 0.36692848801612854,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8635761737823486,
      "rewards/margins": 0.8385453224182129,
      "rewards/rejected": 1.0250307321548462,
      "step": 875
    },
    {
      "drift/chosen_abs": 0.24594731628894806,
      "drift/rejected_abs": 0.18863597512245178,
      "epoch": 0.840797802460289,
      "grad_norm": 194.0,
      "learning_rate": 1.0487681063345854e-09,
      "logits/chosen": -2.1177544593811035,
      "logits/rejected": -2.1003167629241943,
      "logps/chosen": -0.2906046509742737,
      "logps/rejected": -0.3307008743286133,
      "loss": 25.163158416748047,
      "loss/core": 25.12496566772461,
      "loss/preference_sft": 0.2906046509742737,
      "loss/reference_anchor": 0.4801712930202484,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.458425521850586,
      "rewards/margins": 0.6850012540817261,
      "rewards/rejected": 1.7734243869781494,
      "step": 880
    },
    {
      "drift/chosen_abs": 0.27374953031539917,
      "drift/rejected_abs": 0.11317259073257446,
      "epoch": 0.8455750627015407,
      "grad_norm": 96.0,
      "learning_rate": 6.089874350439505e-10,
      "logits/chosen": -2.0427865982055664,
      "logits/rejected": -2.1189193725585938,
      "logps/chosen": -0.2813889980316162,
      "logps/rejected": -0.2768685817718506,
      "loss": 23.570148468017578,
      "loss/core": 23.551849365234375,
      "loss/preference_sft": 0.2813889980316162,
      "loss/reference_anchor": 0.215844064950943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.7366836071014404,
      "rewards/margins": 1.611907958984375,
      "rewards/rejected": 1.1247756481170654,
      "step": 885
    },
    {
      "drift/chosen_abs": 0.246209055185318,
      "drift/rejected_abs": 0.1439971625804901,
      "epoch": 0.8503523229427923,
      "grad_norm": 130.0,
      "learning_rate": 2.8790337876233305e-10,
      "logits/chosen": -1.9758459329605103,
      "logits/rejected": -2.097522735595703,
      "logps/chosen": -0.29943814873695374,
      "logps/rejected": -0.27435192465782166,
      "loss": 24.266326904296875,
      "loss/core": 24.23988914489746,
      "loss/preference_sft": 0.29943814873695374,
      "loss/reference_anchor": 0.3225429058074951,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.461657762527466,
      "rewards/margins": 1.0233776569366455,
      "rewards/rejected": 1.4382802248001099,
      "step": 890
    },
    {
      "drift/chosen_abs": 0.23041865229606628,
      "drift/rejected_abs": 0.15222403407096863,
      "epoch": 0.855129583184044,
      "grad_norm": 172.0,
      "learning_rate": 8.566875611068503e-11,
      "logits/chosen": -2.044543504714966,
      "logits/rejected": -2.07798433303833,
      "logps/chosen": -0.27221474051475525,
      "logps/rejected": -0.2905210852622986,
      "loss": 24.323848724365234,
      "loss/core": 24.309627532958984,
      "loss/preference_sft": 0.27221474051475525,
      "loss/reference_anchor": 0.162424698472023,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.3031530380249023,
      "rewards/margins": 0.7896323800086975,
      "rewards/rejected": 1.51352059841156,
      "step": 895
    },
    {
      "drift/chosen_abs": 0.31285884976387024,
      "drift/rejected_abs": 0.08625397086143494,
      "epoch": 0.8599068434252956,
      "grad_norm": 135.0,
      "learning_rate": 2.3798198163782478e-12,
      "logits/chosen": -2.055809736251831,
      "logits/rejected": -2.176356315612793,
      "logps/chosen": -0.2678307294845581,
      "logps/rejected": -0.268280953168869,
      "loss": 23.679527282714844,
      "loss/core": 23.63846778869629,
      "loss/preference_sft": 0.2678307294845581,
      "loss/reference_anchor": 0.5206762552261353,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 3.1283411979675293,
      "rewards/margins": 2.278930187225342,
      "rewards/rejected": 0.8494111895561218,
      "step": 900
    },
    {
      "epoch": 0.8599068434252956,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 24.184869588216145,
      "train_runtime": 7109.0777,
      "train_samples_per_second": 2.026,
      "train_steps_per_second": 0.127
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}