{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2860548271752086,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "drift/chosen_abs": 0.2889288067817688,
      "drift/rejected_abs": 0.17525029182434082,
      "epoch": 0.0015891934843067143,
      "grad_norm": 1.0546875,
      "learning_rate": 1.111111111111111e-09,
      "logits/chosen": -2.227959632873535,
      "logits/rejected": -2.153649091720581,
      "logps/chosen": -0.25035396218299866,
      "logps/rejected": -0.25459814071655273,
      "loss": 0.2694305181503296,
      "loss/core": 0.24971142411231995,
      "loss/preference_sft": 0.2522751986980438,
      "loss/reference_anchor": 0.3691546618938446,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.8892879486083984,
      "rewards/margins": 1.1368046998977661,
      "rewards/rejected": 1.7524830102920532,
      "ronpo/logit": 0.11368048191070557,
      "ronpo/residual": 0.11023612320423126,
      "ronpo/residual_abs": 0.22715234756469727,
      "ronpo/target": 0.0034443773329257965,
      "ronpo/target_abs": 0.06975062191486359,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 5
    },
    {
      "drift/chosen_abs": 0.2513688802719116,
      "drift/rejected_abs": 0.17290709912776947,
      "epoch": 0.0031783869686134287,
      "grad_norm": 4.5,
      "learning_rate": 2.5e-09,
      "logits/chosen": -1.9819161891937256,
      "logits/rejected": -1.923046350479126,
      "logps/chosen": -0.26421818137168884,
      "logps/rejected": -0.2721954882144928,
      "loss": 0.17001162469387054,
      "loss/core": 0.15246111154556274,
      "loss/preference_sft": 0.2747360169887543,
      "loss/reference_anchor": 0.3235367238521576,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5129077434539795,
      "rewards/margins": 0.78448885679245,
      "rewards/rejected": 1.7284189462661743,
      "ronpo/logit": 0.07844887673854828,
      "ronpo/residual": 0.06979227811098099,
      "ronpo/residual_abs": 0.19728776812553406,
      "ronpo/target": 0.008656605146825314,
      "ronpo/target_abs": 0.09380026161670685,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 10
    },
    {
      "drift/chosen_abs": 0.14740121364593506,
      "drift/rejected_abs": 0.08022782951593399,
      "epoch": 0.004767580452920143,
      "grad_norm": 7.3125,
      "learning_rate": 3.8888888888888884e-09,
      "logits/chosen": -2.2918307781219482,
      "logits/rejected": -2.2467479705810547,
      "logps/chosen": -0.27619749307632446,
      "logps/rejected": -0.2822367548942566,
      "loss": 0.05183436721563339,
      "loss/core": 0.047430895268917084,
      "loss/preference_sft": 0.277311235666275,
      "loss/reference_anchor": 0.060338426381349564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.4728786945343018,
      "rewards/margins": 0.6718867421150208,
      "rewards/rejected": 0.8009918928146362,
      "ronpo/logit": 0.06718868762254715,
      "ronpo/residual": 0.102389857172966,
      "ronpo/residual_abs": 0.14397867023944855,
      "ronpo/target": -0.035201169550418854,
      "ronpo/target_abs": 0.08456156402826309,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 15
    },
    {
      "drift/chosen_abs": 0.1409628540277481,
      "drift/rejected_abs": 0.07020283490419388,
      "epoch": 0.006356773937226857,
      "grad_norm": 1.4765625,
      "learning_rate": 5.277777777777778e-09,
      "logits/chosen": -2.291287422180176,
      "logits/rejected": -2.2165215015411377,
      "logps/chosen": -0.27579012513160706,
      "logps/rejected": -0.278958261013031,
      "loss": 0.03074510395526886,
      "loss/core": 0.026744594797492027,
      "loss/preference_sft": 0.2790845036506653,
      "loss/reference_anchor": 0.052101779729127884,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.409259557723999,
      "rewards/margins": 0.7112404704093933,
      "rewards/rejected": 0.6980190277099609,
      "ronpo/logit": 0.07112405449151993,
      "ronpo/residual": 0.08393178880214691,
      "ronpo/residual_abs": 0.11383052915334702,
      "ronpo/target": -0.012807758525013924,
      "ronpo/target_abs": 0.07244060188531876,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 20
    },
    {
      "drift/chosen_abs": 0.2341737300157547,
      "drift/rejected_abs": 0.07688029110431671,
      "epoch": 0.007945967421533572,
      "grad_norm": 1.0625,
      "learning_rate": 6.666666666666666e-09,
      "logits/chosen": -2.354504346847534,
      "logits/rejected": -2.393965721130371,
      "logps/chosen": -0.2399778664112091,
      "logps/rejected": -0.2485647201538086,
      "loss": 0.30489665269851685,
      "loss/core": 0.29363757371902466,
      "loss/preference_sft": 0.2436579465866089,
      "loss/reference_anchor": 0.20081612467765808,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3414459228515625,
      "rewards/margins": 1.5734647512435913,
      "rewards/rejected": 0.7679811716079712,
      "ronpo/logit": 0.15734648704528809,
      "ronpo/residual": 0.15989667177200317,
      "ronpo/residual_abs": 0.2039765864610672,
      "ronpo/target": -0.0025501928757876158,
      "ronpo/target_abs": 0.07882604002952576,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 25
    },
    {
      "drift/chosen_abs": 0.20403337478637695,
      "drift/rejected_abs": 0.08686282485723495,
      "epoch": 0.009535160905840286,
      "grad_norm": 344.0,
      "learning_rate": 8.055555555555556e-09,
      "logits/chosen": -2.2312631607055664,
      "logits/rejected": -2.1263251304626465,
      "logps/chosen": -0.28779473900794983,
      "logps/rejected": -0.28351494669914246,
      "loss": 0.18434959650039673,
      "loss/core": 0.1724001169204712,
      "loss/preference_sft": 0.29089125990867615,
      "loss/reference_anchor": 0.20990124344825745,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.0403337478637695,
      "rewards/margins": 1.1722705364227295,
      "rewards/rejected": 0.8680631518363953,
      "ronpo/logit": 0.11722707748413086,
      "ronpo/residual": 0.10378406196832657,
      "ronpo/residual_abs": 0.1859159767627716,
      "ronpo/target": 0.013443012721836567,
      "ronpo/target_abs": 0.09407193213701248,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 30
    },
    {
      "drift/chosen_abs": 0.20877280831336975,
      "drift/rejected_abs": 0.13062386214733124,
      "epoch": 0.011124354390147001,
      "grad_norm": 4.0,
      "learning_rate": 9.444444444444444e-09,
      "logits/chosen": -2.1255409717559814,
      "logits/rejected": -2.06231951713562,
      "logps/chosen": -0.2710401117801666,
      "logps/rejected": -0.28147274255752563,
      "loss": 0.05778823047876358,
      "loss/core": 0.04806242510676384,
      "loss/preference_sft": 0.27798718214035034,
      "loss/reference_anchor": 0.16671733558177948,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0877280235290527,
      "rewards/margins": 0.7836453318595886,
      "rewards/rejected": 1.3040828704833984,
      "ronpo/logit": 0.0783645361661911,
      "ronpo/residual": 0.06768552213907242,
      "ronpo/residual_abs": 0.1325724571943283,
      "ronpo/target": 0.01067900937050581,
      "ronpo/target_abs": 0.08033877611160278,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 35
    },
    {
      "drift/chosen_abs": 0.1339804232120514,
      "drift/rejected_abs": 0.1121961921453476,
      "epoch": 0.012713547874453715,
      "grad_norm": 444.0,
      "learning_rate": 1.0833333333333333e-08,
      "logits/chosen": -2.0807948112487793,
      "logits/rejected": -2.042069435119629,
      "logps/chosen": -0.31901049613952637,
      "logps/rejected": -0.3213141858577728,
      "loss": 0.2600531280040741,
      "loss/core": 0.2503146529197693,
      "loss/preference_sft": 0.31382906436920166,
      "loss/reference_anchor": 0.16338738799095154,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.3386170864105225,
      "rewards/margins": 0.21692340075969696,
      "rewards/rejected": 1.1216938495635986,
      "ronpo/logit": 0.021692335605621338,
      "ronpo/residual": 0.011326847597956657,
      "ronpo/residual_abs": 0.17393314838409424,
      "ronpo/target": 0.010365481488406658,
      "ronpo/target_abs": 0.08013413846492767,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 40
    },
    {
      "drift/chosen_abs": 0.2002873420715332,
      "drift/rejected_abs": 0.10398294776678085,
      "epoch": 0.014302741358760428,
      "grad_norm": 4.0,
      "learning_rate": 1.2222222222222222e-08,
      "logits/chosen": -2.149164915084839,
      "logits/rejected": -2.077511787414551,
      "logps/chosen": -0.30421438813209534,
      "logps/rejected": -0.29620206356048584,
      "loss": 0.09899889677762985,
      "loss/core": 0.09022404253482819,
      "loss/preference_sft": 0.2990511357784271,
      "loss/reference_anchor": 0.14559200406074524,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.0019478797912598,
      "rewards/margins": 0.9650747179985046,
      "rewards/rejected": 1.0368731021881104,
      "ronpo/logit": 0.0965074822306633,
      "ronpo/residual": 0.09062016755342484,
      "ronpo/residual_abs": 0.1583360731601715,
      "ronpo/target": 0.005887320265173912,
      "ronpo/target_abs": 0.08980895578861237,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 45
    },
    {
      "drift/chosen_abs": 0.21013769507408142,
      "drift/rejected_abs": 0.1717991977930069,
      "epoch": 0.015891934843067144,
      "grad_norm": 7.65625,
      "learning_rate": 1.361111111111111e-08,
      "logits/chosen": -2.2315473556518555,
      "logits/rejected": -2.2804598808288574,
      "logps/chosen": -0.38569122552871704,
      "logps/rejected": -0.37548166513442993,
      "loss": 0.49833402037620544,
      "loss/core": 0.47320446372032166,
      "loss/preference_sft": 0.3764234185218811,
      "loss/reference_anchor": 0.4649490416049957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.10087251663208,
      "rewards/margins": 1.4145400524139404,
      "rewards/rejected": 0.6863324642181396,
      "ronpo/logit": 0.14145398139953613,
      "ronpo/residual": 0.14585429430007935,
      "ronpo/residual_abs": 0.2349962443113327,
      "ronpo/target": -0.004400297999382019,
      "ronpo/target_abs": 0.0763171911239624,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 50
    },
    {
      "drift/chosen_abs": 0.24356035888195038,
      "drift/rejected_abs": 0.1136392131447792,
      "epoch": 0.01748112832737386,
      "grad_norm": 1.1015625,
      "learning_rate": 1.5e-08,
      "logits/chosen": -2.132157802581787,
      "logits/rejected": -2.104531764984131,
      "logps/chosen": -0.30124813318252563,
      "logps/rejected": -0.29356569051742554,
      "loss": 0.29936105012893677,
      "loss/core": 0.28574955463409424,
      "loss/preference_sft": 0.28952398896217346,
      "loss/reference_anchor": 0.2432773858308792,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4335732460021973,
      "rewards/margins": 1.3003393411636353,
      "rewards/rejected": 1.133233666419983,
      "ronpo/logit": 0.130033940076828,
      "ronpo/residual": 0.13970455527305603,
      "ronpo/residual_abs": 0.19125238060951233,
      "ronpo/target": -0.009670635685324669,
      "ronpo/target_abs": 0.07653583586215973,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 55
    },
    {
      "drift/chosen_abs": 0.2864144444465637,
      "drift/rejected_abs": 0.0761997327208519,
      "epoch": 0.01907032181168057,
      "grad_norm": 7.1875,
      "learning_rate": 1.6388888888888888e-08,
      "logits/chosen": -2.0191774368286133,
      "logits/rejected": -2.0377559661865234,
      "logps/chosen": -0.2580411732196808,
      "logps/rejected": -0.2605469822883606,
      "loss": 0.6949197053909302,
      "loss/core": 0.6738607883453369,
      "loss/preference_sft": 0.2634747624397278,
      "loss/reference_anchor": 0.39483022689819336,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.863614559173584,
      "rewards/margins": 2.115543842315674,
      "rewards/rejected": 0.7480708360671997,
      "ronpo/logit": 0.2115544080734253,
      "ronpo/residual": 0.21217253804206848,
      "ronpo/residual_abs": 0.25770795345306396,
      "ronpo/target": -0.0006181513890624046,
      "ronpo/target_abs": 0.07183288037776947,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 60
    },
    {
      "drift/chosen_abs": 0.23456306755542755,
      "drift/rejected_abs": 0.0846903845667839,
      "epoch": 0.020659515295987287,
      "grad_norm": 168.0,
      "learning_rate": 1.7777777777777777e-08,
      "logits/chosen": -2.3910279273986816,
      "logits/rejected": -2.4622697830200195,
      "logps/chosen": -0.25069719552993774,
      "logps/rejected": -0.24407532811164856,
      "loss": 0.2673589289188385,
      "loss/core": 0.2575306296348572,
      "loss/preference_sft": 0.24895858764648438,
      "loss/reference_anchor": 0.17166933417320251,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.345244884490967,
      "rewards/margins": 1.499186635017395,
      "rewards/rejected": 0.846058189868927,
      "ronpo/logit": 0.14991866052150726,
      "ronpo/residual": 0.12659938633441925,
      "ronpo/residual_abs": 0.19241955876350403,
      "ronpo/target": 0.02331927977502346,
      "ronpo/target_abs": 0.07524405419826508,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 65
    },
    {
      "drift/chosen_abs": 0.21925799548625946,
      "drift/rejected_abs": 0.11180920898914337,
      "epoch": 0.022248708780294002,
      "grad_norm": 2.71875,
      "learning_rate": 1.9166666666666666e-08,
      "logits/chosen": -2.1304917335510254,
      "logits/rejected": -2.13004994392395,
      "logps/chosen": -0.26294752955436707,
      "logps/rejected": -0.26812273263931274,
      "loss": 0.0758952647447586,
      "loss/core": 0.06884925067424774,
      "loss/preference_sft": 0.26684775948524475,
      "loss/reference_anchor": 0.11423534154891968,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1923234462738037,
      "rewards/margins": 1.074231505393982,
      "rewards/rejected": 1.1180920600891113,
      "ronpo/logit": 0.10742314904928207,
      "ronpo/residual": 0.09753496944904327,
      "ronpo/residual_abs": 0.14825595915317535,
      "ronpo/target": 0.00988816749304533,
      "ronpo/target_abs": 0.08186240494251251,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 70
    },
    {
      "drift/chosen_abs": 0.2804720997810364,
      "drift/rejected_abs": 0.1483515202999115,
      "epoch": 0.023837902264600714,
      "grad_norm": 5.84375,
      "learning_rate": 2.0555555555555552e-08,
      "logits/chosen": -2.107518434524536,
      "logits/rejected": -2.0409445762634277,
      "logps/chosen": -0.2804359197616577,
      "logps/rejected": -0.27236875891685486,
      "loss": 0.681021511554718,
      "loss/core": 0.6588231921195984,
      "loss/preference_sft": 0.271370530128479,
      "loss/reference_anchor": 0.41682863235473633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.8040549755096436,
      "rewards/margins": 1.3575153350830078,
      "rewards/rejected": 1.4465398788452148,
      "ronpo/logit": 0.13575153052806854,
      "ronpo/residual": 0.14553678035736084,
      "ronpo/residual_abs": 0.30461931228637695,
      "ronpo/target": -0.009785245172679424,
      "ronpo/target_abs": 0.08345939964056015,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 75
    },
    {
      "drift/chosen_abs": 0.32475045323371887,
      "drift/rejected_abs": 0.13860228657722473,
      "epoch": 0.02542709574890743,
      "grad_norm": 42.5,
      "learning_rate": 2.1944444444444445e-08,
      "logits/chosen": -2.056220531463623,
      "logits/rejected": -2.0234427452087402,
      "logps/chosen": -0.27260422706604004,
      "logps/rejected": -0.27340400218963623,
      "loss": 0.372364342212677,
      "loss/core": 0.35138756036758423,
      "loss/preference_sft": 0.2743496894836426,
      "loss/reference_anchor": 0.39210090041160583,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2468693256378174,
      "rewards/margins": 1.8645988702774048,
      "rewards/rejected": 1.3822702169418335,
      "ronpo/logit": 0.18645988404750824,
      "ronpo/residual": 0.18637283146381378,
      "ronpo/residual_abs": 0.2369922697544098,
      "ronpo/target": 8.704736683284864e-05,
      "ronpo/target_abs": 0.08356542885303497,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 80
    },
    {
      "drift/chosen_abs": 0.22833116352558136,
      "drift/rejected_abs": 0.07853401452302933,
      "epoch": 0.027016289233214145,
      "grad_norm": 988.0,
      "learning_rate": 2.3333333333333334e-08,
      "logits/chosen": -2.3675479888916016,
      "logits/rejected": -2.3091373443603516,
      "logps/chosen": -0.2940252423286438,
      "logps/rejected": -0.2664296329021454,
      "loss": 0.5738587975502014,
      "loss/core": 0.5565118789672852,
      "loss/preference_sft": 0.2698623538017273,
      "loss/reference_anchor": 0.31994983553886414,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.281959056854248,
      "rewards/margins": 1.5006506443023682,
      "rewards/rejected": 0.7813081741333008,
      "ronpo/logit": 0.1500650942325592,
      "ronpo/residual": 0.13329221308231354,
      "ronpo/residual_abs": 0.20770840346813202,
      "ronpo/target": 0.01677287369966507,
      "ronpo/target_abs": 0.08873967826366425,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 85
    },
    {
      "drift/chosen_abs": 0.1896965652704239,
      "drift/rejected_abs": 0.16110625863075256,
      "epoch": 0.028605482717520857,
      "grad_norm": 1.7578125,
      "learning_rate": 2.4722222222222223e-08,
      "logits/chosen": -2.261641025543213,
      "logits/rejected": -2.271144390106201,
      "logps/chosen": -0.2831428349018097,
      "logps/rejected": -0.2776196002960205,
      "loss": 0.27627962827682495,
      "loss/core": 0.2642122805118561,
      "loss/preference_sft": 0.27972379326820374,
      "loss/reference_anchor": 0.21337434649467468,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.896965742111206,
      "rewards/margins": 0.2912505865097046,
      "rewards/rejected": 1.6057151556015015,
      "ronpo/logit": 0.029125045984983444,
      "ronpo/residual": 0.029582571238279343,
      "ronpo/residual_abs": 0.18990905582904816,
      "ronpo/target": -0.0004575133207254112,
      "ronpo/target_abs": 0.08104494959115982,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 90
    },
    {
      "drift/chosen_abs": 0.22530770301818848,
      "drift/rejected_abs": 0.10128001868724823,
      "epoch": 0.030194676201827572,
      "grad_norm": 3.734375,
      "learning_rate": 2.4998495746616843e-08,
      "logits/chosen": -2.339174747467041,
      "logits/rejected": -2.257371425628662,
      "logps/chosen": -0.27327409386634827,
      "logps/rejected": -0.26891353726387024,
      "loss": 0.345085084438324,
      "loss/core": 0.33327919244766235,
      "loss/preference_sft": 0.26861560344696045,
      "loss/reference_anchor": 0.20925641059875488,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.2530770301818848,
      "rewards/margins": 1.2411054372787476,
      "rewards/rejected": 1.0119714736938477,
      "ronpo/logit": 0.12411054223775864,
      "ronpo/residual": 0.1508522927761078,
      "ronpo/residual_abs": 0.2290908843278885,
      "ronpo/target": -0.0267417524009943,
      "ronpo/target_abs": 0.08890058100223541,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 95
    },
    {
      "drift/chosen_abs": 0.2532444894313812,
      "drift/rejected_abs": 0.09977379441261292,
      "epoch": 0.03178386968613429,
      "grad_norm": 8.625,
      "learning_rate": 2.4992385337738698e-08,
      "logits/chosen": -1.9698337316513062,
      "logits/rejected": -2.0415987968444824,
      "logps/chosen": -0.29794543981552124,
      "logps/rejected": -0.28950685262680054,
      "loss": 0.24947309494018555,
      "loss/core": 0.23802785575389862,
      "loss/preference_sft": 0.29647988080978394,
      "loss/reference_anchor": 0.1992568075656891,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.531574249267578,
      "rewards/margins": 1.5353649854660034,
      "rewards/rejected": 0.9962092638015747,
      "ronpo/logit": 0.15353651344776154,
      "ronpo/residual": 0.15389743447303772,
      "ronpo/residual_abs": 0.2095203399658203,
      "ronpo/target": -0.00036093126982450485,
      "ronpo/target_abs": 0.08967583626508713,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 100
    },
    {
      "drift/chosen_abs": 0.3917911648750305,
      "drift/rejected_abs": 0.127794548869133,
      "epoch": 0.033373063170441,
      "grad_norm": 93.5,
      "learning_rate": 2.4981577053636144e-08,
      "logits/chosen": -1.9294116497039795,
      "logits/rejected": -1.936621904373169,
      "logps/chosen": -0.2786913514137268,
      "logps/rejected": -0.2831571400165558,
      "loss": 0.699501633644104,
      "loss/core": 0.6670955419540405,
      "loss/preference_sft": 0.2841565012931824,
      "loss/reference_anchor": 0.6197081208229065,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.9174468517303467,
      "rewards/margins": 2.6591506004333496,
      "rewards/rejected": 1.258296251296997,
      "ronpo/logit": 0.26591506600379944,
      "ronpo/residual": 0.2671971917152405,
      "ronpo/residual_abs": 0.31767672300338745,
      "ronpo/target": -0.0012821194250136614,
      "ronpo/target_abs": 0.08636602014303207,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 105
    },
    {
      "drift/chosen_abs": 0.2426718771457672,
      "drift/rejected_abs": 0.12293334305286407,
      "epoch": 0.03496225665474772,
      "grad_norm": 8.375,
      "learning_rate": 2.4966074958862813e-08,
      "logits/chosen": -2.245562791824341,
      "logits/rejected": -2.205705165863037,
      "logps/chosen": -0.29507607221603394,
      "logps/rejected": -0.2873607277870178,
      "loss": 0.12289254367351532,
      "loss/core": 0.1113683208823204,
      "loss/preference_sft": 0.29041385650634766,
      "loss/reference_anchor": 0.20144303143024445,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.424837589263916,
      "rewards/margins": 1.2106255292892456,
      "rewards/rejected": 1.21421217918396,
      "ronpo/logit": 0.12106256186962128,
      "ronpo/residual": 0.11163101345300674,
      "ronpo/residual_abs": 0.1918853521347046,
      "ronpo/target": 0.009431545622646809,
      "ronpo/target_abs": 0.10175038874149323,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 110
    },
    {
      "drift/chosen_abs": 0.27447623014450073,
      "drift/rejected_abs": 0.13834591209888458,
      "epoch": 0.03655145013905443,
      "grad_norm": 1.7265625,
      "learning_rate": 2.4945884883122553e-08,
      "logits/chosen": -1.8843482732772827,
      "logits/rejected": -1.775320053100586,
      "logps/chosen": -0.2798824906349182,
      "logps/rejected": -0.2814459502696991,
      "loss": 0.1541265845298767,
      "loss/core": 0.14217844605445862,
      "loss/preference_sft": 0.27655261754989624,
      "loss/reference_anchor": 0.21130748093128204,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.742548942565918,
      "rewards/margins": 1.366241693496704,
      "rewards/rejected": 1.376307725906372,
      "ronpo/logit": 0.13662415742874146,
      "ronpo/residual": 0.14770525693893433,
      "ronpo/residual_abs": 0.18340587615966797,
      "ronpo/target": -0.011081096716225147,
      "ronpo/target_abs": 0.07495449483394623,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 115
    },
    {
      "drift/chosen_abs": 0.25327441096305847,
      "drift/rejected_abs": 0.11703983694314957,
      "epoch": 0.03814064362336114,
      "grad_norm": 0.9609375,
      "learning_rate": 2.492101441907714e-08,
      "logits/chosen": -2.0191433429718018,
      "logits/rejected": -2.067561626434326,
      "logps/chosen": -0.30190160870552063,
      "logps/rejected": -0.30017122626304626,
      "loss": 0.30577465891838074,
      "loss/core": 0.28840842843055725,
      "loss/preference_sft": 0.2985996901988983,
      "loss/reference_anchor": 0.31746405363082886,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.5322983264923096,
      "rewards/margins": 1.3638817071914673,
      "rewards/rejected": 1.1684163808822632,
      "ronpo/logit": 0.1363881528377533,
      "ronpo/residual": 0.11740622669458389,
      "ronpo/residual_abs": 0.22061511874198914,
      "ronpo/target": 0.0189819298684597,
      "ronpo/target_abs": 0.08912766724824905,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 120
    },
    {
      "drift/chosen_abs": 0.12704667448997498,
      "drift/rejected_abs": 0.07079822570085526,
      "epoch": 0.03972983710766786,
      "grad_norm": 1.2265625,
      "learning_rate": 2.4891472919490977e-08,
      "logits/chosen": -2.272683620452881,
      "logits/rejected": -2.193810224533081,
      "logps/chosen": -0.2744141221046448,
      "logps/rejected": -0.2814023196697235,
      "loss": 0.04335567355155945,
      "loss/core": 0.03851154074072838,
      "loss/preference_sft": 0.26887914538383484,
      "loss/reference_anchor": 0.06999474763870239,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.2701770067214966,
      "rewards/margins": 0.5637924671173096,
      "rewards/rejected": 0.7063844799995422,
      "ronpo/logit": 0.05637924745678902,
      "ronpo/residual": 0.0646718442440033,
      "ronpo/residual_abs": 0.11568011343479156,
      "ronpo/target": -0.008292593993246555,
      "ronpo/target_abs": 0.08368505537509918,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 125
    },
    {
      "drift/chosen_abs": 0.3699837327003479,
      "drift/rejected_abs": 0.1531117707490921,
      "epoch": 0.04131903059197457,
      "grad_norm": 15.5,
      "learning_rate": 2.4857271493713895e-08,
      "logits/chosen": -2.0499277114868164,
      "logits/rejected": -1.9798452854156494,
      "logps/chosen": -0.2802344560623169,
      "logps/rejected": -0.3006633222103119,
      "loss": 0.4787680506706238,
      "loss/core": 0.4418778419494629,
      "loss/preference_sft": 0.2814800441265106,
      "loss/reference_anchor": 0.7096576690673828,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.6996917724609375,
      "rewards/margins": 2.1705682277679443,
      "rewards/rejected": 1.529123306274414,
      "ronpo/logit": 0.21705684065818787,
      "ronpo/residual": 0.2116149216890335,
      "ronpo/residual_abs": 0.26738792657852173,
      "ronpo/target": 0.005441917572170496,
      "ronpo/target_abs": 0.07667852938175201,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 130
    },
    {
      "drift/chosen_abs": 0.2286110669374466,
      "drift/rejected_abs": 0.17728818953037262,
      "epoch": 0.04290822407628129,
      "grad_norm": 99.5,
      "learning_rate": 2.481842300350339e-08,
      "logits/chosen": -2.156745195388794,
      "logits/rejected": -2.1426777839660645,
      "logps/chosen": -0.2545807361602783,
      "logps/rejected": -0.2720378339290619,
      "loss": 0.4073014259338379,
      "loss/core": 0.39105960726737976,
      "loss/preference_sft": 0.26137062907218933,
      "loss/reference_anchor": 0.2986990809440613,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.285310745239258,
      "rewards/margins": 0.5148512125015259,
      "rewards/rejected": 1.770459532737732,
      "ronpo/logit": 0.05148511379957199,
      "ronpo/residual": 0.06029368191957474,
      "ronpo/residual_abs": 0.22104015946388245,
      "ronpo/target": -0.00880857277661562,
      "ronpo/target_abs": 0.08236870914697647,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 135
    },
    {
      "drift/chosen_abs": 0.21387775242328644,
      "drift/rejected_abs": 0.1563216596841812,
      "epoch": 0.044497417560588004,
      "grad_norm": 41.75,
      "learning_rate": 2.4774942058187855e-08,
      "logits/chosen": -2.0665993690490723,
      "logits/rejected": -2.035794734954834,
      "logps/chosen": -0.28517797589302063,
      "logps/rejected": -0.28532570600509644,
      "loss": 0.26362624764442444,
      "loss/core": 0.2508805990219116,
      "loss/preference_sft": 0.28584787249565125,
      "loss/reference_anchor": 0.22632861137390137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1381399631500244,
      "rewards/margins": 0.5762137174606323,
      "rewards/rejected": 1.561926245689392,
      "ronpo/logit": 0.05762135982513428,
      "ronpo/residual": 0.059470035135746,
      "ronpo/residual_abs": 0.2042684555053711,
      "ronpo/target": -0.0018486687913537025,
      "ronpo/target_abs": 0.08461553603410721,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 140
    },
    {
      "drift/chosen_abs": 0.3112896680831909,
      "drift/rejected_abs": 0.13714732229709625,
      "epoch": 0.04608661104489471,
      "grad_norm": 226.0,
      "learning_rate": 2.472684500917257e-08,
      "logits/chosen": -2.104536771774292,
      "logits/rejected": -2.14062237739563,
      "logps/chosen": -0.2743372917175293,
      "logps/rejected": -0.2655439078807831,
      "loss": 0.2187369167804718,
      "loss/core": 0.1985674947500229,
      "loss/preference_sft": 0.2756105661392212,
      "loss/reference_anchor": 0.3758276104927063,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.1124117374420166,
      "rewards/margins": 1.7423970699310303,
      "rewards/rejected": 1.3700147867202759,
      "ronpo/logit": 0.17423972487449646,
      "ronpo/residual": 0.18899571895599365,
      "ronpo/residual_abs": 0.2230168879032135,
      "ronpo/target": -0.014756004326045513,
      "ronpo/target_abs": 0.08157791197299957,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 145
    },
    {
      "drift/chosen_abs": 0.4021173417568207,
      "drift/rejected_abs": 0.11387000232934952,
      "epoch": 0.04767580452920143,
      "grad_norm": 2.390625,
      "learning_rate": 2.467414994379065e-08,
      "logits/chosen": -2.0298092365264893,
      "logits/rejected": -2.1323561668395996,
      "logps/chosen": -0.29558226466178894,
      "logps/rejected": -0.27471041679382324,
      "loss": 1.039489507675171,
      "loss/core": 1.0037477016448975,
      "loss/preference_sft": 0.286141037940979,
      "loss/reference_anchor": 0.6862256526947021,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 4.021132469177246,
      "rewards/margins": 2.884185314178467,
      "rewards/rejected": 1.136947751045227,
      "ronpo/logit": 0.28841856122016907,
      "ronpo/residual": 0.2740480601787567,
      "ronpo/residual_abs": 0.33974286913871765,
      "ronpo/target": 0.014370448887348175,
      "ronpo/target_abs": 0.08442968875169754,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 150
    },
    {
      "drift/chosen_abs": 0.3120030462741852,
      "drift/rejected_abs": 0.10412098467350006,
      "epoch": 0.04926499801350814,
      "grad_norm": 3.84375,
      "learning_rate": 2.4616876678501113e-08,
      "logits/chosen": -2.170553207397461,
      "logits/rejected": -2.1755318641662598,
      "logps/chosen": -0.265367329120636,
      "logps/rejected": -0.26216381788253784,
      "loss": 0.7199813723564148,
      "loss/core": 0.6925104260444641,
      "loss/preference_sft": 0.2665371000766754,
      "loss/reference_anchor": 0.522765040397644,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.119339942932129,
      "rewards/margins": 2.079453229904175,
      "rewards/rejected": 1.0398868322372437,
      "ronpo/logit": 0.207945317029953,
      "ronpo/residual": 0.22640779614448547,
      "ronpo/residual_abs": 0.26298147439956665,
      "ronpo/target": -0.018462475389242172,
      "ronpo/target_abs": 0.07875766605138779,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 155
    },
    {
      "drift/chosen_abs": 0.1994825154542923,
      "drift/rejected_abs": 0.06770407408475876,
      "epoch": 0.05085419149781486,
      "grad_norm": 856.0,
      "learning_rate": 2.4555046751436738e-08,
      "logits/chosen": -2.207547426223755,
      "logits/rejected": -2.225343942642212,
      "logps/chosen": -0.32042816281318665,
      "logps/rejected": -0.2738822400569916,
      "loss": 0.559381365776062,
      "loss/core": 0.5423927903175354,
      "loss/preference_sft": 0.2767319679260254,
      "loss/reference_anchor": 0.3120982050895691,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.9948251247406006,
      "rewards/margins": 1.318454384803772,
      "rewards/rejected": 0.6763707399368286,
      "ronpo/logit": 0.13184544444084167,
      "ronpo/residual": 0.1333637535572052,
      "ronpo/residual_abs": 0.19957098364830017,
      "ronpo/target": -0.0015183339128270745,
      "ronpo/target_abs": 0.09095166623592377,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 160
    },
    {
      "drift/chosen_abs": 0.13100005686283112,
      "drift/rejected_abs": 0.05130317807197571,
      "epoch": 0.052443384982121574,
      "grad_norm": 4.15625,
      "learning_rate": 2.4488683414304425e-08,
      "logits/chosen": -2.1318647861480713,
      "logits/rejected": -2.010211229324341,
      "logps/chosen": -0.28451108932495117,
      "logps/rejected": -0.29346317052841187,
      "loss": 0.1088658794760704,
      "loss/core": 0.10401229560375214,
      "loss/preference_sft": 0.2892284393310547,
      "loss/reference_anchor": 0.06814897060394287,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.3097032308578491,
      "rewards/margins": 0.8015567660331726,
      "rewards/rejected": 0.5081464052200317,
      "ronpo/logit": 0.08015568554401398,
      "ronpo/residual": 0.07226266711950302,
      "ronpo/residual_abs": 0.1403239220380783,
      "ronpo/target": 0.007893010973930359,
      "ronpo/target_abs": 0.07677415758371353,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 165
    },
    {
      "drift/chosen_abs": 0.317328542470932,
      "drift/rejected_abs": 0.16766056418418884,
      "epoch": 0.05403257846642829,
      "grad_norm": 1.078125,
      "learning_rate": 2.4417811623641206e-08,
      "logits/chosen": -2.027057647705078,
      "logits/rejected": -2.019274950027466,
      "logps/chosen": -0.2885628640651703,
      "logps/rejected": -0.2752649486064911,
      "loss": 0.4365611672401428,
      "loss/core": 0.4138968586921692,
      "loss/preference_sft": 0.28210192918777466,
      "loss/reference_anchor": 0.4250755310058594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.172837734222412,
      "rewards/margins": 1.4988738298416138,
      "rewards/rejected": 1.6739639043807983,
      "ronpo/logit": 0.14988741278648376,
      "ronpo/residual": 0.13528050482273102,
      "ronpo/residual_abs": 0.2508905827999115,
      "ronpo/target": 0.014606885612010956,
      "ronpo/target_abs": 0.07879899442195892,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 170
    },
    {
      "drift/chosen_abs": 0.19914527237415314,
      "drift/rejected_abs": 0.11187348514795303,
      "epoch": 0.055621771950735005,
      "grad_norm": 30.125,
      "learning_rate": 2.4342458031429115e-08,
      "logits/chosen": -2.262404680252075,
      "logits/rejected": -2.1995086669921875,
      "logps/chosen": -0.2932138741016388,
      "logps/rejected": -0.30438342690467834,
      "loss": 0.06648578494787216,
      "loss/core": 0.05691850930452347,
      "loss/preference_sft": 0.29610422253608704,
      "loss/reference_anchor": 0.16173511743545532,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9914528131484985,
      "rewards/margins": 0.8727973103523254,
      "rewards/rejected": 1.1186553239822388,
      "ronpo/logit": 0.08727972954511642,
      "ronpo/residual": 0.0997995063662529,
      "ronpo/residual_abs": 0.1461952179670334,
      "ronpo/target": -0.012519779615104198,
      "ronpo/target_abs": 0.08836988359689713,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 175
    },
    {
      "drift/chosen_abs": 0.4732588231563568,
      "drift/rejected_abs": 0.22694897651672363,
      "epoch": 0.057210965435041714,
      "grad_norm": 37.25,
      "learning_rate": 2.4262650975072444e-08,
      "logits/chosen": -2.074978828430176,
      "logits/rejected": -2.051633358001709,
      "logps/chosen": -0.2896614074707031,
      "logps/rejected": -0.28108644485473633,
      "loss": 0.9334848523139954,
      "loss/core": 0.8751218914985657,
      "loss/preference_sft": 0.2792913615703583,
      "loss/reference_anchor": 1.1393296718597412,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 4.732588291168213,
      "rewards/margins": 2.463618755340576,
      "rewards/rejected": 2.2689692974090576,
      "ronpo/logit": 0.24636192619800568,
      "ronpo/residual": 0.2580645680427551,
      "ronpo/residual_abs": 0.31068843603134155,
      "ronpo/target": -0.011702699586749077,
      "ronpo/target_abs": 0.07820998132228851,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 180
    },
    {
      "drift/chosen_abs": 0.12275053560733795,
      "drift/rejected_abs": 0.06439492851495743,
      "epoch": 0.05880015891934843,
      "grad_norm": 5.625,
      "learning_rate": 2.4178420466741218e-08,
      "logits/chosen": -2.226714849472046,
      "logits/rejected": -2.3291454315185547,
      "logps/chosen": -0.29459166526794434,
      "logps/rejected": -0.29199695587158203,
      "loss": 0.03509552404284477,
      "loss/core": 0.03178117424249649,
      "loss/preference_sft": 0.2933792173862457,
      "loss/reference_anchor": 0.036949075758457184,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.227303385734558,
      "rewards/margins": 0.5856597423553467,
      "rewards/rejected": 0.6416436433792114,
      "ronpo/logit": 0.058565981686115265,
      "ronpo/residual": 0.07432404160499573,
      "ronpo/residual_abs": 0.13376492261886597,
      "ronpo/target": -0.01575806364417076,
      "ronpo/target_abs": 0.10001115500926971,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 185
    },
    {
      "drift/chosen_abs": 0.12490232288837433,
      "drift/rejected_abs": 0.055704616010189056,
      "epoch": 0.060389352403655144,
      "grad_norm": 0.9296875,
      "learning_rate": 2.408979818208484e-08,
      "logits/chosen": -2.3149032592773438,
      "logits/rejected": -2.411414861679077,
      "logps/chosen": -0.30525779724121094,
      "logps/rejected": -0.3026205003261566,
      "loss": 0.052305638790130615,
      "loss/core": 0.04764842614531517,
      "loss/preference_sft": 0.3053823411464691,
      "loss/reference_anchor": 0.06260595470666885,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.247715950012207,
      "rewards/margins": 0.6911423802375793,
      "rewards/rejected": 0.5565735697746277,
      "ronpo/logit": 0.06911423802375793,
      "ronpo/residual": 0.0638674646615982,
      "ronpo/residual_abs": 0.12851160764694214,
      "ronpo/target": 0.005246774293482304,
      "ronpo/target_abs": 0.08141742646694183,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 190
    },
    {
      "drift/chosen_abs": 0.21187469363212585,
      "drift/rejected_abs": 0.11845757067203522,
      "epoch": 0.06197854588796186,
      "grad_norm": 6.75,
      "learning_rate": 2.3996817448320193e-08,
      "logits/chosen": -2.055734872817993,
      "logits/rejected": -2.031132698059082,
      "logps/chosen": -0.27869275212287903,
      "logps/rejected": -0.27192652225494385,
      "loss": 0.06286276876926422,
      "loss/core": 0.055714428424835205,
      "loss/preference_sft": 0.26679545640945435,
      "loss/reference_anchor": 0.11628717184066772,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.1181392669677734,
      "rewards/margins": 0.9341791868209839,
      "rewards/rejected": 1.1839600801467896,
      "ronpo/logit": 0.0934179276227951,
      "ronpo/residual": 0.09014447033405304,
      "ronpo/residual_abs": 0.14351728558540344,
      "ronpo/target": 0.0032734572887420654,
      "ronpo/target_abs": 0.08329368382692337,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 195
    },
    {
      "drift/chosen_abs": 0.2836345136165619,
      "drift/rejected_abs": 0.08946871757507324,
      "epoch": 0.06356773937226858,
      "grad_norm": 8.1875,
      "learning_rate": 2.389951323169861e-08,
      "logits/chosen": -2.040783166885376,
      "logits/rejected": -2.161843776702881,
      "logps/chosen": -0.2890108823776245,
      "logps/rejected": -0.2789885699748993,
      "loss": 0.4890855848789215,
      "loss/core": 0.4716506600379944,
      "loss/preference_sft": 0.28799134492874146,
      "loss/reference_anchor": 0.31989890336990356,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.835172176361084,
      "rewards/margins": 1.9416320323944092,
      "rewards/rejected": 0.8935402035713196,
      "ronpo/logit": 0.1941632330417633,
      "ronpo/residual": 0.18651123344898224,
      "ronpo/residual_abs": 0.23668105900287628,
      "ronpo/target": 0.007652005646377802,
      "ronpo/target_abs": 0.07496481388807297,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 200
    },
    {
      "drift/chosen_abs": 0.2938988208770752,
      "drift/rejected_abs": 0.10776060819625854,
      "epoch": 0.06515693285657528,
      "grad_norm": 94.0,
      "learning_rate": 2.3797922124356508e-08,
      "logits/chosen": -2.003150463104248,
      "logits/rejected": -2.0846657752990723,
      "logps/chosen": -0.28210049867630005,
      "logps/rejected": -0.28749504685401917,
      "loss": 0.755161464214325,
      "loss/core": 0.7267810106277466,
      "loss/preference_sft": 0.2828027009963989,
      "loss/reference_anchor": 0.5393306016921997,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.935922861099243,
      "rewards/margins": 1.859521508216858,
      "rewards/rejected": 1.0764009952545166,
      "ronpo/logit": 0.18595215678215027,
      "ronpo/residual": 0.21726493537425995,
      "ronpo/residual_abs": 0.2416853904724121,
      "ronpo/target": -0.03131275624036789,
      "ronpo/target_abs": 0.07854778319597244,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 205
    },
    {
      "drift/chosen_abs": 0.21909186244010925,
      "drift/rejected_abs": 0.05819239094853401,
      "epoch": 0.066746126340882,
      "grad_norm": 2.984375,
      "learning_rate": 2.3692082330554585e-08,
      "logits/chosen": -2.2509193420410156,
      "logits/rejected": -2.169978141784668,
      "logps/chosen": -0.307937890291214,
      "logps/rejected": -0.31472259759902954,
      "loss": 1.0144634246826172,
      "loss/core": 0.9857050776481628,
      "loss/preference_sft": 0.31031107902526855,
      "loss/reference_anchor": 0.544138491153717,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.1909189224243164,
      "rewards/margins": 1.6100225448608398,
      "rewards/rejected": 0.5808963179588318,
      "ronpo/logit": 0.1610022485256195,
      "ronpo/residual": 0.17502841353416443,
      "ronpo/residual_abs": 0.22845324873924255,
      "ronpo/target": -0.01402618270367384,
      "ronpo/target_abs": 0.08586090058088303,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 210
    },
    {
      "drift/chosen_abs": 0.32701677083969116,
      "drift/rejected_abs": 0.16467604041099548,
      "epoch": 0.06833531982518871,
      "grad_norm": 1.0234375,
      "learning_rate": 2.3582033652310766e-08,
      "logits/chosen": -2.1693928241729736,
      "logits/rejected": -2.157107353210449,
      "logps/chosen": -0.26543909311294556,
      "logps/rejected": -0.2721554636955261,
      "loss": 0.30226364731788635,
      "loss/core": 0.27254778146743774,
      "loss/preference_sft": 0.2698844075202942,
      "loss/reference_anchor": 0.5673290491104126,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 3.270167827606201,
      "rewards/margins": 1.624214768409729,
      "rewards/rejected": 1.6459531784057617,
      "ronpo/logit": 0.16242146492004395,
      "ronpo/residual": 0.15607237815856934,
      "ronpo/residual_abs": 0.21735577285289764,
      "ronpo/target": 0.006349098868668079,
      "ronpo/target_abs": 0.0811850056052208,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 215
    },
    {
      "drift/chosen_abs": 0.1198434829711914,
      "drift/rejected_abs": 0.08144821226596832,
      "epoch": 0.06992451330949544,
      "grad_norm": 0.71875,
      "learning_rate": 2.3467817474432273e-08,
      "logits/chosen": -2.216386079788208,
      "logits/rejected": -2.084725856781006,
      "logps/chosen": -0.30442219972610474,
      "logps/rejected": -0.29973161220550537,
      "loss": 0.05327620357275009,
      "loss/core": 0.04937787353992462,
      "loss/preference_sft": 0.3065362572669983,
      "loss/reference_anchor": 0.04731312766671181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1957999467849731,
      "rewards/margins": 0.38699817657470703,
      "rewards/rejected": 0.8088018298149109,
      "ronpo/logit": 0.038699813187122345,
      "ronpo/residual": 0.05509736388921738,
      "ronpo/residual_abs": 0.13401086628437042,
      "ronpo/target": -0.016397548839449883,
      "ronpo/target_abs": 0.08902524411678314,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 220
    },
    {
      "drift/chosen_abs": 0.23227110505104065,
      "drift/rejected_abs": 0.10326921939849854,
      "epoch": 0.07151370679380215,
      "grad_norm": 6.21875,
      "learning_rate": 2.3349476748952507e-08,
      "logits/chosen": -2.1769394874572754,
      "logits/rejected": -2.194044351577759,
      "logps/chosen": -0.28651732206344604,
      "logps/rejected": -0.28518351912498474,
      "loss": 0.1896631419658661,
      "loss/core": 0.17952075600624084,
      "loss/preference_sft": 0.27889543771743774,
      "loss/reference_anchor": 0.1749579906463623,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.32271146774292,
      "rewards/margins": 1.2900190353393555,
      "rewards/rejected": 1.0326921939849854,
      "ronpo/logit": 0.1290019154548645,
      "ronpo/residual": 0.11321617662906647,
      "ronpo/residual_abs": 0.17566923797130585,
      "ronpo/target": 0.015785735100507736,
      "ronpo/target_abs": 0.07616916298866272,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 225
    },
    {
      "drift/chosen_abs": 0.26629695296287537,
      "drift/rejected_abs": 0.09605975449085236,
      "epoch": 0.07310290027810885,
      "grad_norm": 2.546875,
      "learning_rate": 2.3227055978978543e-08,
      "logits/chosen": -2.242928981781006,
      "logits/rejected": -2.2605459690093994,
      "logps/chosen": -0.26110613346099854,
      "logps/rejected": -0.2610822319984436,
      "loss": 0.419999897480011,
      "loss/core": 0.40420547127723694,
      "loss/preference_sft": 0.2564738988876343,
      "loss/reference_anchor": 0.2902412414550781,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.662400960922241,
      "rewards/margins": 1.7032798528671265,
      "rewards/rejected": 0.9591208696365356,
      "ronpo/logit": 0.17032800614833832,
      "ronpo/residual": 0.1783141791820526,
      "ronpo/residual_abs": 0.22243869304656982,
      "ronpo/target": -0.00798617023974657,
      "ronpo/target_abs": 0.09001145511865616,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 230
    },
    {
      "drift/chosen_abs": 0.46215638518333435,
      "drift/rejected_abs": 0.23638899624347687,
      "epoch": 0.07469209376241558,
      "grad_norm": 50.5,
      "learning_rate": 2.310060120195532e-08,
      "logits/chosen": -2.113866090774536,
      "logits/rejected": -2.116377115249634,
      "logps/chosen": -0.2798251509666443,
      "logps/rejected": -0.2868059575557709,
      "loss": 1.1930959224700928,
      "loss/core": 1.1315306425094604,
      "loss/preference_sft": 0.27541080117225647,
      "loss/reference_anchor": 1.20376455783844,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.620245933532715,
      "rewards/margins": 2.3461127281188965,
      "rewards/rejected": 2.2741329669952393,
      "ronpo/logit": 0.23461131751537323,
      "ronpo/residual": 0.237228661775589,
      "ronpo/residual_abs": 0.40296974778175354,
      "ronpo/target": -0.0026173379737883806,
      "ronpo/target_abs": 0.08310168236494064,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 235
    },
    {
      "drift/chosen_abs": 0.2902994751930237,
      "drift/rejected_abs": 0.16377928853034973,
      "epoch": 0.07628128724672228,
      "grad_norm": 204.0,
      "learning_rate": 2.2970159972352866e-08,
      "logits/chosen": -2.1415343284606934,
      "logits/rejected": -2.2179665565490723,
      "logps/chosen": -0.2673754096031189,
      "logps/rejected": -0.2882601320743561,
      "loss": 0.22219261527061462,
      "loss/core": 0.19527706503868103,
      "loss/preference_sft": 0.28320834040641785,
      "loss/reference_anchor": 0.5099908113479614,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.9026601314544678,
      "rewards/margins": 1.265976905822754,
      "rewards/rejected": 1.636683464050293,
      "ronpo/logit": 0.12659768760204315,
      "ronpo/residual": 0.11025957018136978,
      "ronpo/residual_abs": 0.18036124110221863,
      "ronpo/target": 0.016338128596544266,
      "ronpo/target_abs": 0.087813600897789,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 240
    },
    {
      "drift/chosen_abs": 0.18804895877838135,
      "drift/rejected_abs": 0.07305173575878143,
      "epoch": 0.07787048073102901,
      "grad_norm": 2.1875,
      "learning_rate": 2.2835781343782966e-08,
      "logits/chosen": -2.182067394256592,
      "logits/rejected": -2.2002573013305664,
      "logps/chosen": -0.29913002252578735,
      "logps/rejected": -0.3051024079322815,
      "loss": 0.2962071895599365,
      "loss/core": 0.28634169697761536,
      "loss/preference_sft": 0.3018152117729187,
      "loss/reference_anchor": 0.1671280860900879,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8793338537216187,
      "rewards/margins": 1.174026370048523,
      "rewards/rejected": 0.7053073644638062,
      "ronpo/logit": 0.11740265041589737,
      "ronpo/residual": 0.12060768902301788,
      "ronpo/residual_abs": 0.1809166967868805,
      "ronpo/target": -0.0032050427980720997,
      "ronpo/target_abs": 0.09324783831834793,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 245
    },
    {
      "drift/chosen_abs": 0.362274706363678,
      "drift/rejected_abs": 0.08525294065475464,
      "epoch": 0.07945967421533572,
      "grad_norm": 7.25,
      "learning_rate": 2.269751585055215e-08,
      "logits/chosen": -2.1129937171936035,
      "logits/rejected": -2.090263843536377,
      "logps/chosen": -0.3094167709350586,
      "logps/rejected": -0.31022873520851135,
      "loss": 1.225620985031128,
      "loss/core": 1.189970850944519,
      "loss/preference_sft": 0.3118210434913635,
      "loss/reference_anchor": 0.6818209886550903,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 3.6219310760498047,
      "rewards/margins": 2.770771026611328,
      "rewards/rejected": 0.851159930229187,
      "ronpo/logit": 0.2770771086215973,
      "ronpo/residual": 0.2786293029785156,
      "ronpo/residual_abs": 0.33625951409339905,
      "ronpo/target": -0.0015522129833698273,
      "ronpo/target_abs": 0.08379620313644409,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 250
    },
    {
      "drift/chosen_abs": 0.4456920623779297,
      "drift/rejected_abs": 0.19766589999198914,
      "epoch": 0.08104886769964244,
      "grad_norm": 7.0625,
      "learning_rate": 2.2555415488657776e-08,
      "logits/chosen": -2.1046249866485596,
      "logits/rejected": -2.107532262802124,
      "logps/chosen": -0.2630499303340912,
      "logps/rejected": -0.27011799812316895,
      "loss": 0.7377499341964722,
      "loss/core": 0.6888483166694641,
      "loss/preference_sft": 0.27254000306129456,
      "loss/reference_anchor": 0.9507792592048645,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 4.456816673278809,
      "rewards/margins": 2.533341407775879,
      "rewards/rejected": 1.9234750270843506,
      "ronpo/logit": 0.2533341646194458,
      "ronpo/residual": 0.25843900442123413,
      "ronpo/residual_abs": 0.31028133630752563,
      "ronpo/target": -0.005104861222207546,
      "ronpo/target_abs": 0.08672384172677994,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 255
    },
    {
      "drift/chosen_abs": 0.32755011320114136,
      "drift/rejected_abs": 0.08800020813941956,
      "epoch": 0.08263806118394915,
      "grad_norm": 972.0,
      "learning_rate": 2.240953369623447e-08,
      "logits/chosen": -2.193181276321411,
      "logits/rejected": -2.168120861053467,
      "logps/chosen": -0.2840411067008972,
      "logps/rejected": -0.28209537267684937,
      "loss": 0.8339856266975403,
      "loss/core": 0.8076542615890503,
      "loss/preference_sft": 0.29532259702682495,
      "loss/reference_anchor": 0.4970928728580475,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2736563682556152,
      "rewards/margins": 2.4421114921569824,
      "rewards/rejected": 0.8315451741218567,
      "ronpo/logit": 0.24421115219593048,
      "ronpo/residual": 0.2369440346956253,
      "ronpo/residual_abs": 0.29261597990989685,
      "ronpo/target": 0.007267133332788944,
      "ronpo/target_abs": 0.08349494636058807,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 260
    },
    {
      "drift/chosen_abs": 0.18214836716651917,
      "drift/rejected_abs": 0.0814029648900032,
      "epoch": 0.08422725466825585,
      "grad_norm": 16.0,
      "learning_rate": 2.225992533345824e-08,
      "logits/chosen": -2.155879497528076,
      "logits/rejected": -2.1550204753875732,
      "logps/chosen": -0.284726619720459,
      "logps/rejected": -0.29086941480636597,
      "loss": 0.08231814950704575,
      "loss/core": 0.07635300606489182,
      "loss/preference_sft": 0.2914847135543823,
      "loss/reference_anchor": 0.0901542603969574,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8214836120605469,
      "rewards/margins": 1.0148638486862183,
      "rewards/rejected": 0.8066196441650391,
      "ronpo/logit": 0.10148639976978302,
      "ronpo/residual": 0.09669841825962067,
      "ronpo/residual_abs": 0.16233278810977936,
      "ronpo/target": 0.004787982441484928,
      "ronpo/target_abs": 0.09343020617961884,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 265
    },
    {
      "drift/chosen_abs": 0.2922074496746063,
      "drift/rejected_abs": 0.16019122302532196,
      "epoch": 0.08581644815256258,
      "grad_norm": 2.734375,
      "learning_rate": 2.210664666191579e-08,
      "logits/chosen": -2.2221102714538574,
      "logits/rejected": -2.1809840202331543,
      "logps/chosen": -0.2800453305244446,
      "logps/rejected": -0.28999465703964233,
      "loss": 0.357001394033432,
      "loss/core": 0.3204185366630554,
      "loss/preference_sft": 0.28067368268966675,
      "loss/reference_anchor": 0.7035894393920898,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9200611114501953,
      "rewards/margins": 1.3199583292007446,
      "rewards/rejected": 1.6001027822494507,
      "ronpo/logit": 0.13199582695960999,
      "ronpo/residual": 0.11801192909479141,
      "ronpo/residual_abs": 0.18810518085956573,
      "ronpo/target": 0.013983900658786297,
      "ronpo/target_abs": 0.08693692088127136,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 270
    },
    {
      "drift/chosen_abs": 0.17948992550373077,
      "drift/rejected_abs": 0.10400255769491196,
      "epoch": 0.08740564163686929,
      "grad_norm": 6.875,
      "learning_rate": 2.1949755323446846e-08,
      "logits/chosen": -2.275407314300537,
      "logits/rejected": -2.3497166633605957,
      "logps/chosen": -0.27847951650619507,
      "logps/rejected": -0.2680722773075104,
      "loss": 0.048314549028873444,
      "loss/core": 0.041888050734996796,
      "loss/preference_sft": 0.27859556674957275,
      "loss/reference_anchor": 0.1006704568862915,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.794899582862854,
      "rewards/margins": 0.7595475912094116,
      "rewards/rejected": 1.0353518724441528,
      "ronpo/logit": 0.07595475763082504,
      "ronpo/residual": 0.05474218726158142,
      "ronpo/residual_abs": 0.13210758566856384,
      "ronpo/target": 0.02121257223188877,
      "ronpo/target_abs": 0.0803799256682396,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 275
    },
    {
      "drift/chosen_abs": 0.1700095236301422,
      "drift/rejected_abs": 0.11016277223825455,
      "epoch": 0.08899483512117601,
      "grad_norm": 52.0,
      "learning_rate": 2.1789310318467428e-08,
      "logits/chosen": -2.180828094482422,
      "logits/rejected": -2.182821750640869,
      "logps/chosen": -0.2840616703033447,
      "logps/rejected": -0.2911655306816101,
      "loss": 0.10237391293048859,
      "loss/core": 0.09611751139163971,
      "loss/preference_sft": 0.28340545296669006,
      "loss/reference_anchor": 0.09678752720355988,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.6990578174591064,
      "rewards/margins": 0.5979472398757935,
      "rewards/rejected": 1.1011104583740234,
      "ronpo/logit": 0.059794723987579346,
      "ronpo/residual": 0.052797090262174606,
      "ronpo/residual_abs": 0.1553640067577362,
      "ronpo/target": 0.006997637450695038,
      "ronpo/target_abs": 0.07295690476894379,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 280
    },
    {
      "drift/chosen_abs": 0.43757256865501404,
      "drift/rejected_abs": 0.14037099480628967,
      "epoch": 0.09058402860548272,
      "grad_norm": 10.9375,
      "learning_rate": 2.162537198378218e-08,
      "logits/chosen": -2.2235183715820312,
      "logits/rejected": -2.194262981414795,
      "logps/chosen": -0.2805050313472748,
      "logps/rejected": -0.2911986708641052,
      "loss": 1.5213605165481567,
      "loss/core": 1.4580514430999756,
      "loss/preference_sft": 0.28713876008987427,
      "loss/reference_anchor": 1.237465739250183,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 4.375100135803223,
      "rewards/margins": 2.9718849658966064,
      "rewards/rejected": 1.4032156467437744,
      "ronpo/logit": 0.29718852043151855,
      "ronpo/residual": 0.30155670642852783,
      "ronpo/residual_abs": 0.34738093614578247,
      "ronpo/target": -0.004368200898170471,
      "ronpo/target_abs": 0.0792548879981041,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 285
    },
    {
      "drift/chosen_abs": 0.2998009920120239,
      "drift/rejected_abs": 0.13739660382270813,
      "epoch": 0.09217322208978942,
      "grad_norm": 133.0,
      "learning_rate": 2.14580019698942e-08,
      "logits/chosen": -2.0153133869171143,
      "logits/rejected": -2.089466094970703,
      "logps/chosen": -0.27183207869529724,
      "logps/rejected": -0.2848559617996216,
      "loss": 0.26796045899391174,
      "loss/core": 0.25352850556373596,
      "loss/preference_sft": 0.2748119533061981,
      "loss/reference_anchor": 0.261157751083374,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.99642276763916,
      "rewards/margins": 1.623225212097168,
      "rewards/rejected": 1.3731976747512817,
      "ronpo/logit": 0.162322536110878,
      "ronpo/residual": 0.1448330283164978,
      "ronpo/residual_abs": 0.23569996654987335,
      "ronpo/target": 0.01748950406908989,
      "ronpo/target_abs": 0.0773894339799881,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 290
    },
    {
      "drift/chosen_abs": 0.19523122906684875,
      "drift/rejected_abs": 0.07123854756355286,
      "epoch": 0.09376241557409615,
      "grad_norm": 178.0,
      "learning_rate": 2.128726321782077e-08,
      "logits/chosen": -2.1933460235595703,
      "logits/rejected": -2.1863884925842285,
      "logps/chosen": -0.30095577239990234,
      "logps/rejected": -0.3060711622238159,
      "loss": 0.24075086414813995,
      "loss/core": 0.2300082892179489,
      "loss/preference_sft": 0.30148637294769287,
      "loss/reference_anchor": 0.18470288813114166,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.951626181602478,
      "rewards/margins": 1.244182825088501,
      "rewards/rejected": 0.7074434161186218,
      "ronpo/logit": 0.12441827356815338,
      "ronpo/residual": 0.1209135428071022,
      "ronpo/residual_abs": 0.17452454566955566,
      "ronpo/target": 0.003504746360704303,
      "ronpo/target_abs": 0.07728487253189087,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 295
    },
    {
      "drift/chosen_abs": 0.13675720989704132,
      "drift/rejected_abs": 0.0812956839799881,
      "epoch": 0.09535160905840286,
      "grad_norm": 5.84375,
      "learning_rate": 2.111321993542385e-08,
      "logits/chosen": -2.407482862472534,
      "logits/rejected": -2.4131016731262207,
      "logps/chosen": -0.29553696513175964,
      "logps/rejected": -0.28668391704559326,
      "loss": 0.03661810979247093,
      "loss/core": 0.03264369070529938,
      "loss/preference_sft": 0.2879699468612671,
      "loss/reference_anchor": 0.05069132521748543,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.3664600849151611,
      "rewards/margins": 0.5552068948745728,
      "rewards/rejected": 0.8112531900405884,
      "ronpo/logit": 0.055520690977573395,
      "ronpo/residual": 0.053940463811159134,
      "ronpo/residual_abs": 0.12251658737659454,
      "ronpo/target": 0.0015802305424585938,
      "ronpo/target_abs": 0.09203118085861206,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 300
    },
    {
      "drift/chosen_abs": 0.2224307805299759,
      "drift/rejected_abs": 0.10239239037036896,
      "epoch": 0.09694080254270958,
      "grad_norm": 3.28125,
      "learning_rate": 2.0935937573264096e-08,
      "logits/chosen": -2.0765604972839355,
      "logits/rejected": -2.104783296585083,
      "logps/chosen": -0.30479732155799866,
      "logps/rejected": -0.29314523935317993,
      "loss": 0.1927420198917389,
      "loss/core": 0.17845560610294342,
      "loss/preference_sft": 0.3036041855812073,
      "loss/reference_anchor": 0.25536784529685974,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.221867322921753,
      "rewards/margins": 1.2011425495147705,
      "rewards/rejected": 1.0207250118255615,
      "ronpo/logit": 0.12011425197124481,
      "ronpo/residual": 0.12593664228916168,
      "ronpo/residual_abs": 0.18047745525836945,
      "ronpo/target": -0.005822394508868456,
      "ronpo/target_abs": 0.0825614184141159,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 305
    },
    {
      "drift/chosen_abs": 0.2303655445575714,
      "drift/rejected_abs": 0.14673031866550446,
      "epoch": 0.09852999602701629,
      "grad_norm": 278.0,
      "learning_rate": 2.0755482799987594e-08,
      "logits/chosen": -2.0974299907684326,
      "logits/rejected": -1.9897010326385498,
      "logps/chosen": -0.30457696318626404,
      "logps/rejected": -0.3037092089653015,
      "loss": 0.09838415682315826,
      "loss/core": 0.08123552054166794,
      "loss/preference_sft": 0.31207767128944397,
      "loss/reference_anchor": 0.3117648661136627,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.3016207218170166,
      "rewards/margins": 0.8351632952690125,
      "rewards/rejected": 1.4664572477340698,
      "ronpo/logit": 0.08351632952690125,
      "ronpo/residual": 0.08266756683588028,
      "ronpo/residual_abs": 0.16260148584842682,
      "ronpo/target": 0.0008487681043334305,
      "ronpo/target_abs": 0.08333858102560043,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 310
    },
    {
      "drift/chosen_abs": 0.21899929642677307,
      "drift/rejected_abs": 0.17108279466629028,
      "epoch": 0.10011918951132301,
      "grad_norm": 1.8046875,
      "learning_rate": 2.0571923477254526e-08,
      "logits/chosen": -2.210271120071411,
      "logits/rejected": -2.2432236671447754,
      "logps/chosen": -0.28812751173973083,
      "logps/rejected": -0.2839018404483795,
      "loss": 0.12934687733650208,
      "loss/core": 0.08804286271333694,
      "loss/preference_sft": 0.28392842411994934,
      "loss/reference_anchor": 0.7976871728897095,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.1899399757385254,
      "rewards/margins": 0.48040738701820374,
      "rewards/rejected": 1.7095324993133545,
      "ronpo/logit": 0.04804074019193649,
      "ronpo/residual": 0.03702558949589729,
      "ronpo/residual_abs": 0.14168190956115723,
      "ronpo/target": 0.011015147902071476,
      "ronpo/target_abs": 0.08301273733377457,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 315
    },
    {
      "drift/chosen_abs": 0.30637550354003906,
      "drift/rejected_abs": 0.15101422369480133,
      "epoch": 0.10170838299562972,
      "grad_norm": 7.90625,
      "learning_rate": 2.038532863421914e-08,
      "logits/chosen": -2.0474765300750732,
      "logits/rejected": -2.046475648880005,
      "logps/chosen": -0.2672887146472931,
      "logps/rejected": -0.2681429982185364,
      "loss": 0.43184512853622437,
      "loss/core": 0.39570650458335876,
      "loss/preference_sft": 0.26500943303108215,
      "loss/reference_anchor": 0.6962720155715942,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.0628886222839355,
      "rewards/margins": 1.5545833110809326,
      "rewards/rejected": 1.5083054304122925,
      "ronpo/logit": 0.15545836091041565,
      "ronpo/residual": 0.15610265731811523,
      "ronpo/residual_abs": 0.2172074019908905,
      "ronpo/target": -0.000644296407699585,
      "ronpo/target_abs": 0.08082158118486404,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 320
    },
    {
      "drift/chosen_abs": 0.2718430161476135,
      "drift/rejected_abs": 0.06493407487869263,
      "epoch": 0.10329757647993643,
      "grad_norm": 23.875,
      "learning_rate": 2.0195768441570726e-08,
      "logits/chosen": -2.213127374649048,
      "logits/rejected": -2.2380900382995605,
      "logps/chosen": -0.3036438524723053,
      "logps/rejected": -0.2892650365829468,
      "loss": 1.196415901184082,
      "loss/core": 1.1635918617248535,
      "loss/preference_sft": 0.2940658628940582,
      "loss/reference_anchor": 0.6270740032196045,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.717806100845337,
      "rewards/margins": 2.0749688148498535,
      "rewards/rejected": 0.642837643623352,
      "ronpo/logit": 0.20749685168266296,
      "ronpo/residual": 0.22607414424419403,
      "ronpo/residual_abs": 0.2697368264198303,
      "ronpo/target": -0.018577273935079575,
      "ronpo/target_abs": 0.09064265340566635,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 325
    },
    {
      "drift/chosen_abs": 0.2503616511821747,
      "drift/rejected_abs": 0.12431076914072037,
      "epoch": 0.10488676996424315,
      "grad_norm": 10.875,
      "learning_rate": 2.0003314185145307e-08,
      "logits/chosen": -2.3221628665924072,
      "logits/rejected": -2.322690010070801,
      "logps/chosen": -0.26718512177467346,
      "logps/rejected": -0.26268628239631653,
      "loss": 0.566504180431366,
      "loss/core": 0.5484728813171387,
      "loss/preference_sft": 0.255176305770874,
      "loss/reference_anchor": 0.3351089656352997,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.5033228397369385,
      "rewards/margins": 1.2658904790878296,
      "rewards/rejected": 1.2374324798583984,
      "ronpo/logit": 0.12658903002738953,
      "ronpo/residual": 0.1411885768175125,
      "ronpo/residual_abs": 0.2634466588497162,
      "ronpo/target": -0.014599531888961792,
      "ronpo/target_abs": 0.09172377735376358,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 330
    },
    {
      "drift/chosen_abs": 0.445903480052948,
      "drift/rejected_abs": 0.21055002510547638,
      "epoch": 0.10647596344854986,
      "grad_norm": 12.875,
      "learning_rate": 1.9808038239117914e-08,
      "logits/chosen": -1.991874098777771,
      "logits/rejected": -1.9406774044036865,
      "logps/chosen": -0.2943147122859955,
      "logps/rejected": -0.294437974691391,
      "loss": 0.7760679721832275,
      "loss/core": 0.7043066024780273,
      "loss/preference_sft": 0.2942696809768677,
      "loss/reference_anchor": 1.4057997465133667,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 4.458679676055908,
      "rewards/margins": 2.356698513031006,
      "rewards/rejected": 2.1019809246063232,
      "ronpo/logit": 0.23566989600658417,
      "ronpo/residual": 0.2460823804140091,
      "ronpo/residual_abs": 0.30592185258865356,
      "ronpo/target": -0.010412474162876606,
      "ronpo/target_abs": 0.07370901107788086,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 335
    },
    {
      "drift/chosen_abs": 0.45900607109069824,
      "drift/rejected_abs": 0.16360488533973694,
      "epoch": 0.10806515693285658,
      "grad_norm": 82.0,
      "learning_rate": 1.9610014038785648e-08,
      "logits/chosen": -1.8805198669433594,
      "logits/rejected": -1.9393765926361084,
      "logps/chosen": -0.28708821535110474,
      "logps/rejected": -0.2747197151184082,
      "loss": 1.0118316411972046,
      "loss/core": 0.9642781019210815,
      "loss/preference_sft": 0.27820974588394165,
      "loss/reference_anchor": 0.9232500791549683,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.589595794677734,
      "rewards/margins": 2.960404396057129,
      "rewards/rejected": 1.6291908025741577,
      "ronpo/logit": 0.29604044556617737,
      "ronpo/residual": 0.3044920861721039,
      "ronpo/residual_abs": 0.3554214537143707,
      "ronpo/target": -0.008451665751636028,
      "ronpo/target_abs": 0.08010601997375488,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 340
    },
    {
      "drift/chosen_abs": 0.24650216102600098,
      "drift/rejected_abs": 0.1208873763680458,
      "epoch": 0.10965435041716329,
      "grad_norm": 58.5,
      "learning_rate": 1.9409316052951656e-08,
      "logits/chosen": -2.178605556488037,
      "logits/rejected": -2.2515740394592285,
      "logps/chosen": -0.2723388671875,
      "logps/rejected": -0.2781441807746887,
      "loss": 0.12823966145515442,
      "loss/core": 0.11076222360134125,
      "loss/preference_sft": 0.28036752343177795,
      "loss/reference_anchor": 0.32151204347610474,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.4640796184539795,
      "rewards/margins": 1.2876027822494507,
      "rewards/rejected": 1.1764768362045288,
      "ronpo/logit": 0.12876029312610626,
      "ronpo/residual": 0.12763699889183044,
      "ronpo/residual_abs": 0.17462731897830963,
      "ronpo/target": 0.0011232979595661163,
      "ronpo/target_abs": 0.07203267514705658,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 345
    },
    {
      "drift/chosen_abs": 0.4136660099029541,
      "drift/rejected_abs": 0.16482646763324738,
      "epoch": 0.11124354390147001,
      "grad_norm": 1.8359375,
      "learning_rate": 1.920601975592047e-08,
      "logits/chosen": -2.267277240753174,
      "logits/rejected": -2.2952818870544434,
      "logps/chosen": -0.23398800194263458,
      "logps/rejected": -0.2486116588115692,
      "loss": 1.0056898593902588,
      "loss/core": 0.9502155184745789,
      "loss/preference_sft": 0.24102886021137238,
      "loss/reference_anchor": 1.0853848457336426,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.136660099029541,
      "rewards/margins": 2.506661891937256,
      "rewards/rejected": 1.629997968673706,
      "ronpo/logit": 0.25066617131233215,
      "ronpo/residual": 0.24287991225719452,
      "ronpo/residual_abs": 0.3027934730052948,
      "ronpo/target": 0.007786286063492298,
      "ronpo/target_abs": 0.08268611133098602,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 350
    },
    {
      "drift/chosen_abs": 0.28172412514686584,
      "drift/rejected_abs": 0.10272540897130966,
      "epoch": 0.11283273738577672,
      "grad_norm": 14.875,
      "learning_rate": 1.900020159911519e-08,
      "logits/chosen": -2.1245241165161133,
      "logits/rejected": -2.0237369537353516,
      "logps/chosen": -0.2827402949333191,
      "logps/rejected": -0.2766692042350769,
      "loss": 0.41723641753196716,
      "loss/core": 0.4000471234321594,
      "loss/preference_sft": 0.2682899832725525,
      "loss/reference_anchor": 0.31695735454559326,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.8172414302825928,
      "rewards/margins": 1.7916021347045898,
      "rewards/rejected": 1.025639533996582,
      "ronpo/logit": 0.1791602075099945,
      "ronpo/residual": 0.16192427277565002,
      "ronpo/residual_abs": 0.23042872548103333,
      "ronpo/target": 0.01723591983318329,
      "ronpo/target_abs": 0.08204632252454758,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 355
    },
    {
      "drift/chosen_abs": 0.21779580414295197,
      "drift/rejected_abs": 0.08848261833190918,
      "epoch": 0.11442193087008343,
      "grad_norm": 11.5,
      "learning_rate": 1.8791938982327247e-08,
      "logits/chosen": -2.206521511077881,
      "logits/rejected": -2.2727904319763184,
      "logps/chosen": -0.2671610713005066,
      "logps/rejected": -0.2585466206073761,
      "loss": 0.09771557152271271,
      "loss/core": 0.09170512855052948,
      "loss/preference_sft": 0.2634662985801697,
      "loss/reference_anchor": 0.09386207908391953,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1769795417785645,
      "rewards/margins": 1.2925890684127808,
      "rewards/rejected": 0.8843904733657837,
      "ronpo/logit": 0.1292589008808136,
      "ronpo/residual": 0.1271723061800003,
      "ronpo/residual_abs": 0.18293924629688263,
      "ronpo/target": 0.002086599823087454,
      "ronpo/target_abs": 0.07021833956241608,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 360
    },
    {
      "drift/chosen_abs": 0.20787136256694794,
      "drift/rejected_abs": 0.13100872933864594,
      "epoch": 0.11601112435439015,
      "grad_norm": 2.125,
      "learning_rate": 1.85813102246095e-08,
      "logits/chosen": -2.0922367572784424,
      "logits/rejected": -2.1322176456451416,
      "logps/chosen": -0.2858452796936035,
      "logps/rejected": -0.3246702551841736,
      "loss": 0.06760017573833466,
      "loss/core": 0.047362055629491806,
      "loss/preference_sft": 0.28588929772377014,
      "loss/reference_anchor": 0.37617334723472595,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0768673419952393,
      "rewards/margins": 0.791536271572113,
      "rewards/rejected": 1.2853310108184814,
      "ronpo/logit": 0.0791536197066307,
      "ronpo/residual": 0.07149665057659149,
      "ronpo/residual_abs": 0.13622984290122986,
      "ronpo/target": 0.007656970527023077,
      "ronpo/target_abs": 0.09360074996948242,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 365
    },
    {
      "drift/chosen_abs": 0.31859681010246277,
      "drift/rejected_abs": 0.21548397839069366,
      "epoch": 0.11760031783869686,
      "grad_norm": 20.375,
      "learning_rate": 1.8368394534823633e-08,
      "logits/chosen": -2.136131763458252,
      "logits/rejected": -2.168311595916748,
      "logps/chosen": -0.27841588854789734,
      "logps/rejected": -0.28144991397857666,
      "loss": 0.16637016832828522,
      "loss/core": 0.10749348253011703,
      "loss/preference_sft": 0.2804015874862671,
      "loss/reference_anchor": 1.1494934558868408,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.1855576038360596,
      "rewards/margins": 1.0312860012054443,
      "rewards/rejected": 2.1542716026306152,
      "ronpo/logit": 0.10312861204147339,
      "ronpo/residual": 0.0984123945236206,
      "ronpo/residual_abs": 0.17065349221229553,
      "ronpo/target": 0.004716214723885059,
      "ronpo/target_abs": 0.08422146737575531,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 370
    },
    {
      "drift/chosen_abs": 0.21396997570991516,
      "drift/rejected_abs": 0.09890774637460709,
      "epoch": 0.11918951132300358,
      "grad_norm": 19.5,
      "learning_rate": 1.815327198185297e-08,
      "logits/chosen": -2.278458833694458,
      "logits/rejected": -2.3031888008117676,
      "logps/chosen": -0.29324546456336975,
      "logps/rejected": -0.28806382417678833,
      "loss": 0.13843554258346558,
      "loss/core": 0.12961502373218536,
      "loss/preference_sft": 0.28800949454307556,
      "loss/reference_anchor": 0.14760896563529968,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.1395039558410645,
      "rewards/margins": 1.1516611576080322,
      "rewards/rejected": 0.9878427386283875,
      "ronpo/logit": 0.11516612768173218,
      "ronpo/residual": 0.10511883348226547,
      "ronpo/residual_abs": 0.1740320473909378,
      "ronpo/target": 0.01004729326814413,
      "ronpo/target_abs": 0.08315954357385635,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 375
    },
    {
      "drift/chosen_abs": 0.2743528187274933,
      "drift/rejected_abs": 0.09948831051588058,
      "epoch": 0.12077870480731029,
      "grad_norm": 0.81640625,
      "learning_rate": 1.793602346449181e-08,
      "logits/chosen": -2.158848524093628,
      "logits/rejected": -2.1761603355407715,
      "logps/chosen": -0.2898809313774109,
      "logps/rejected": -0.28347843885421753,
      "loss": 0.5179681181907654,
      "loss/core": 0.49899888038635254,
      "loss/preference_sft": 0.28954416513442993,
      "loss/reference_anchor": 0.35042867064476013,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.7419450283050537,
      "rewards/margins": 1.7553457021713257,
      "rewards/rejected": 0.9865990877151489,
      "ronpo/logit": 0.17553459107875824,
      "ronpo/residual": 0.17026254534721375,
      "ronpo/residual_abs": 0.2293941080570221,
      "ronpo/target": 0.0052720606327056885,
      "ronpo/target_abs": 0.0783097967505455,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 380
    },
    {
      "drift/chosen_abs": 0.2517983317375183,
      "drift/rejected_abs": 0.14854536950588226,
      "epoch": 0.12236789829161701,
      "grad_norm": 63.5,
      "learning_rate": 1.7716730681022724e-08,
      "logits/chosen": -2.2751753330230713,
      "logits/rejected": -2.2461843490600586,
      "logps/chosen": -0.2717970013618469,
      "logps/rejected": -0.2640511393547058,
      "loss": 0.1832820177078247,
      "loss/core": 0.17082270979881287,
      "loss/preference_sft": 0.26608261466026306,
      "loss/reference_anchor": 0.22257855534553528,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.516340494155884,
      "rewards/margins": 1.0328803062438965,
      "rewards/rejected": 1.4834601879119873,
      "ronpo/logit": 0.10328805446624756,
      "ronpo/residual": 0.0827520564198494,
      "ronpo/residual_abs": 0.15645729005336761,
      "ronpo/target": 0.020535994321107864,
      "ronpo/target_abs": 0.08894232660531998,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 385
    },
    {
      "drift/chosen_abs": 0.21122761070728302,
      "drift/rejected_abs": 0.12187887728214264,
      "epoch": 0.12395709177592372,
      "grad_norm": 2.46875,
      "learning_rate": 1.749547609849315e-08,
      "logits/chosen": -2.0779099464416504,
      "logits/rejected": -2.118682384490967,
      "logps/chosen": -0.3044072091579437,
      "logps/rejected": -0.2954100966453552,
      "loss": 0.09059147536754608,
      "loss/core": 0.07689197361469269,
      "loss/preference_sft": 0.3024619221687317,
      "loss/reference_anchor": 0.24374382197856903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1094882488250732,
      "rewards/margins": 0.8933243751525879,
      "rewards/rejected": 1.2161636352539062,
      "ronpo/logit": 0.0893324464559555,
      "ronpo/residual": 0.0991659015417099,
      "ronpo/residual_abs": 0.1529773473739624,
      "ronpo/target": -0.00983346439898014,
      "ronpo/target_abs": 0.09204504638910294,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 390
    },
    {
      "drift/chosen_abs": 0.21260306239128113,
      "drift/rejected_abs": 0.0686182901263237,
      "epoch": 0.12554628526023043,
      "grad_norm": 2.84375,
      "learning_rate": 1.7272342921702935e-08,
      "logits/chosen": -2.158278465270996,
      "logits/rejected": -2.2224063873291016,
      "logps/chosen": -0.31105631589889526,
      "logps/rejected": -0.2972649335861206,
      "loss": 0.35496413707733154,
      "loss/core": 0.34252816438674927,
      "loss/preference_sft": 0.3063923716545105,
      "loss/reference_anchor": 0.2180800884962082,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1244759559631348,
      "rewards/margins": 1.4389787912368774,
      "rewards/rejected": 0.6854972839355469,
      "ronpo/logit": 0.1438978910446167,
      "ronpo/residual": 0.12823660671710968,
      "ronpo/residual_abs": 0.1940845549106598,
      "ronpo/target": 0.015661273151636124,
      "ronpo/target_abs": 0.0806753933429718,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 395
    },
    {
      "drift/chosen_abs": 0.19167861342430115,
      "drift/rejected_abs": 0.08439207822084427,
      "epoch": 0.12713547874453715,
      "grad_norm": 8.3125,
      "learning_rate": 1.7047415061914393e-08,
      "logits/chosen": -2.2295565605163574,
      "logits/rejected": -2.3195853233337402,
      "logps/chosen": -0.25695428252220154,
      "logps/rejected": -0.2609720826148987,
      "loss": 0.07849901169538498,
      "loss/core": 0.07116608321666718,
      "loss/preference_sft": 0.2507200539112091,
      "loss/reference_anchor": 0.12158646434545517,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9159801006317139,
      "rewards/margins": 1.0806057453155518,
      "rewards/rejected": 0.8353743553161621,
      "ronpo/logit": 0.1080605760216713,
      "ronpo/residual": 0.11492305994033813,
      "ronpo/residual_abs": 0.1632971316576004,
      "ronpo/target": -0.0068624913692474365,
      "ronpo/target_abs": 0.08553292602300644,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 400
    },
    {
      "drift/chosen_abs": 0.17035509645938873,
      "drift/rejected_abs": 0.14627881348133087,
      "epoch": 0.12872467222884387,
      "grad_norm": 2.34375,
      "learning_rate": 1.6820777105296708e-08,
      "logits/chosen": -2.2909557819366455,
      "logits/rejected": -2.2994208335876465,
      "logps/chosen": -0.29540935158729553,
      "logps/rejected": -0.2884915769100189,
      "loss": 0.29835981130599976,
      "loss/core": 0.2854141592979431,
      "loss/preference_sft": 0.2857155203819275,
      "loss/reference_anchor": 0.230341836810112,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7012271881103516,
      "rewards/margins": 0.239228293299675,
      "rewards/rejected": 1.4619989395141602,
      "ronpo/logit": 0.023922838270664215,
      "ronpo/residual": 0.02246231772005558,
      "ronpo/residual_abs": 0.18744370341300964,
      "ronpo/target": 0.001460521249100566,
      "ronpo/target_abs": 0.08477913588285446,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 405
    },
    {
      "drift/chosen_abs": 0.18731415271759033,
      "drift/rejected_abs": 0.05840852111577988,
      "epoch": 0.13031386571315057,
      "grad_norm": 0.6875,
      "learning_rate": 1.6592514281116553e-08,
      "logits/chosen": -2.229135751724243,
      "logits/rejected": -2.1434459686279297,
      "logps/chosen": -0.28917819261550903,
      "logps/rejected": -0.2873764634132385,
      "loss": 0.35508978366851807,
      "loss/core": 0.344736784696579,
      "loss/preference_sft": 0.28428682684898376,
      "loss/reference_anchor": 0.17863091826438904,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8712348937988281,
      "rewards/margins": 1.3082423210144043,
      "rewards/rejected": 0.5629926919937134,
      "ronpo/logit": 0.1308242380619049,
      "ronpo/residual": 0.14936430752277374,
      "ronpo/residual_abs": 0.17734959721565247,
      "ronpo/target": -0.01854007877409458,
      "ronpo/target_abs": 0.06559576839208603,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 410
    },
    {
      "drift/chosen_abs": 0.12775962054729462,
      "drift/rejected_abs": 0.05946607142686844,
      "epoch": 0.1319030591974573,
      "grad_norm": 18.5,
      "learning_rate": 1.6362712429686844e-08,
      "logits/chosen": -2.2688026428222656,
      "logits/rejected": -2.2994446754455566,
      "logps/chosen": -0.2852744460105896,
      "logps/rejected": -0.2741158604621887,
      "loss": 0.04457860440015793,
      "loss/core": 0.04096058011054993,
      "loss/preference_sft": 0.27498331665992737,
      "loss/reference_anchor": 0.04486214369535446,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.2775468826293945,
      "rewards/margins": 0.6865289807319641,
      "rewards/rejected": 0.5910179018974304,
      "ronpo/logit": 0.06865289807319641,
      "ronpo/residual": 0.061678577214479446,
      "ronpo/residual_abs": 0.13630971312522888,
      "ronpo/target": 0.006974323187023401,
      "ronpo/target_abs": 0.08561506867408752,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 415
    },
    {
      "drift/chosen_abs": 0.3900828957557678,
      "drift/rejected_abs": 0.12692275643348694,
      "epoch": 0.133492252681764,
      "grad_norm": 2.71875,
      "learning_rate": 1.6131457970085686e-08,
      "logits/chosen": -2.1172633171081543,
      "logits/rejected": -2.235016345977783,
      "logps/chosen": -0.2711094319820404,
      "logps/rejected": -0.2714807391166687,
      "loss": 0.8514937162399292,
      "loss/core": 0.8164560198783875,
      "loss/preference_sft": 0.2708297073841095,
      "loss/reference_anchor": 0.6736721992492676,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.9008281230926514,
      "rewards/margins": 2.7281997203826904,
      "rewards/rejected": 1.17262864112854,
      "ronpo/logit": 0.27282002568244934,
      "ronpo/residual": 0.25294801592826843,
      "ronpo/residual_abs": 0.3054654002189636,
      "ronpo/target": 0.019871985539793968,
      "ronpo/target_abs": 0.07743876427412033,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 420
    },
    {
      "drift/chosen_abs": 0.4248512387275696,
      "drift/rejected_abs": 0.13988152146339417,
      "epoch": 0.1350814461660707,
      "grad_norm": 310.0,
      "learning_rate": 1.5898837867657726e-08,
      "logits/chosen": -2.166339159011841,
      "logits/rejected": -2.1603424549102783,
      "logps/chosen": -0.2978591322898865,
      "logps/rejected": -0.2953459620475769,
      "loss": 1.2260562181472778,
      "loss/core": 1.1734199523925781,
      "loss/preference_sft": 0.29464253783226013,
      "loss/reference_anchor": 1.0232583284378052,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.247471332550049,
      "rewards/margins": 2.849874258041382,
      "rewards/rejected": 1.3975974321365356,
      "ronpo/logit": 0.2849874496459961,
      "ronpo/residual": 0.2766841650009155,
      "ronpo/residual_abs": 0.3479006886482239,
      "ronpo/target": 0.008303266949951649,
      "ronpo/target_abs": 0.09610433876514435,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 425
    },
    {
      "drift/chosen_abs": 0.18739613890647888,
      "drift/rejected_abs": 0.09084701538085938,
      "epoch": 0.13667063965037743,
      "grad_norm": 10.5,
      "learning_rate": 1.5664939601310025e-08,
      "logits/chosen": -2.4331743717193604,
      "logits/rejected": -2.4204342365264893,
      "logps/chosen": -0.2795220911502838,
      "logps/rejected": -0.2753976583480835,
      "loss": 0.10062910616397858,
      "loss/core": 0.09226439893245697,
      "loss/preference_sft": 0.2744489312171936,
      "loss/reference_anchor": 0.13984903693199158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8734142780303955,
      "rewards/margins": 0.9657185673713684,
      "rewards/rejected": 0.9076957702636719,
      "ronpo/logit": 0.09657186269760132,
      "ronpo/residual": 0.07882784307003021,
      "ronpo/residual_abs": 0.14449390769004822,
      "ronpo/target": 0.017744004726409912,
      "ronpo/target_abs": 0.07996980845928192,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 430
    },
    {
      "drift/chosen_abs": 0.17314964532852173,
      "drift/rejected_abs": 0.09341616928577423,
      "epoch": 0.13825983313468415,
      "grad_norm": 12.5,
      "learning_rate": 1.542985113061481e-08,
      "logits/chosen": -2.1819088459014893,
      "logits/rejected": -2.1771626472473145,
      "logps/chosen": -0.2796979546546936,
      "logps/rejected": -0.2803123891353607,
      "loss": 0.04070793837308884,
      "loss/core": 0.03540237620472908,
      "loss/preference_sft": 0.2738858759403229,
      "loss/reference_anchor": 0.07872271537780762,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.730942964553833,
      "rewards/margins": 0.7972149848937988,
      "rewards/rejected": 0.9337277412414551,
      "ronpo/logit": 0.07972150295972824,
      "ronpo/residual": 0.054669272154569626,
      "ronpo/residual_abs": 0.12845924496650696,
      "ronpo/target": 0.025052234530448914,
      "ronpo/target_abs": 0.07804572582244873,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 435
    },
    {
      "drift/chosen_abs": 0.318293035030365,
      "drift/rejected_abs": 0.13249650597572327,
      "epoch": 0.13984902661899087,
      "grad_norm": 4.15625,
      "learning_rate": 1.51936608627315e-08,
      "logits/chosen": -2.273724317550659,
      "logits/rejected": -2.299027919769287,
      "logps/chosen": -0.2951945960521698,
      "logps/rejected": -0.29159849882125854,
      "loss": 0.7339528203010559,
      "loss/core": 0.6881879568099976,
      "loss/preference_sft": 0.2977384924888611,
      "loss/reference_anchor": 0.885523796081543,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.182380437850952,
      "rewards/margins": 1.8633613586425781,
      "rewards/rejected": 1.319018840789795,
      "ronpo/logit": 0.18633611500263214,
      "ronpo/residual": 0.19329491257667542,
      "ronpo/residual_abs": 0.24592044949531555,
      "ronpo/target": -0.006958787329494953,
      "ronpo/target_abs": 0.0852695107460022,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 440
    },
    {
      "drift/chosen_abs": 0.18986327946186066,
      "drift/rejected_abs": 0.16925686597824097,
      "epoch": 0.14143822010329757,
      "grad_norm": 260.0,
      "learning_rate": 1.4956457619160377e-08,
      "logits/chosen": -2.2318367958068848,
      "logits/rejected": -2.154519557952881,
      "logps/chosen": -0.28089621663093567,
      "logps/rejected": -0.2713889479637146,
      "loss": 0.25093281269073486,
      "loss/core": 0.2370346039533615,
      "loss/preference_sft": 0.27891331911087036,
      "loss/reference_anchor": 0.25007274746894836,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8986326456069946,
      "rewards/margins": 0.20648033916950226,
      "rewards/rejected": 1.6921522617340088,
      "ronpo/logit": 0.020648030564188957,
      "ronpo/residual": 0.02750701829791069,
      "ronpo/residual_abs": 0.20267800986766815,
      "ronpo/target": -0.006858985871076584,
      "ronpo/target_abs": 0.08476842194795609,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 445
    },
    {
      "drift/chosen_abs": 0.3151724934577942,
      "drift/rejected_abs": 0.1027737632393837,
      "epoch": 0.1430274135876043,
      "grad_norm": 41.0,
      "learning_rate": 1.4718330602340439e-08,
      "logits/chosen": -2.264042615890503,
      "logits/rejected": -2.327329397201538,
      "logps/chosen": -0.2729445695877075,
      "logps/rejected": -0.28299084305763245,
      "loss": 0.7172930836677551,
      "loss/core": 0.6890506148338318,
      "loss/preference_sft": 0.2784006595611572,
      "loss/reference_anchor": 0.5370103120803833,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.150481939315796,
      "rewards/margins": 2.1262738704681396,
      "rewards/rejected": 1.0242083072662354,
      "ronpo/logit": 0.2126273810863495,
      "ronpo/residual": 0.21831850707530975,
      "ronpo/residual_abs": 0.26258429884910583,
      "ronpo/target": -0.005691120866686106,
      "ronpo/target_abs": 0.0731092318892479,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 450
    },
    {
      "drift/chosen_abs": 0.20744338631629944,
      "drift/rejected_abs": 0.09908546507358551,
      "epoch": 0.144616607071911,
      "grad_norm": 10.25,
      "learning_rate": 1.4479369362104036e-08,
      "logits/chosen": -2.3172354698181152,
      "logits/rejected": -2.3511035442352295,
      "logps/chosen": -0.284721314907074,
      "logps/rejected": -0.28263527154922485,
      "loss": 0.10904005914926529,
      "loss/core": 0.10059742629528046,
      "loss/preference_sft": 0.2903629243373871,
      "loss/reference_anchor": 0.13981646299362183,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0744338035583496,
      "rewards/margins": 1.083627462387085,
      "rewards/rejected": 0.9908064603805542,
      "ronpo/logit": 0.10836274921894073,
      "ronpo/residual": 0.107516810297966,
      "ronpo/residual_abs": 0.15838299691677094,
      "ronpo/target": 0.0008459277451038361,
      "ronpo/target_abs": 0.0736476257443428,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 455
    },
    {
      "drift/chosen_abs": 0.37808623909950256,
      "drift/rejected_abs": 0.1648140251636505,
      "epoch": 0.1462058005562177,
      "grad_norm": 121.5,
      "learning_rate": 1.4239663762000817e-08,
      "logits/chosen": -1.8078231811523438,
      "logits/rejected": -1.6678974628448486,
      "logps/chosen": -0.2803362011909485,
      "logps/rejected": -0.2994075417518616,
      "loss": 0.551475465297699,
      "loss/core": 0.5203845500946045,
      "loss/preference_sft": 0.28328341245651245,
      "loss/reference_anchor": 0.5934896469116211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.7770161628723145,
      "rewards/margins": 2.1362390518188477,
      "rewards/rejected": 1.6407768726348877,
      "ronpo/logit": 0.21362391114234924,
      "ronpo/residual": 0.21224097907543182,
      "ronpo/residual_abs": 0.2707666754722595,
      "ronpo/target": 0.0013829537201672792,
      "ronpo/target_abs": 0.08258362114429474,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 460
    },
    {
      "drift/chosen_abs": 0.34338995814323425,
      "drift/rejected_abs": 0.23440727591514587,
      "epoch": 0.14779499404052443,
      "grad_norm": 400.0,
      "learning_rate": 1.3999303945503746e-08,
      "logits/chosen": -2.0976834297180176,
      "logits/rejected": -2.116819381713867,
      "logps/chosen": -0.2706809341907501,
      "logps/rejected": -0.27307775616645813,
      "loss": 0.8269802331924438,
      "loss/core": 0.7959731221199036,
      "loss/preference_sft": 0.2685784101486206,
      "loss/reference_anchor": 0.5932824611663818,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4336636066436768,
      "rewards/margins": 1.0909364223480225,
      "rewards/rejected": 2.3427271842956543,
      "ronpo/logit": 0.10909364372491837,
      "ronpo/residual": 0.10874764621257782,
      "ronpo/residual_abs": 0.3142366111278534,
      "ronpo/target": 0.00034598278580233455,
      "ronpo/target_abs": 0.07308773696422577,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 465
    },
    {
      "drift/chosen_abs": 0.29909831285476685,
      "drift/rejected_abs": 0.11643582582473755,
      "epoch": 0.14938418752483115,
      "grad_norm": 1.9375,
      "learning_rate": 1.3758380302109807e-08,
      "logits/chosen": -2.075563430786133,
      "logits/rejected": -2.044642210006714,
      "logps/chosen": -0.254794180393219,
      "logps/rejected": -0.2602981626987457,
      "loss": 0.7801807522773743,
      "loss/core": 0.7529562711715698,
      "loss/preference_sft": 0.26095765829086304,
      "loss/reference_anchor": 0.5183919668197632,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.989849090576172,
      "rewards/margins": 1.8269249200820923,
      "rewards/rejected": 1.1629244089126587,
      "ronpo/logit": 0.1826924830675125,
      "ronpo/residual": 0.19307713210582733,
      "ronpo/residual_abs": 0.2564256191253662,
      "ronpo/target": -0.010384641587734222,
      "ronpo/target_abs": 0.08509234338998795,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 470
    },
    {
      "drift/chosen_abs": 0.2505967915058136,
      "drift/rejected_abs": 0.09651152044534683,
      "epoch": 0.15097338100913787,
      "grad_norm": 9.1875,
      "learning_rate": 1.3516983433348227e-08,
      "logits/chosen": -2.19514536857605,
      "logits/rejected": -2.130986213684082,
      "logps/chosen": -0.3052103817462921,
      "logps/rejected": -0.29925405979156494,
      "loss": 0.22467193007469177,
      "loss/core": 0.21474584937095642,
      "loss/preference_sft": 0.3062250018119812,
      "loss/reference_anchor": 0.16789942979812622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.505239486694336,
      "rewards/margins": 1.5401643514633179,
      "rewards/rejected": 0.9650753140449524,
      "ronpo/logit": 0.15401646494865417,
      "ronpo/residual": 0.15226852893829346,
      "ronpo/residual_abs": 0.21335072815418243,
      "ronpo/target": 0.0017479099333286285,
      "ronpo/target_abs": 0.07254009693861008,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 475
    },
    {
      "drift/chosen_abs": 0.22144432365894318,
      "drift/rejected_abs": 0.10890810191631317,
      "epoch": 0.15256257449344457,
      "grad_norm": 1.0859375,
      "learning_rate": 1.3275204118708941e-08,
      "logits/chosen": -2.119616746902466,
      "logits/rejected": -2.1420140266418457,
      "logps/chosen": -0.273566871881485,
      "logps/rejected": -0.2686625123023987,
      "loss": 0.08422975242137909,
      "loss/core": 0.07651440799236298,
      "loss/preference_sft": 0.2724953591823578,
      "loss/reference_anchor": 0.12705738842487335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.21227765083313,
      "rewards/margins": 1.1317625045776367,
      "rewards/rejected": 1.0805152654647827,
      "ronpo/logit": 0.11317624896764755,
      "ronpo/residual": 0.1295435130596161,
      "ronpo/residual_abs": 0.1753176897764206,
      "ronpo/target": -0.016367264091968536,
      "ronpo/target_abs": 0.08774258196353912,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 480
    },
    {
      "drift/chosen_abs": 0.2984105944633484,
      "drift/rejected_abs": 0.1611432582139969,
      "epoch": 0.1541517679777513,
      "grad_norm": 5.5,
      "learning_rate": 1.3033133281504133e-08,
      "logits/chosen": -1.902937889099121,
      "logits/rejected": -2.0238137245178223,
      "logps/chosen": -0.28192150592803955,
      "logps/rejected": -0.27930665016174316,
      "loss": 0.13234123587608337,
      "loss/core": 0.11475740373134613,
      "loss/preference_sft": 0.2823740839958191,
      "loss/reference_anchor": 0.3234391510486603,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.983555793762207,
      "rewards/margins": 1.3728954792022705,
      "rewards/rejected": 1.6106599569320679,
      "ronpo/logit": 0.13728956878185272,
      "ronpo/residual": 0.1287764012813568,
      "ronpo/residual_abs": 0.18728485703468323,
      "ronpo/target": 0.008513162843883038,
      "ronpo/target_abs": 0.08558246493339539,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 485
    },
    {
      "drift/chosen_abs": 0.3685315251350403,
      "drift/rejected_abs": 0.150931254029274,
      "epoch": 0.15574096146205801,
      "grad_norm": 290.0,
      "learning_rate": 1.2790861954675703e-08,
      "logits/chosen": -2.10400390625,
      "logits/rejected": -2.0856916904449463,
      "logps/chosen": -0.28186899423599243,
      "logps/rejected": -0.28309863805770874,
      "loss": 0.6792070269584656,
      "loss/core": 0.643970787525177,
      "loss/preference_sft": 0.28243130445480347,
      "loss/reference_anchor": 0.676482617855072,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.6826233863830566,
      "rewards/margins": 2.1746814250946045,
      "rewards/rejected": 1.5079416036605835,
      "ronpo/logit": 0.21746817231178284,
      "ronpo/residual": 0.21048131585121155,
      "ronpo/residual_abs": 0.2691686451435089,
      "ronpo/target": 0.006986844353377819,
      "ronpo/target_abs": 0.09098772704601288,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 490
    },
    {
      "drift/chosen_abs": 0.22006180882453918,
      "drift/rejected_abs": 0.18455646932125092,
      "epoch": 0.1573301549463647,
      "grad_norm": 396.0,
      "learning_rate": 1.2548481246561504e-08,
      "logits/chosen": -2.1262259483337402,
      "logits/rejected": -2.0967445373535156,
      "logps/chosen": -0.2868593633174896,
      "logps/rejected": -0.27408647537231445,
      "loss": 1.1514532566070557,
      "loss/core": 1.1166924238204956,
      "loss/preference_sft": 0.28249964118003845,
      "loss/reference_anchor": 0.6669629812240601,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.1990761756896973,
      "rewards/margins": 0.357932448387146,
      "rewards/rejected": 1.8411436080932617,
      "ronpo/logit": 0.03579322621226311,
      "ronpo/residual": 0.049376267939805984,
      "ronpo/residual_abs": 0.3220534324645996,
      "ronpo/target": -0.01358303613960743,
      "ronpo/target_abs": 0.09055624902248383,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 495
    },
    {
      "drift/chosen_abs": 0.2433398962020874,
      "drift/rejected_abs": 0.14733143150806427,
      "epoch": 0.15891934843067143,
      "grad_norm": 4.125,
      "learning_rate": 1.2306082306633209e-08,
      "logits/chosen": -2.231499195098877,
      "logits/rejected": -2.0890326499938965,
      "logps/chosen": -0.2816493809223175,
      "logps/rejected": -0.27643585205078125,
      "loss": 0.23166105151176453,
      "loss/core": 0.21609747409820557,
      "loss/preference_sft": 0.2753569185733795,
      "loss/reference_anchor": 0.28373581171035767,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.4331910610198975,
      "rewards/margins": 0.9780365228652954,
      "rewards/rejected": 1.455154538154602,
      "ronpo/logit": 0.09780365228652954,
      "ronpo/residual": 0.10129328817129135,
      "ronpo/residual_abs": 0.21842603385448456,
      "ronpo/target": -0.0034896410070359707,
      "ronpo/target_abs": 0.08457918465137482,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 500
    },
    {
      "drift/chosen_abs": 0.16049498319625854,
      "drift/rejected_abs": 0.10690776258707047,
      "epoch": 0.16050854191497815,
      "grad_norm": 4.5625,
      "learning_rate": 1.2063756291218741e-08,
      "logits/chosen": -2.284985065460205,
      "logits/rejected": -2.205040454864502,
      "logps/chosen": -0.29578253626823425,
      "logps/rejected": -0.2804624140262604,
      "loss": 0.03737588971853256,
      "loss/core": 0.0324200838804245,
      "loss/preference_sft": 0.2899005115032196,
      "loss/reference_anchor": 0.070126011967659,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6018108129501343,
      "rewards/margins": 0.5337648987770081,
      "rewards/rejected": 1.0680458545684814,
      "ronpo/logit": 0.05337648466229439,
      "ronpo/residual": 0.051874689757823944,
      "ronpo/residual_abs": 0.12874646484851837,
      "ronpo/target": 0.0015017971163615584,
      "ronpo/target_abs": 0.08839758485555649,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 505
    },
    {
      "drift/chosen_abs": 0.31590768694877625,
      "drift/rejected_abs": 0.1603766679763794,
      "epoch": 0.16209773539928488,
      "grad_norm": 2.671875,
      "learning_rate": 1.1821594329222079e-08,
      "logits/chosen": -1.9426066875457764,
      "logits/rejected": -1.9659500122070312,
      "logps/chosen": -0.26106467843055725,
      "logps/rejected": -0.2636715769767761,
      "loss": 0.4681622087955475,
      "loss/core": 0.435793936252594,
      "loss/preference_sft": 0.26362866163253784,
      "loss/reference_anchor": 0.6210030317306519,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.1584575176239014,
      "rewards/margins": 1.5580565929412842,
      "rewards/rejected": 1.6004009246826172,
      "ronpo/logit": 0.15580564737319946,
      "ronpo/residual": 0.15252608060836792,
      "ronpo/residual_abs": 0.2079734355211258,
      "ronpo/target": 0.0032795709557831287,
      "ronpo/target_abs": 0.07548246532678604,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 510
    },
    {
      "drift/chosen_abs": 0.2593478560447693,
      "drift/rejected_abs": 0.15457691252231598,
      "epoch": 0.16368692888359157,
      "grad_norm": 5.03125,
      "learning_rate": 1.157968748785344e-08,
      "logits/chosen": -2.1415348052978516,
      "logits/rejected": -2.2097504138946533,
      "logps/chosen": -0.28786981105804443,
      "logps/rejected": -0.27476245164871216,
      "loss": 0.3538331389427185,
      "loss/core": 0.33824825286865234,
      "loss/preference_sft": 0.2842520475387573,
      "loss/reference_anchor": 0.2832731306552887,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5934786796569824,
      "rewards/margins": 1.050806999206543,
      "rewards/rejected": 1.542672038078308,
      "ronpo/logit": 0.10508068650960922,
      "ronpo/residual": 0.1014178991317749,
      "ronpo/residual_abs": 0.22226110100746155,
      "ronpo/target": 0.0036627918016165495,
      "ronpo/target_abs": 0.07639802247285843,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 515
    },
    {
      "drift/chosen_abs": 0.2165084183216095,
      "drift/rejected_abs": 0.10440798103809357,
      "epoch": 0.1652761223678983,
      "grad_norm": 3.078125,
      "learning_rate": 1.1338126738382598e-08,
      "logits/chosen": -2.1498093605041504,
      "logits/rejected": -2.1602375507354736,
      "logps/chosen": -0.2931273579597473,
      "logps/rejected": -0.30199140310287476,
      "loss": 0.261959433555603,
      "loss/core": 0.2352810651063919,
      "loss/preference_sft": 0.293254554271698,
      "loss/reference_anchor": 0.5042413473129272,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.16508412361145,
      "rewards/margins": 1.1286433935165405,
      "rewards/rejected": 1.0364407300949097,
      "ronpo/logit": 0.11286433786153793,
      "ronpo/residual": 0.10544689744710922,
      "ronpo/residual_abs": 0.17088401317596436,
      "ronpo/target": 0.007417450193315744,
      "ronpo/target_abs": 0.08988238126039505,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 520
    },
    {
      "drift/chosen_abs": 0.22024352848529816,
      "drift/rejected_abs": 0.06529395282268524,
      "epoch": 0.16686531585220502,
      "grad_norm": 7.9375,
      "learning_rate": 1.1097002921928317e-08,
      "logits/chosen": -1.9056131839752197,
      "logits/rejected": -2.050123929977417,
      "logps/chosen": -0.3115862309932709,
      "logps/rejected": -0.3198958933353424,
      "loss": 0.2833040654659271,
      "loss/core": 0.2724967300891876,
      "loss/preference_sft": 0.32389670610427856,
      "loss/reference_anchor": 0.18375670909881592,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.201336145401001,
      "rewards/margins": 1.5499281883239746,
      "rewards/rejected": 0.6514078378677368,
      "ronpo/logit": 0.15499281883239746,
      "ronpo/residual": 0.1652340143918991,
      "ronpo/residual_abs": 0.22205249965190887,
      "ronpo/target": -0.010241201147437096,
      "ronpo/target_abs": 0.09030992537736893,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 525
    },
    {
      "drift/chosen_abs": 0.15349416434764862,
      "drift/rejected_abs": 0.06507018208503723,
      "epoch": 0.1684545093365117,
      "grad_norm": 1.2734375,
      "learning_rate": 1.0856406715296717e-08,
      "logits/chosen": -2.3231589794158936,
      "logits/rejected": -2.317765712738037,
      "logps/chosen": -0.2665853500366211,
      "logps/rejected": -0.2610515356063843,
      "loss": 0.06234220415353775,
      "loss/core": 0.05808652564883232,
      "loss/preference_sft": 0.26546263694763184,
      "loss/reference_anchor": 0.05856741592288017,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.534586787223816,
      "rewards/margins": 0.8877205848693848,
      "rewards/rejected": 0.6468662023544312,
      "ronpo/logit": 0.08877205848693848,
      "ronpo/residual": 0.09845416247844696,
      "ronpo/residual_abs": 0.13878455758094788,
      "ronpo/target": -0.009682102128863335,
      "ronpo/target_abs": 0.08094774186611176,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 530
    },
    {
      "drift/chosen_abs": 0.3475359380245209,
      "drift/rejected_abs": 0.1439954936504364,
      "epoch": 0.17004370282081843,
      "grad_norm": 3.171875,
      "learning_rate": 1.0616428596881461e-08,
      "logits/chosen": -2.2414088249206543,
      "logits/rejected": -2.255117893218994,
      "logps/chosen": -0.31528395414352417,
      "logps/rejected": -0.3214128613471985,
      "loss": 1.5130736827850342,
      "loss/core": 1.4584028720855713,
      "loss/preference_sft": 0.31637319922447205,
      "loss/reference_anchor": 1.0617750883102417,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.473903179168701,
      "rewards/margins": 2.195761203765869,
      "rewards/rejected": 1.2781422138214111,
      "ronpo/logit": 0.21957612037658691,
      "ronpo/residual": 0.2536545991897583,
      "ronpo/residual_abs": 0.3511970341205597,
      "ronpo/target": -0.034078482538461685,
      "ronpo/target_abs": 0.09332413971424103,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 535
    },
    {
      "drift/chosen_abs": 0.24628369510173798,
      "drift/rejected_abs": 0.1078454852104187,
      "epoch": 0.17163289630512515,
      "grad_norm": 4.6875,
      "learning_rate": 1.0377158812638491e-08,
      "logits/chosen": -2.202691078186035,
      "logits/rejected": -2.2384159564971924,
      "logps/chosen": -0.27798351645469666,
      "logps/rejected": -0.29006585478782654,
      "loss": 0.22769439220428467,
      "loss/core": 0.20758168399333954,
      "loss/preference_sft": 0.28700748085975647,
      "loss/reference_anchor": 0.37355345487594604,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.4620206356048584,
      "rewards/margins": 1.38406240940094,
      "rewards/rejected": 1.0779582262039185,
      "ronpo/logit": 0.13840624690055847,
      "ronpo/residual": 0.1439955234527588,
      "ronpo/residual_abs": 0.1996123194694519,
      "ronpo/target": -0.005589271429926157,
      "ronpo/target_abs": 0.07924790680408478,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 540
    },
    {
      "drift/chosen_abs": 0.3985271155834198,
      "drift/rejected_abs": 0.24583449959754944,
      "epoch": 0.17322208978943188,
      "grad_norm": 3.984375,
      "learning_rate": 1.0138687342148249e-08,
      "logits/chosen": -1.992592453956604,
      "logits/rejected": -2.0166046619415283,
      "logps/chosen": -0.2606894373893738,
      "logps/rejected": -0.27958837151527405,
      "loss": 0.46023693680763245,
      "loss/core": 0.41777506470680237,
      "loss/preference_sft": 0.27719753980636597,
      "loss/reference_anchor": 0.8215174674987793,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.9839775562286377,
      "rewards/margins": 1.532893419265747,
      "rewards/rejected": 2.4510836601257324,
      "ronpo/logit": 0.15328934788703918,
      "ronpo/residual": 0.15502643585205078,
      "ronpo/residual_abs": 0.2659107744693756,
      "ronpo/target": -0.0017371115973219275,
      "ronpo/target_abs": 0.0876903235912323,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 545
    },
    {
      "drift/chosen_abs": 0.2757337689399719,
      "drift/rejected_abs": 0.13784222304821014,
      "epoch": 0.17481128327373857,
      "grad_norm": 0.8828125,
      "learning_rate": 9.90110386477801e-09,
      "logits/chosen": -2.144867420196533,
      "logits/rejected": -2.1006226539611816,
      "logps/chosen": -0.2549815773963928,
      "logps/rejected": -0.26524803042411804,
      "loss": 0.307639479637146,
      "loss/core": 0.28966981172561646,
      "loss/preference_sft": 0.26307356357574463,
      "loss/reference_anchor": 0.3330860733985901,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.7563281059265137,
      "rewards/margins": 1.3787453174591064,
      "rewards/rejected": 1.3775826692581177,
      "ronpo/logit": 0.1378745287656784,
      "ronpo/residual": 0.14565442502498627,
      "ronpo/residual_abs": 0.20090845227241516,
      "ronpo/target": -0.007779897656291723,
      "ronpo/target_abs": 0.0823056623339653,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 550
    },
    {
      "drift/chosen_abs": 0.12533283233642578,
      "drift/rejected_abs": 0.05521406978368759,
      "epoch": 0.1764004767580453,
      "grad_norm": 0.640625,
      "learning_rate": 9.664497725957164e-09,
      "logits/chosen": -2.216212511062622,
      "logits/rejected": -2.258756160736084,
      "logps/chosen": -0.311422199010849,
      "logps/rejected": -0.30880364775657654,
      "loss": 0.07220657169818878,
      "loss/core": 0.06694497168064117,
      "loss/preference_sft": 0.3110686242580414,
      "loss/reference_anchor": 0.07412528246641159,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.252854347229004,
      "rewards/margins": 0.7011260390281677,
      "rewards/rejected": 0.551728367805481,
      "ronpo/logit": 0.07011260092258453,
      "ronpo/residual": 0.0683169811964035,
      "ronpo/residual_abs": 0.1411377489566803,
      "ronpo/target": 0.0017956193769350648,
      "ronpo/target_abs": 0.09536027908325195,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 555
    },
    {
      "drift/chosen_abs": 0.2655150294303894,
      "drift/rejected_abs": 0.12211163341999054,
      "epoch": 0.17798967024235202,
      "grad_norm": 16.5,
      "learning_rate": 9.428957903578044e-09,
      "logits/chosen": -2.08587646484375,
      "logits/rejected": -2.091303586959839,
      "logps/chosen": -0.2719891667366028,
      "logps/rejected": -0.27926185727119446,
      "loss": 0.254443883895874,
      "loss/core": 0.24059149622917175,
      "loss/preference_sft": 0.2721792161464691,
      "loss/reference_anchor": 0.24982962012290955,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.653385639190674,
      "rewards/margins": 1.4334917068481445,
      "rewards/rejected": 1.2198941707611084,
      "ronpo/logit": 0.14334917068481445,
      "ronpo/residual": 0.13729219138622284,
      "ronpo/residual_abs": 0.18650557100772858,
      "ronpo/target": 0.0060569727793335915,
      "ronpo/target_abs": 0.0743868425488472,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 560
    },
    {
      "drift/chosen_abs": 0.2600178122520447,
      "drift/rejected_abs": 0.11504121869802475,
      "epoch": 0.1795788637266587,
      "grad_norm": 28.25,
      "learning_rate": 9.194572974534975e-09,
      "logits/chosen": -2.041768789291382,
      "logits/rejected": -2.0671210289001465,
      "logps/chosen": -0.34080713987350464,
      "logps/rejected": -0.3354662358760834,
      "loss": 0.36407026648521423,
      "loss/core": 0.34170204401016235,
      "loss/preference_sft": 0.34413477778434753,
      "loss/reference_anchor": 0.4129505157470703,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.597466230392456,
      "rewards/margins": 1.4497805833816528,
      "rewards/rejected": 1.1476857662200928,
      "ronpo/logit": 0.14497807621955872,
      "ronpo/residual": 0.144731804728508,
      "ronpo/residual_abs": 0.20529747009277344,
      "ronpo/target": 0.0002462614211253822,
      "ronpo/target_abs": 0.08707638084888458,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 565
    },
    {
      "drift/chosen_abs": 0.41662201285362244,
      "drift/rejected_abs": 0.09962382912635803,
      "epoch": 0.18116805721096543,
      "grad_norm": 9.9375,
      "learning_rate": 8.96143108141412e-09,
      "logits/chosen": -2.1486594676971436,
      "logits/rejected": -2.119309902191162,
      "logps/chosen": -0.27083221077919006,
      "logps/rejected": -0.2647841274738312,
      "loss": 1.456986904144287,
      "loss/core": 1.414247989654541,
      "loss/preference_sft": 0.2618175148963928,
      "loss/reference_anchor": 0.8285951614379883,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.165776252746582,
      "rewards/margins": 3.1706364154815674,
      "rewards/rejected": 0.9951398968696594,
      "ronpo/logit": 0.31706368923187256,
      "ronpo/residual": 0.30529677867889404,
      "ronpo/residual_abs": 0.3695726692676544,
      "ronpo/target": 0.011766904965043068,
      "ronpo/target_abs": 0.0811844915151596,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 570
    },
    {
      "drift/chosen_abs": 0.19189342856407166,
      "drift/rejected_abs": 0.09652575105428696,
      "epoch": 0.18275725069527216,
      "grad_norm": 3.328125,
      "learning_rate": 8.72961989934668e-09,
      "logits/chosen": -2.0099151134490967,
      "logits/rejected": -2.0976462364196777,
      "logps/chosen": -0.28342169523239136,
      "logps/rejected": -0.27922505140304565,
      "loss": 0.05769554525613785,
      "loss/core": 0.052024852484464645,
      "loss/preference_sft": 0.28891006112098694,
      "loss/reference_anchor": 0.08452272415161133,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9166200160980225,
      "rewards/margins": 0.9536026120185852,
      "rewards/rejected": 0.963017463684082,
      "ronpo/logit": 0.09536024928092957,
      "ronpo/residual": 0.10312417894601822,
      "ronpo/residual_abs": 0.14294111728668213,
      "ronpo/target": -0.00776391988620162,
      "ronpo/target_abs": 0.07904698699712753,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 575
    },
    {
      "drift/chosen_abs": 0.36418384313583374,
      "drift/rejected_abs": 0.1662895232439041,
      "epoch": 0.18434644417957885,
      "grad_norm": 1.7734375,
      "learning_rate": 8.499226603037854e-09,
      "logits/chosen": -2.0011439323425293,
      "logits/rejected": -2.031702756881714,
      "logps/chosen": -0.28904083371162415,
      "logps/rejected": -0.2685038447380066,
      "loss": 0.5859975814819336,
      "loss/core": 0.5528149008750916,
      "loss/preference_sft": 0.28503909707069397,
      "loss/reference_anchor": 0.6351504921913147,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.640242099761963,
      "rewards/margins": 1.9781475067138672,
      "rewards/rejected": 1.6620944738388062,
      "ronpo/logit": 0.19781477749347687,
      "ronpo/residual": 0.19819863140583038,
      "ronpo/residual_abs": 0.2739180326461792,
      "ronpo/target": -0.0003838575503323227,
      "ronpo/target_abs": 0.08456491678953171,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 580
    },
    {
      "drift/chosen_abs": 0.23182368278503418,
      "drift/rejected_abs": 0.09531630575656891,
      "epoch": 0.18593563766388557,
      "grad_norm": 1.9375,
      "learning_rate": 8.270337833983987e-09,
      "logits/chosen": -1.9567636251449585,
      "logits/rejected": -1.9816787242889404,
      "logps/chosen": -0.28648510575294495,
      "logps/rejected": -0.28638434410095215,
      "loss": 0.15469267964363098,
      "loss/core": 0.14479199051856995,
      "loss/preference_sft": 0.29087167978286743,
      "loss/reference_anchor": 0.16892696917057037,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.317553997039795,
      "rewards/margins": 1.3662830591201782,
      "rewards/rejected": 0.9512712359428406,
      "ronpo/logit": 0.13662829995155334,
      "ronpo/residual": 0.1309715062379837,
      "ronpo/residual_abs": 0.17593583464622498,
      "ronpo/target": 0.00565680768340826,
      "ronpo/target_abs": 0.0699964389204979,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 585
    },
    {
      "drift/chosen_abs": 0.25730934739112854,
      "drift/rejected_abs": 0.1230391412973404,
      "epoch": 0.1875248311481923,
      "grad_norm": 1.375,
      "learning_rate": 8.04303966789025e-09,
      "logits/chosen": -2.118440866470337,
      "logits/rejected": -2.0865204334259033,
      "logps/chosen": -0.3093511760234833,
      "logps/rejected": -0.3141013979911804,
      "loss": 0.5824509263038635,
      "loss/core": 0.5587186217308044,
      "loss/preference_sft": 0.3041399419307709,
      "loss/reference_anchor": 0.4442318379878998,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.570599317550659,
      "rewards/margins": 1.3402273654937744,
      "rewards/rejected": 1.2303721904754639,
      "ronpo/logit": 0.1340227574110031,
      "ronpo/residual": 0.14667926728725433,
      "ronpo/residual_abs": 0.22119927406311035,
      "ronpo/target": -0.012656529434025288,
      "ronpo/target_abs": 0.08529551327228546,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 590
    },
    {
      "drift/chosen_abs": 0.23181328177452087,
      "drift/rejected_abs": 0.11188916116952896,
      "epoch": 0.18911402463249902,
      "grad_norm": 6.5,
      "learning_rate": 7.817417582301098e-09,
      "logits/chosen": -1.9782168865203857,
      "logits/rejected": -1.9370613098144531,
      "logps/chosen": -0.2896624207496643,
      "logps/rejected": -0.29374343156814575,
      "loss": 0.1985751986503601,
      "loss/core": 0.18759621679782867,
      "loss/preference_sft": 0.29568907618522644,
      "loss/reference_anchor": 0.1900101602077484,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.3166379928588867,
      "rewards/margins": 1.202042818069458,
      "rewards/rejected": 1.1145952939987183,
      "ronpo/logit": 0.12020426988601685,
      "ronpo/residual": 0.11585943400859833,
      "ronpo/residual_abs": 0.1900613158941269,
      "ronpo/target": 0.00434483215212822,
      "ronpo/target_abs": 0.08699861913919449,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 595
    },
    {
      "drift/chosen_abs": 0.2037758082151413,
      "drift/rejected_abs": 0.14159931242465973,
      "epoch": 0.1907032181168057,
      "grad_norm": 152.0,
      "learning_rate": 7.59355642445566e-09,
      "logits/chosen": -2.1320338249206543,
      "logits/rejected": -2.1046671867370605,
      "logps/chosen": -0.29946523904800415,
      "logps/rejected": -0.3209082782268524,
      "loss": 0.4402243494987488,
      "loss/core": 0.4250958561897278,
      "loss/preference_sft": 0.3151097893714905,
      "loss/reference_anchor": 0.27105993032455444,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.0345118045806885,
      "rewards/margins": 0.6213451623916626,
      "rewards/rejected": 1.4131665229797363,
      "ronpo/logit": 0.0621345154941082,
      "ronpo/residual": 0.06725134700536728,
      "ronpo/residual_abs": 0.2370101511478424,
      "ronpo/target": -0.005116832908242941,
      "ronpo/target_abs": 0.07861164957284927,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 600
    },
    {
      "drift/chosen_abs": 0.34658682346343994,
      "drift/rejected_abs": 0.1579512059688568,
      "epoch": 0.19229241160111243,
      "grad_norm": 1.8359375,
      "learning_rate": 7.371540379380149e-09,
      "logits/chosen": -2.0286879539489746,
      "logits/rejected": -1.953050971031189,
      "logps/chosen": -0.283130407333374,
      "logps/rejected": -0.27908605337142944,
      "loss": 0.4860695004463196,
      "loss/core": 0.45568352937698364,
      "loss/preference_sft": 0.28036171197891235,
      "loss/reference_anchor": 0.5796834826469421,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.456225633621216,
      "rewards/margins": 1.8863617181777954,
      "rewards/rejected": 1.5698635578155518,
      "ronpo/logit": 0.1886361837387085,
      "ronpo/residual": 0.18362456560134888,
      "ronpo/residual_abs": 0.25072842836380005,
      "ronpo/target": 0.005011610221117735,
      "ronpo/target_abs": 0.09113539010286331,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 605
    },
    {
      "drift/chosen_abs": 0.33414921164512634,
      "drift/rejected_abs": 0.15187159180641174,
      "epoch": 0.19388160508541916,
      "grad_norm": 87.5,
      "learning_rate": 7.151452938229324e-09,
      "logits/chosen": -2.0221667289733887,
      "logits/rejected": -1.9719922542572021,
      "logps/chosen": -0.31065306067466736,
      "logps/rejected": -0.294253408908844,
      "loss": 0.34279918670654297,
      "loss/core": 0.31768709421157837,
      "loss/preference_sft": 0.3136529326438904,
      "loss/reference_anchor": 0.470877081155777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.3398075103759766,
      "rewards/margins": 1.8233705759048462,
      "rewards/rejected": 1.5164368152618408,
      "ronpo/logit": 0.18233706057071686,
      "ronpo/residual": 0.1946430653333664,
      "ronpo/residual_abs": 0.24323885142803192,
      "ronpo/target": -0.012306014075875282,
      "ronpo/target_abs": 0.0871143639087677,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 610
    },
    {
      "drift/chosen_abs": 0.2550937533378601,
      "drift/rejected_abs": 0.11376242339611053,
      "epoch": 0.19547079856972585,
      "grad_norm": 125.5,
      "learning_rate": 6.933376866888882e-09,
      "logits/chosen": -2.2011666297912598,
      "logits/rejected": -2.162236452102661,
      "logps/chosen": -0.28771674633026123,
      "logps/rejected": -0.28812357783317566,
      "loss": 0.3027460277080536,
      "loss/core": 0.28452661633491516,
      "loss/preference_sft": 0.2859073579311371,
      "loss/reference_anchor": 0.3357972800731659,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.5509376525878906,
      "rewards/margins": 1.4323103427886963,
      "rewards/rejected": 1.1186271905899048,
      "ronpo/logit": 0.14323106408119202,
      "ronpo/residual": 0.14810332655906677,
      "ronpo/residual_abs": 0.1831142008304596,
      "ronpo/target": -0.004872290883213282,
      "ronpo/target_abs": 0.07001273334026337,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 615
    },
    {
      "drift/chosen_abs": 0.22261254489421844,
      "drift/rejected_abs": 0.10966954380273819,
      "epoch": 0.19705999205403257,
      "grad_norm": 12.0,
      "learning_rate": 6.717394174850605e-09,
      "logits/chosen": -2.0854969024658203,
      "logits/rejected": -2.0211100578308105,
      "logps/chosen": -0.2912364602088928,
      "logps/rejected": -0.2868305444717407,
      "loss": 0.10874249786138535,
      "loss/core": 0.1000494584441185,
      "loss/preference_sft": 0.28769445419311523,
      "loss/reference_anchor": 0.14509165287017822,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.2254793643951416,
      "rewards/margins": 1.1289622783660889,
      "rewards/rejected": 1.0965168476104736,
      "ronpo/logit": 0.11289622634649277,
      "ronpo/residual": 0.11616779863834381,
      "ronpo/residual_abs": 0.1724853515625,
      "ronpo/target": -0.003271549940109253,
      "ronpo/target_abs": 0.07885955274105072,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 620
    },
    {
      "drift/chosen_abs": 0.24589157104492188,
      "drift/rejected_abs": 0.09249149262905121,
      "epoch": 0.1986491855383393,
      "grad_norm": 49.0,
      "learning_rate": 6.503586084371926e-09,
      "logits/chosen": -2.0363283157348633,
      "logits/rejected": -2.0404984951019287,
      "logps/chosen": -0.2965640425682068,
      "logps/rejected": -0.2985779047012329,
      "loss": 0.34146690368652344,
      "loss/core": 0.32861843705177307,
      "loss/preference_sft": 0.29958051443099976,
      "loss/reference_anchor": 0.22701041400432587,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.457526445388794,
      "rewards/margins": 1.536529302597046,
      "rewards/rejected": 0.920997142791748,
      "ronpo/logit": 0.15365295112133026,
      "ronpo/residual": 0.15617410838603973,
      "ronpo/residual_abs": 0.213802307844162,
      "ronpo/target": -0.0025211633183062077,
      "ronpo/target_abs": 0.08408491313457489,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 625
    },
    {
      "drift/chosen_abs": 0.23458974063396454,
      "drift/rejected_abs": 0.08947785198688507,
      "epoch": 0.20023837902264602,
      "grad_norm": 2.875,
      "learning_rate": 6.29203299993155e-09,
      "logits/chosen": -2.325148105621338,
      "logits/rejected": -2.414926052093506,
      "logps/chosen": -0.2667160928249359,
      "logps/rejected": -0.26361387968063354,
      "loss": 0.3130393922328949,
      "loss/core": 0.2962197959423065,
      "loss/preference_sft": 0.2673772871494293,
      "loss/reference_anchor": 0.3096539378166199,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.345794439315796,
      "rewards/margins": 1.4539344310760498,
      "rewards/rejected": 0.8918598890304565,
      "ronpo/logit": 0.14539344608783722,
      "ronpo/residual": 0.13748937845230103,
      "ronpo/residual_abs": 0.20279105007648468,
      "ronpo/target": 0.007904065772891045,
      "ronpo/target_abs": 0.08239643275737762,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 630
    },
    {
      "drift/chosen_abs": 0.28986817598342896,
      "drift/rejected_abs": 0.10853831470012665,
      "epoch": 0.2018275725069527,
      "grad_norm": 15.375,
      "learning_rate": 6.0828144779926565e-09,
      "logits/chosen": -2.111020565032959,
      "logits/rejected": -2.1508452892303467,
      "logps/chosen": -0.2651686668395996,
      "logps/rejected": -0.2719859182834625,
      "loss": 0.3829234540462494,
      "loss/core": 0.36607077717781067,
      "loss/preference_sft": 0.269239604473114,
      "loss/reference_anchor": 0.31012919545173645,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.8979406356811523,
      "rewards/margins": 1.8216516971588135,
      "rewards/rejected": 1.0762889385223389,
      "ronpo/logit": 0.18216517567634583,
      "ronpo/residual": 0.1788938343524933,
      "ronpo/residual_abs": 0.22935160994529724,
      "ronpo/target": 0.003271352034062147,
      "ronpo/target_abs": 0.07819235324859619,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 635
    },
    {
      "drift/chosen_abs": 0.3245066702365875,
      "drift/rejected_abs": 0.09787418693304062,
      "epoch": 0.20341676599125944,
      "grad_norm": 1.3046875,
      "learning_rate": 5.87600919708494e-09,
      "logits/chosen": -2.0167291164398193,
      "logits/rejected": -2.0083279609680176,
      "logps/chosen": -0.3133408725261688,
      "logps/rejected": -0.3174097239971161,
      "loss": 0.7297495603561401,
      "loss/core": 0.7002626657485962,
      "loss/preference_sft": 0.3104119300842285,
      "loss/reference_anchor": 0.5586963891983032,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.2447280883789062,
      "rewards/margins": 2.300731897354126,
      "rewards/rejected": 0.9439964294433594,
      "ronpo/logit": 0.23007316887378693,
      "ronpo/residual": 0.23132848739624023,
      "ronpo/residual_abs": 0.27878475189208984,
      "ronpo/target": -0.0012552805710583925,
      "ronpo/target_abs": 0.08154843747615814,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 640
    },
    {
      "drift/chosen_abs": 0.2655944526195526,
      "drift/rejected_abs": 0.11456499993801117,
      "epoch": 0.20500595947556616,
      "grad_norm": 7.34375,
      "learning_rate": 5.671694928216829e-09,
      "logits/chosen": -2.1220450401306152,
      "logits/rejected": -2.127779722213745,
      "logps/chosen": -0.3085785508155823,
      "logps/rejected": -0.29471153020858765,
      "loss": 0.3396216034889221,
      "loss/core": 0.32221999764442444,
      "loss/preference_sft": 0.3059234619140625,
      "loss/reference_anchor": 0.3174397945404053,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.6536717414855957,
      "rewards/margins": 1.5116037130355835,
      "rewards/rejected": 1.1420681476593018,
      "ronpo/logit": 0.15116038918495178,
      "ronpo/residual": 0.14593927562236786,
      "ronpo/residual_abs": 0.2119034230709076,
      "ronpo/target": 0.005221105180680752,
      "ronpo/target_abs": 0.07934162765741348,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 645
    },
    {
      "drift/chosen_abs": 0.15132535994052887,
      "drift/rejected_abs": 0.09072579443454742,
      "epoch": 0.20659515295987285,
      "grad_norm": 1.859375,
      "learning_rate": 5.469948505629e-09,
      "logits/chosen": -2.2047553062438965,
      "logits/rejected": -2.1337642669677734,
      "logps/chosen": -0.30132001638412476,
      "logps/rejected": -0.2931322455406189,
      "loss": 0.06630890816450119,
      "loss/core": 0.06135622784495354,
      "loss/preference_sft": 0.3019540309906006,
      "loss/reference_anchor": 0.06885839998722076,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.5123118162155151,
      "rewards/margins": 0.6547757983207703,
      "rewards/rejected": 0.8575360178947449,
      "ronpo/logit": 0.06547758728265762,
      "ronpo/residual": 0.06879720091819763,
      "ronpo/residual_abs": 0.14413335919380188,
      "ronpo/target": -0.003319614799693227,
      "ronpo/target_abs": 0.08891183882951736,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 650
    },
    {
      "drift/chosen_abs": 0.20433752238750458,
      "drift/rejected_abs": 0.06546036899089813,
      "epoch": 0.20818434644417957,
      "grad_norm": 160.0,
      "learning_rate": 5.270845797900168e-09,
      "logits/chosen": -2.2784745693206787,
      "logits/rejected": -2.2848639488220215,
      "logps/chosen": -0.28383541107177734,
      "logps/rejected": -0.29783403873443604,
      "loss": 0.3861464858055115,
      "loss/core": 0.3738086223602295,
      "loss/preference_sft": 0.2895219326019287,
      "loss/reference_anchor": 0.21780526638031006,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.042762517929077,
      "rewards/margins": 1.3915107250213623,
      "rewards/rejected": 0.6512519121170044,
      "ronpo/logit": 0.13915108144283295,
      "ronpo/residual": 0.16224148869514465,
      "ronpo/residual_abs": 0.21403923630714417,
      "ronpo/target": -0.02309040166437626,
      "ronpo/target_abs": 0.0899852067232132,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 655
    },
    {
      "drift/chosen_abs": 0.1753084361553192,
      "drift/rejected_abs": 0.07402169704437256,
      "epoch": 0.2097735399284863,
      "grad_norm": 6.65625,
      "learning_rate": 5.07446167941601e-09,
      "logits/chosen": -2.2544825077056885,
      "logits/rejected": -2.2972869873046875,
      "logps/chosen": -0.2956244647502899,
      "logps/rejected": -0.28930968046188354,
      "loss": 0.1263386309146881,
      "loss/core": 0.11955497413873672,
      "loss/preference_sft": 0.29426223039627075,
      "loss/reference_anchor": 0.10624738782644272,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.7521049976348877,
      "rewards/margins": 1.014744520187378,
      "rewards/rejected": 0.7373603582382202,
      "ronpo/logit": 0.10147446393966675,
      "ronpo/residual": 0.09581993520259857,
      "ronpo/residual_abs": 0.14587488770484924,
      "ronpo/target": 0.005654525477439165,
      "ronpo/target_abs": 0.07976684719324112,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 660
    },
    {
      "drift/chosen_abs": 0.23486240208148956,
      "drift/rejected_abs": 0.13577520847320557,
      "epoch": 0.21136273341279302,
      "grad_norm": 0.6953125,
      "learning_rate": 4.880870002211964e-09,
      "logits/chosen": -2.0763564109802246,
      "logits/rejected": -2.0201761722564697,
      "logps/chosen": -0.2823403477668762,
      "logps/rejected": -0.2865101993083954,
      "loss": 0.24251846969127655,
      "loss/core": 0.22713109850883484,
      "loss/preference_sft": 0.284621000289917,
      "loss/reference_anchor": 0.2792838513851166,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.3464529514312744,
      "rewards/margins": 0.9889312982559204,
      "rewards/rejected": 1.3575217723846436,
      "ronpo/logit": 0.09889312833547592,
      "ronpo/residual": 0.10298037528991699,
      "ronpo/residual_abs": 0.1839233934879303,
      "ronpo/target": -0.004087253473699093,
      "ronpo/target_abs": 0.09008091688156128,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 665
    },
    {
      "drift/chosen_abs": 0.20962342619895935,
      "drift/rejected_abs": 0.07850834727287292,
      "epoch": 0.2129519268970997,
      "grad_norm": 5.40625,
      "learning_rate": 4.6901435682005e-09,
      "logits/chosen": -2.3460006713867188,
      "logits/rejected": -2.3546290397644043,
      "logps/chosen": -0.26797959208488464,
      "logps/rejected": -0.27196618914604187,
      "loss": 0.2039221227169037,
      "loss/core": 0.1944776475429535,
      "loss/preference_sft": 0.2721707820892334,
      "loss/reference_anchor": 0.16167283058166504,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.094529390335083,
      "rewards/margins": 1.3098270893096924,
      "rewards/rejected": 0.7847023010253906,
      "ronpo/logit": 0.13098271191120148,
      "ronpo/residual": 0.15417954325675964,
      "ronpo/residual_abs": 0.19339290261268616,
      "ronpo/target": -0.02319684997200966,
      "ronpo/target_abs": 0.08436106145381927,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 670
    },
    {
      "drift/chosen_abs": 0.40202194452285767,
      "drift/rejected_abs": 0.08130165189504623,
      "epoch": 0.21454112038140644,
      "grad_norm": 1.90625,
      "learning_rate": 4.502354101793314e-09,
      "logits/chosen": -1.9860864877700806,
      "logits/rejected": -2.099212646484375,
      "logps/chosen": -0.2779615521430969,
      "logps/rejected": -0.26965683698654175,
      "loss": 1.4497228860855103,
      "loss/core": 1.408434271812439,
      "loss/preference_sft": 0.2684791684150696,
      "loss/reference_anchor": 0.7989233732223511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 4.020218849182129,
      "rewards/margins": 3.2108778953552246,
      "rewards/rejected": 0.8093411326408386,
      "ronpo/logit": 0.3210877776145935,
      "ronpo/residual": 0.3173868954181671,
      "ronpo/residual_abs": 0.3734723925590515,
      "ronpo/target": 0.003700877772644162,
      "ronpo/target_abs": 0.09122522175312042,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 675
    },
    {
      "drift/chosen_abs": 0.19475491344928741,
      "drift/rejected_abs": 0.1171645075082779,
      "epoch": 0.21613031386571316,
      "grad_norm": 3.171875,
      "learning_rate": 4.317572222928703e-09,
      "logits/chosen": -1.9540669918060303,
      "logits/rejected": -1.9495131969451904,
      "logps/chosen": -0.2881017029285431,
      "logps/rejected": -0.28433549404144287,
      "loss": 0.06940118968486786,
      "loss/core": 0.05890883132815361,
      "loss/preference_sft": 0.285450279712677,
      "loss/reference_anchor": 0.18130208551883698,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9457314014434814,
      "rewards/margins": 0.7747111320495605,
      "rewards/rejected": 1.1710201501846313,
      "ronpo/logit": 0.07747111469507217,
      "ronpo/residual": 0.08135292679071426,
      "ronpo/residual_abs": 0.13843822479248047,
      "ronpo/target": -0.0038818151224404573,
      "ronpo/target_abs": 0.08158716559410095,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 680
    },
    {
      "drift/chosen_abs": 0.18266239762306213,
      "drift/rejected_abs": 0.09462437778711319,
      "epoch": 0.21771950735001985,
      "grad_norm": 3.421875,
      "learning_rate": 4.135867420514276e-09,
      "logits/chosen": -1.9716804027557373,
      "logits/rejected": -2.060253620147705,
      "logps/chosen": -0.2921956181526184,
      "logps/rejected": -0.2996564209461212,
      "loss": 0.05632435530424118,
      "loss/core": 0.05092400312423706,
      "loss/preference_sft": 0.29816001653671265,
      "loss/reference_anchor": 0.07819100469350815,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.825413465499878,
      "rewards/margins": 0.8813973665237427,
      "rewards/rejected": 0.9440159797668457,
      "ronpo/logit": 0.08813974261283875,
      "ronpo/residual": 0.08088794350624084,
      "ronpo/residual_abs": 0.14101745188236237,
      "ronpo/target": 0.007251805160194635,
      "ronpo/target_abs": 0.08236397802829742,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 685
    },
    {
      "drift/chosen_abs": 0.4524809420108795,
      "drift/rejected_abs": 0.1857810914516449,
      "epoch": 0.21930870083432658,
      "grad_norm": 434.0,
      "learning_rate": 3.957308026295035e-09,
      "logits/chosen": -1.9309346675872803,
      "logits/rejected": -1.9748626947402954,
      "logps/chosen": -0.27637702226638794,
      "logps/rejected": -0.271381676197052,
      "loss": 0.8734847903251648,
      "loss/core": 0.8166624903678894,
      "loss/preference_sft": 0.2828080356121063,
      "loss/reference_anchor": 1.1081647872924805,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 4.524248123168945,
      "rewards/margins": 2.668072462081909,
      "rewards/rejected": 1.856175422668457,
      "ronpo/logit": 0.2668072581291199,
      "ronpo/residual": 0.25683221220970154,
      "ronpo/residual_abs": 0.3273332417011261,
      "ronpo/target": 0.009975056163966656,
      "ronpo/target_abs": 0.09136839956045151,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 690
    },
    {
      "drift/chosen_abs": 0.18365684151649475,
      "drift/rejected_abs": 0.11019404232501984,
      "epoch": 0.2208978943186333,
      "grad_norm": 12.5,
      "learning_rate": 3.781961189156609e-09,
      "logits/chosen": -2.160966634750366,
      "logits/rejected": -2.1480095386505127,
      "logps/chosen": -0.298018217086792,
      "logps/rejected": -0.30044853687286377,
      "loss": 0.10467197746038437,
      "loss/core": 0.0970427542924881,
      "loss/preference_sft": 0.2983938455581665,
      "loss/reference_anchor": 0.12274493277072906,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8359683752059937,
      "rewards/margins": 0.7349220514297485,
      "rewards/rejected": 1.1010462045669556,
      "ronpo/logit": 0.07349221408367157,
      "ronpo/residual": 0.05487983301281929,
      "ronpo/residual_abs": 0.16237375140190125,
      "ronpo/target": 0.01861238107085228,
      "ronpo/target_abs": 0.07778690755367279,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 695
    },
    {
      "drift/chosen_abs": 0.22338958084583282,
      "drift/rejected_abs": 0.13935190439224243,
      "epoch": 0.22248708780294002,
      "grad_norm": 6.15625,
      "learning_rate": 3.609892849873286e-09,
      "logits/chosen": -2.2161097526550293,
      "logits/rejected": -2.176314115524292,
      "logps/chosen": -0.321241557598114,
      "logps/rejected": -0.3120388686656952,
      "loss": 0.4451649785041809,
      "loss/core": 0.4236728250980377,
      "loss/preference_sft": 0.3101545572280884,
      "loss/reference_anchor": 0.398827463388443,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.233896017074585,
      "rewards/margins": 0.842010498046875,
      "rewards/rejected": 1.39188551902771,
      "ronpo/logit": 0.08420103788375854,
      "ronpo/residual": 0.0672958642244339,
      "ronpo/residual_abs": 0.19979240000247955,
      "ronpo/target": 0.016905177384614944,
      "ronpo/target_abs": 0.06755395233631134,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 700
    },
    {
      "drift/chosen_abs": 0.1843368113040924,
      "drift/rejected_abs": 0.08406703919172287,
      "epoch": 0.22407628128724671,
      "grad_norm": 3.265625,
      "learning_rate": 3.441167716310389e-09,
      "logits/chosen": -2.2613587379455566,
      "logits/rejected": -2.366189479827881,
      "logps/chosen": -0.2671636939048767,
      "logps/rejected": -0.2705652117729187,
      "loss": 0.07335400581359863,
      "loss/core": 0.06695351749658585,
      "loss/preference_sft": 0.27716490626335144,
      "loss/reference_anchor": 0.10029320418834686,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8426601886749268,
      "rewards/margins": 1.0054227113723755,
      "rewards/rejected": 0.8372377157211304,
      "ronpo/logit": 0.10054226219654083,
      "ronpo/residual": 0.08631283044815063,
      "ronpo/residual_abs": 0.14657628536224365,
      "ronpo/target": 0.014229427091777325,
      "ronpo/target_abs": 0.08256835490465164,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 705
    },
    {
      "drift/chosen_abs": 0.28539392352104187,
      "drift/rejected_abs": 0.10650794208049774,
      "epoch": 0.22566547477155344,
      "grad_norm": 108.0,
      "learning_rate": 3.275849239090295e-09,
      "logits/chosen": -2.166898727416992,
      "logits/rejected": -2.178861141204834,
      "logps/chosen": -0.30302557349205017,
      "logps/rejected": -0.3004283905029297,
      "loss": 0.707269549369812,
      "loss/core": 0.6864758729934692,
      "loss/preference_sft": 0.3012581765651703,
      "loss/reference_anchor": 0.38574856519699097,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.8538074493408203,
      "rewards/margins": 1.9227615594863892,
      "rewards/rejected": 0.9310457110404968,
      "ronpo/logit": 0.19227614998817444,
      "ronpo/residual": 0.18570926785469055,
      "ronpo/residual_abs": 0.25992774963378906,
      "ronpo/target": 0.006566896103322506,
      "ronpo/target_abs": 0.08580425381660461,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 710
    },
    {
      "drift/chosen_abs": 0.16650691628456116,
      "drift/rejected_abs": 0.09241025149822235,
      "epoch": 0.22725466825586016,
      "grad_norm": 0.92578125,
      "learning_rate": 3.11399958773126e-09,
      "logits/chosen": -2.047321081161499,
      "logits/rejected": -1.9679555892944336,
      "logps/chosen": -0.30258211493492126,
      "logps/rejected": -0.29988473653793335,
      "loss": 0.07320668548345566,
      "loss/core": 0.06755281984806061,
      "loss/preference_sft": 0.30704405903816223,
      "loss/reference_anchor": 0.08237285912036896,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.6642707586288452,
      "rewards/margins": 0.740811824798584,
      "rewards/rejected": 0.9234589338302612,
      "ronpo/logit": 0.07408119738101959,
      "ronpo/residual": 0.07377193868160248,
      "ronpo/residual_abs": 0.150286003947258,
      "ronpo/target": 0.0003092583210673183,
      "ronpo/target_abs": 0.08682587742805481,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 715
    },
    {
      "drift/chosen_abs": 0.3417677581310272,
      "drift/rejected_abs": 0.1455438882112503,
      "epoch": 0.22884386174016685,
      "grad_norm": 508.0,
      "learning_rate": 2.955679627267997e-09,
      "logits/chosen": -1.951354742050171,
      "logits/rejected": -2.0309109687805176,
      "logps/chosen": -0.2951735854148865,
      "logps/rejected": -0.28592047095298767,
      "loss": 0.5634924173355103,
      "loss/core": 0.5344976782798767,
      "loss/preference_sft": 0.2908739447593689,
      "loss/reference_anchor": 0.5508082509040833,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.4174652099609375,
      "rewards/margins": 2.024085283279419,
      "rewards/rejected": 1.393379807472229,
      "ronpo/logit": 0.20240852236747742,
      "ronpo/residual": 0.18394221365451813,
      "ronpo/residual_abs": 0.25016313791275024,
      "ronpo/target": 0.018466297537088394,
      "ronpo/target_abs": 0.07852284610271454,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 720
    },
    {
      "drift/chosen_abs": 0.295247346162796,
      "drift/rejected_abs": 0.06837625056505203,
      "epoch": 0.23043305522447358,
      "grad_norm": 4.125,
      "learning_rate": 2.8009488953628218e-09,
      "logits/chosen": -2.0851798057556152,
      "logits/rejected": -2.0262653827667236,
      "logps/chosen": -0.2910119891166687,
      "logps/rejected": -0.2917248606681824,
      "loss": 1.1185815334320068,
      "loss/core": 1.0883275270462036,
      "loss/preference_sft": 0.2941739857196808,
      "loss/reference_anchor": 0.5756635069847107,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9508135318756104,
      "rewards/margins": 2.2699127197265625,
      "rewards/rejected": 0.6809011101722717,
      "ronpo/logit": 0.22699125111103058,
      "ronpo/residual": 0.23236696422100067,
      "ronpo/residual_abs": 0.28527194261550903,
      "ronpo/target": -0.005375738255679607,
      "ronpo/target_abs": 0.08742313086986542,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 725
    },
    {
      "drift/chosen_abs": 0.30962568521499634,
      "drift/rejected_abs": 0.09282008558511734,
      "epoch": 0.2320222487087803,
      "grad_norm": 1712.0,
      "learning_rate": 2.649865579915976e-09,
      "logits/chosen": -2.178410291671753,
      "logits/rejected": -2.213578462600708,
      "logps/chosen": -0.30459070205688477,
      "logps/rejected": -0.2987954616546631,
      "loss": 0.8561824560165405,
      "loss/core": 0.8262929916381836,
      "loss/preference_sft": 0.3033561408519745,
      "loss/reference_anchor": 0.5674565434455872,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.0953526496887207,
      "rewards/margins": 2.171635389328003,
      "rewards/rejected": 0.9237171411514282,
      "ronpo/logit": 0.21716353297233582,
      "ronpo/residual": 0.20029985904693604,
      "ronpo/residual_abs": 0.2601030468940735,
      "ronpo/target": 0.016863666474819183,
      "ronpo/target_abs": 0.07844550907611847,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 730
    },
    {
      "drift/chosen_abs": 0.41004911065101624,
      "drift/rejected_abs": 0.15294381976127625,
      "epoch": 0.23361144219308702,
      "grad_norm": 11.3125,
      "learning_rate": 2.5024864971835508e-09,
      "logits/chosen": -1.8713624477386475,
      "logits/rejected": -1.9245052337646484,
      "logps/chosen": -0.31250444054603577,
      "logps/rejected": -0.2822606861591339,
      "loss": 1.3640409708023071,
      "loss/core": 1.3203461170196533,
      "loss/preference_sft": 0.28716808557510376,
      "loss/reference_anchor": 0.8451792597770691,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.098076343536377,
      "rewards/margins": 2.5743887424468994,
      "rewards/rejected": 1.5236876010894775,
      "ronpo/logit": 0.25743886828422546,
      "ronpo/residual": 0.2717561721801758,
      "ronpo/residual_abs": 0.36912673711776733,
      "ronpo/target": -0.014317306689918041,
      "ronpo/target_abs": 0.08681128919124603,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 735
    },
    {
      "drift/chosen_abs": 0.40702804923057556,
      "drift/rejected_abs": 0.17471705377101898,
      "epoch": 0.23520063567739372,
      "grad_norm": 8.0625,
      "learning_rate": 2.3588670704111995e-09,
      "logits/chosen": -2.0632808208465576,
      "logits/rejected": -2.0540781021118164,
      "logps/chosen": -0.27535516023635864,
      "logps/rejected": -0.27248817682266235,
      "loss": 0.8561609983444214,
      "loss/core": 0.8036273717880249,
      "loss/preference_sft": 0.2769920825958252,
      "loss/reference_anchor": 1.0229724645614624,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 4.0702805519104,
      "rewards/margins": 2.323690414428711,
      "rewards/rejected": 1.7465903759002686,
      "ronpo/logit": 0.232369065284729,
      "ronpo/residual": 0.21827132999897003,
      "ronpo/residual_abs": 0.3096594512462616,
      "ronpo/target": 0.014097737148404121,
      "ronpo/target_abs": 0.09412308037281036,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 740
    },
    {
      "drift/chosen_abs": 0.27846285700798035,
      "drift/rejected_abs": 0.13982626795768738,
      "epoch": 0.23678982916170044,
      "grad_norm": 1.015625,
      "learning_rate": 2.219061308991721e-09,
      "logits/chosen": -1.9662797451019287,
      "logits/rejected": -1.9410368204116821,
      "logps/chosen": -0.31107696890830994,
      "logps/rejected": -0.29558494687080383,
      "loss": 0.4125809669494629,
      "loss/core": 0.39409929513931274,
      "loss/preference_sft": 0.3031148314476013,
      "loss/reference_anchor": 0.3393229842185974,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.7828688621520996,
      "rewards/margins": 1.401062250137329,
      "rewards/rejected": 1.3818066120147705,
      "ronpo/logit": 0.14010624587535858,
      "ronpo/residual": 0.1506578028202057,
      "ronpo/residual_abs": 0.22105178236961365,
      "ronpo/target": -0.010551569052040577,
      "ronpo/target_abs": 0.07253938913345337,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 745
    },
    {
      "drift/chosen_abs": 0.493083655834198,
      "drift/rejected_abs": 0.08905399590730667,
      "epoch": 0.23837902264600716,
      "grad_norm": 35.0,
      "learning_rate": 2.0831217881543558e-09,
      "logits/chosen": -1.8739421367645264,
      "logits/rejected": -2.0240061283111572,
      "logps/chosen": -0.32642191648483276,
      "logps/rejected": -0.3034181594848633,
      "loss": 1.7241404056549072,
      "loss/core": 1.6729081869125366,
      "loss/preference_sft": 0.30714109539985657,
      "loss/reference_anchor": 0.9939321279525757,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.929997444152832,
      "rewards/margins": 4.04317045211792,
      "rewards/rejected": 0.8868273496627808,
      "ronpo/logit": 0.40431705117225647,
      "ronpo/residual": 0.39883852005004883,
      "ronpo/residual_abs": 0.4577086567878723,
      "ronpo/target": 0.00547849852591753,
      "ronpo/target_abs": 0.09150009602308273,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 750
    },
    {
      "drift/chosen_abs": 0.5546950101852417,
      "drift/rejected_abs": 0.2534758150577545,
      "epoch": 0.23996821613031386,
      "grad_norm": 173.0,
      "learning_rate": 1.951099629193366e-09,
      "logits/chosen": -1.7793035507202148,
      "logits/rejected": -1.708032250404358,
      "logps/chosen": -0.26937034726142883,
      "logps/rejected": -0.2777889668941498,
      "loss": 1.7983328104019165,
      "loss/core": 1.7351124286651611,
      "loss/preference_sft": 0.264911413192749,
      "loss/reference_anchor": 1.2379202842712402,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 5.546171188354492,
      "rewards/margins": 3.0143425464630127,
      "rewards/rejected": 2.5318284034729004,
      "ronpo/logit": 0.3014342784881592,
      "ronpo/residual": 0.2820184528827667,
      "ronpo/residual_abs": 0.4880169928073883,
      "ronpo/target": 0.019415846094489098,
      "ronpo/target_abs": 0.08249348402023315,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 755
    },
    {
      "drift/chosen_abs": 0.25574034452438354,
      "drift/rejected_abs": 0.10070861876010895,
      "epoch": 0.24155740961462058,
      "grad_norm": 1.3828125,
      "learning_rate": 1.823044480243431e-09,
      "logits/chosen": -2.2029078006744385,
      "logits/rejected": -2.2537450790405273,
      "logps/chosen": -0.2618194818496704,
      "logps/rejected": -0.26269036531448364,
      "loss": 0.4067454934120178,
      "loss/core": 0.3905092477798462,
      "loss/preference_sft": 0.270187646150589,
      "loss/reference_anchor": 0.2977067232131958,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5571861267089844,
      "rewards/margins": 1.5527894496917725,
      "rewards/rejected": 1.004396677017212,
      "ronpo/logit": 0.15527895092964172,
      "ronpo/residual": 0.13664746284484863,
      "ronpo/residual_abs": 0.21219086647033691,
      "ronpo/target": 0.01863149181008339,
      "ronpo/target_abs": 0.07967917621135712,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 760
    },
    {
      "drift/chosen_abs": 0.16783472895622253,
      "drift/rejected_abs": 0.09149236977100372,
      "epoch": 0.2431466030989273,
      "grad_norm": 1.359375,
      "learning_rate": 1.699004497608994e-09,
      "logits/chosen": -2.3972268104553223,
      "logits/rejected": -2.3360962867736816,
      "logps/chosen": -0.2857986092567444,
      "logps/rejected": -0.2897695004940033,
      "loss": 0.04794660210609436,
      "loss/core": 0.04115251451730728,
      "loss/preference_sft": 0.28739649057388306,
      "loss/reference_anchor": 0.10714216530323029,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.677700400352478,
      "rewards/margins": 0.7631028890609741,
      "rewards/rejected": 0.9145975112915039,
      "ronpo/logit": 0.07631029188632965,
      "ronpo/residual": 0.07975935935974121,
      "ronpo/residual_abs": 0.14109909534454346,
      "ronpo/target": -0.0034490670077502728,
      "ronpo/target_abs": 0.0860501304268837,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 765
    },
    {
      "drift/chosen_abs": 0.3305698037147522,
      "drift/rejected_abs": 0.10161750018596649,
      "epoch": 0.24473579658323402,
      "grad_norm": 57.0,
      "learning_rate": 1.579026327654666e-09,
      "logits/chosen": -1.9631493091583252,
      "logits/rejected": -2.093764543533325,
      "logps/chosen": -0.2555374801158905,
      "logps/rejected": -0.27383852005004883,
      "loss": 0.6095373630523682,
      "loss/core": 0.5895863771438599,
      "loss/preference_sft": 0.27055805921554565,
      "loss/reference_anchor": 0.3719633221626282,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.3039238452911377,
      "rewards/margins": 2.3924434185028076,
      "rewards/rejected": 0.9114801287651062,
      "ronpo/logit": 0.23924437165260315,
      "ronpo/residual": 0.251014769077301,
      "ronpo/residual_abs": 0.2923506796360016,
      "ronpo/target": -0.011770381592214108,
      "ronpo/target_abs": 0.08334879577159882,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 770
    },
    {
      "drift/chosen_abs": 0.34415099024772644,
      "drift/rejected_abs": 0.14330972731113434,
      "epoch": 0.24632499006754072,
      "grad_norm": 1.3828125,
      "learning_rate": 1.4631550892634126e-09,
      "logits/chosen": -1.82931387424469,
      "logits/rejected": -1.8169645071029663,
      "logps/chosen": -0.2950168550014496,
      "logps/rejected": -0.2965241074562073,
      "loss": 0.551666259765625,
      "loss/core": 0.5294795036315918,
      "loss/preference_sft": 0.2889275848865509,
      "loss/reference_anchor": 0.4148431420326233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.439793825149536,
      "rewards/margins": 2.046867847442627,
      "rewards/rejected": 1.3929258584976196,
      "ronpo/logit": 0.20468676090240479,
      "ronpo/residual": 0.21019069850444794,
      "ronpo/residual_abs": 0.25614383816719055,
      "ronpo/target": -0.00550392922013998,
      "ronpo/target_abs": 0.09071238338947296,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 775
    },
    {
      "drift/chosen_abs": 0.3496606945991516,
      "drift/rejected_abs": 0.15519936382770538,
      "epoch": 0.24791418355184744,
      "grad_norm": 2.953125,
      "learning_rate": 1.3514343568692133e-09,
      "logits/chosen": -2.155691385269165,
      "logits/rejected": -2.17978572845459,
      "logps/chosen": -0.27056634426116943,
      "logps/rejected": -0.2793372571468353,
      "loss": 0.3566157817840576,
      "loss/core": 0.33336490392684937,
      "loss/preference_sft": 0.2702133059501648,
      "loss/reference_anchor": 0.43799638748168945,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4961299896240234,
      "rewards/margins": 1.9453226327896118,
      "rewards/rejected": 1.5508073568344116,
      "ronpo/logit": 0.19453224539756775,
      "ronpo/residual": 0.2001989632844925,
      "ronpo/residual_abs": 0.2824886739253998,
      "ronpo/target": -0.00566669087857008,
      "ronpo/target_abs": 0.08824430406093597,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 780
    },
    {
      "drift/chosen_abs": 0.47809037566185,
      "drift/rejected_abs": 0.10961903631687164,
      "epoch": 0.24950337703615416,
      "grad_norm": 4.03125,
      "learning_rate": 1.2439061440704723e-09,
      "logits/chosen": -2.104382038116455,
      "logits/rejected": -2.215803623199463,
      "logps/chosen": -0.2707865536212921,
      "logps/rejected": -0.26247674226760864,
      "loss": 1.8099985122680664,
      "loss/core": 1.7617250680923462,
      "loss/preference_sft": 0.27354034781455994,
      "loss/reference_anchor": 0.9381135106086731,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.774395942687988,
      "rewards/margins": 3.7094340324401855,
      "rewards/rejected": 1.0649611949920654,
      "ronpo/logit": 0.37094348669052124,
      "ronpo/residual": 0.37685176730155945,
      "ronpo/residual_abs": 0.4302590489387512,
      "ronpo/target": -0.005908310413360596,
      "ronpo/target_abs": 0.08773339539766312,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 785
    },
    {
      "drift/chosen_abs": 0.3598199188709259,
      "drift/rejected_abs": 0.12054183334112167,
      "epoch": 0.25109257052046086,
      "grad_norm": 208.0,
      "learning_rate": 1.1406108878304468e-09,
      "logits/chosen": -2.0071957111358643,
      "logits/rejected": -2.026601552963257,
      "logps/chosen": -0.31191474199295044,
      "logps/rejected": -0.29044240713119507,
      "loss": 0.5209998488426208,
      "loss/core": 0.4963608384132385,
      "loss/preference_sft": 0.3023087680339813,
      "loss/reference_anchor": 0.4625491201877594,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.5980522632598877,
      "rewards/margins": 2.4207026958465576,
      "rewards/rejected": 1.1773498058319092,
      "ronpo/logit": 0.2420702427625656,
      "ronpo/residual": 0.24213683605194092,
      "ronpo/residual_abs": 0.30981531739234924,
      "ronpo/target": -6.654262688243762e-05,
      "ronpo/target_abs": 0.08093582093715668,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 790
    },
    {
      "drift/chosen_abs": 0.2403639853000641,
      "drift/rejected_abs": 0.11303351819515228,
      "epoch": 0.2526817640047676,
      "grad_norm": 1.0234375,
      "learning_rate": 1.0415874332705382e-09,
      "logits/chosen": -2.1333625316619873,
      "logits/rejected": -2.168160915374756,
      "logps/chosen": -0.25950658321380615,
      "logps/rejected": -0.25287556648254395,
      "loss": 0.1263783574104309,
      "loss/core": 0.11855258047580719,
      "loss/preference_sft": 0.24921035766601562,
      "loss/reference_anchor": 0.13159438967704773,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.4028306007385254,
      "rewards/margins": 1.274418592453003,
      "rewards/rejected": 1.1284120082855225,
      "ronpo/logit": 0.12744185328483582,
      "ronpo/residual": 0.12694984674453735,
      "ronpo/residual_abs": 0.19664797186851501,
      "ronpo/target": 0.0004920096835121512,
      "ronpo/target_abs": 0.08606652170419693,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 795
    },
    {
      "drift/chosen_abs": 0.2836977541446686,
      "drift/rejected_abs": 0.15890619158744812,
      "epoch": 0.2542709574890743,
      "grad_norm": 17.75,
      "learning_rate": 9.468730190622484e-10,
      "logits/chosen": -2.169442653656006,
      "logits/rejected": -2.2410874366760254,
      "logps/chosen": -0.26969194412231445,
      "logps/rejected": -0.27040061354637146,
      "loss": 0.22600746154785156,
      "loss/core": 0.18485912680625916,
      "loss/preference_sft": 0.26238492131233215,
      "loss/reference_anchor": 0.7967280149459839,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.8366193771362305,
      "rewards/margins": 1.2492666244506836,
      "rewards/rejected": 1.587352991104126,
      "ronpo/logit": 0.12492667138576508,
      "ronpo/residual": 0.11071588844060898,
      "ronpo/residual_abs": 0.17527928948402405,
      "ronpo/target": 0.014210781082510948,
      "ronpo/target_abs": 0.07547663152217865,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 800
    },
    {
      "drift/chosen_abs": 0.3189254403114319,
      "drift/rejected_abs": 0.1377745419740677,
      "epoch": 0.255860150973381,
      "grad_norm": 196.0,
      "learning_rate": 8.565032634232195e-10,
      "logits/chosen": -2.275631904602051,
      "logits/rejected": -2.237766742706299,
      "logps/chosen": -0.3036332428455353,
      "logps/rejected": -0.30700454115867615,
      "loss": 0.9040266275405884,
      "loss/core": 0.8683737516403198,
      "loss/preference_sft": 0.2961069643497467,
      "loss/reference_anchor": 0.6834448575973511,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.1892545223236084,
      "rewards/margins": 1.8560397624969482,
      "rewards/rejected": 1.3332146406173706,
      "ronpo/logit": 0.1856040060520172,
      "ronpo/residual": 0.17766329646110535,
      "ronpo/residual_abs": 0.3051329255104065,
      "ronpo/target": 0.007940702140331268,
      "ronpo/target_abs": 0.08157651126384735,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 805
    },
    {
      "drift/chosen_abs": 0.21167591214179993,
      "drift/rejected_abs": 0.11085629463195801,
      "epoch": 0.25744934445768775,
      "grad_norm": 7.65625,
      "learning_rate": 7.705121507227019e-10,
      "logits/chosen": -2.0630433559417725,
      "logits/rejected": -2.1151208877563477,
      "logps/chosen": -0.2830791175365448,
      "logps/rejected": -0.28679680824279785,
      "loss": 0.05285482853651047,
      "loss/core": 0.046348024159669876,
      "loss/preference_sft": 0.2829565405845642,
      "loss/reference_anchor": 0.10184051841497421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.114663600921631,
      "rewards/margins": 1.0063090324401855,
      "rewards/rejected": 1.1083545684814453,
      "ronpo/logit": 0.10063091665506363,
      "ronpo/residual": 0.09154792875051498,
      "ronpo/residual_abs": 0.1593920886516571,
      "ronpo/target": 0.009082979522645473,
      "ronpo/target_abs": 0.08527089655399323,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 810
    },
    {
      "drift/chosen_abs": 0.25946545600891113,
      "drift/rejected_abs": 0.16305600106716156,
      "epoch": 0.25903853794199444,
      "grad_norm": 13.125,
      "learning_rate": 6.889320187013991e-10,
      "logits/chosen": -2.107985019683838,
      "logits/rejected": -2.0296835899353027,
      "logps/chosen": -0.26717063784599304,
      "logps/rejected": -0.2779156267642975,
      "loss": 0.23226383328437805,
      "loss/core": 0.22093184292316437,
      "loss/preference_sft": 0.26824456453323364,
      "loss/reference_anchor": 0.19981497526168823,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.594414710998535,
      "rewards/margins": 0.9783709645271301,
      "rewards/rejected": 1.616044044494629,
      "ronpo/logit": 0.09783709049224854,
      "ronpo/residual": 0.07867446541786194,
      "ronpo/residual_abs": 0.22327768802642822,
      "ronpo/target": 0.019162625074386597,
      "ronpo/target_abs": 0.07972420752048492,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 815
    },
    {
      "drift/chosen_abs": 0.1991885006427765,
      "drift/rejected_abs": 0.11371134221553802,
      "epoch": 0.26062773142630113,
      "grad_norm": 2.765625,
      "learning_rate": 6.117935463105808e-10,
      "logits/chosen": -2.0220797061920166,
      "logits/rejected": -1.9778063297271729,
      "logps/chosen": -0.2764715254306793,
      "logps/rejected": -0.2888193130493164,
      "loss": 0.07094263285398483,
      "loss/core": 0.06167911738157272,
      "loss/preference_sft": 0.2758772373199463,
      "loss/reference_anchor": 0.1576823890209198,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9880034923553467,
      "rewards/margins": 0.851704478263855,
      "rewards/rejected": 1.1362990140914917,
      "ronpo/logit": 0.0851704329252243,
      "ronpo/residual": 0.09892288595438004,
      "ronpo/residual_abs": 0.15847130119800568,
      "ronpo/target": -0.013752436265349388,
      "ronpo/target_abs": 0.08235911279916763,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 820
    },
    {
      "drift/chosen_abs": 0.288280189037323,
      "drift/rejected_abs": 0.1897338479757309,
      "epoch": 0.2622169249106079,
      "grad_norm": 78.0,
      "learning_rate": 5.391257421749826e-10,
      "logits/chosen": -2.2123422622680664,
      "logits/rejected": -2.1602697372436523,
      "logps/chosen": -0.2984291613101959,
      "logps/rejected": -0.3011440932750702,
      "loss": 0.3465138375759125,
      "loss/core": 0.3028988242149353,
      "loss/preference_sft": 0.2975984513759613,
      "loss/reference_anchor": 0.8425408601760864,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.8811376094818115,
      "rewards/margins": 0.9950310587882996,
      "rewards/rejected": 1.8861068487167358,
      "ronpo/logit": 0.09950310736894608,
      "ronpo/residual": 0.09332647174596786,
      "ronpo/residual_abs": 0.21904487907886505,
      "ronpo/target": 0.006176639813929796,
      "ronpo/target_abs": 0.0774788185954094,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 825
    },
    {
      "drift/chosen_abs": 0.1840457022190094,
      "drift/rejected_abs": 0.11357797682285309,
      "epoch": 0.2638061183949146,
      "grad_norm": 34.75,
      "learning_rate": 4.709559336838461e-10,
      "logits/chosen": -2.2351510524749756,
      "logits/rejected": -2.236767292022705,
      "logps/chosen": -0.27836111187934875,
      "logps/rejected": -0.2729012370109558,
      "loss": 0.13388831913471222,
      "loss/core": 0.12568429112434387,
      "loss/preference_sft": 0.2821260094642639,
      "loss/reference_anchor": 0.1358683705329895,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8400017023086548,
      "rewards/margins": 0.7065569758415222,
      "rewards/rejected": 1.1334445476531982,
      "ronpo/logit": 0.0706556960940361,
      "ronpo/residual": 0.05800905078649521,
      "ronpo/residual_abs": 0.18142782151699066,
      "ronpo/target": 0.012646635994315147,
      "ronpo/target_abs": 0.09177650511264801,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 830
    },
    {
      "drift/chosen_abs": 0.4068368971347809,
      "drift/rejected_abs": 0.1611480414867401,
      "epoch": 0.2653953118792213,
      "grad_norm": 24.625,
      "learning_rate": 4.0730975671420253e-10,
      "logits/chosen": -2.1904501914978027,
      "logits/rejected": -2.1559247970581055,
      "logps/chosen": -0.242096945643425,
      "logps/rejected": -0.2459816187620163,
      "loss": 1.0775315761566162,
      "loss/core": 1.0418219566345215,
      "loss/preference_sft": 0.24760499596595764,
      "loss/reference_anchor": 0.6894329786300659,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.068270683288574,
      "rewards/margins": 2.4569103717803955,
      "rewards/rejected": 1.6113605499267578,
      "ronpo/logit": 0.24569101631641388,
      "ronpo/residual": 0.24678480625152588,
      "ronpo/residual_abs": 0.35724449157714844,
      "ronpo/target": -0.001093805069103837,
      "ronpo/target_abs": 0.08144358545541763,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 835
    },
    {
      "drift/chosen_abs": 0.2863476276397705,
      "drift/rejected_abs": 0.13716426491737366,
      "epoch": 0.266984505363528,
      "grad_norm": 0.734375,
      "learning_rate": 3.482111459902695e-10,
      "logits/chosen": -2.364701986312866,
      "logits/rejected": -2.425690174102783,
      "logps/chosen": -0.2671613395214081,
      "logps/rejected": -0.264343798160553,
      "loss": 0.348602831363678,
      "loss/core": 0.32751885056495667,
      "loss/preference_sft": 0.27554088830947876,
      "loss/reference_anchor": 0.3941258490085602,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.8634347915649414,
      "rewards/margins": 1.4927136898040771,
      "rewards/rejected": 1.3707211017608643,
      "ronpo/logit": 0.1492713987827301,
      "ronpo/residual": 0.15882544219493866,
      "ronpo/residual_abs": 0.20727181434631348,
      "ronpo/target": -0.009554052725434303,
      "ronpo/target_abs": 0.09001000225543976,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 840
    },
    {
      "drift/chosen_abs": 0.2981222867965698,
      "drift/rejected_abs": 0.18478380143642426,
      "epoch": 0.2685736988478347,
      "grad_norm": 6.46875,
      "learning_rate": 2.9368232608258796e-10,
      "logits/chosen": -2.143329381942749,
      "logits/rejected": -2.1359808444976807,
      "logps/chosen": -0.24788551032543182,
      "logps/rejected": -0.23464611172676086,
      "loss": 0.09194006025791168,
      "loss/core": 0.07542302459478378,
      "loss/preference_sft": 0.24129502475261688,
      "loss/reference_anchor": 0.30621108412742615,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.979228973388672,
      "rewards/margins": 1.13218092918396,
      "rewards/rejected": 1.8470484018325806,
      "ronpo/logit": 0.11321810632944107,
      "ronpo/residual": 0.09495650231838226,
      "ronpo/residual_abs": 0.16129747033119202,
      "ronpo/target": 0.018261589109897614,
      "ronpo/target_abs": 0.08132471889257431,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 845
    },
    {
      "drift/chosen_abs": 0.41143712401390076,
      "drift/rejected_abs": 0.14703157544136047,
      "epoch": 0.2701628923321414,
      "grad_norm": 80.0,
      "learning_rate": 2.4374380305025865e-10,
      "logits/chosen": -1.953258752822876,
      "logits/rejected": -1.88783860206604,
      "logps/chosen": -0.27198326587677,
      "logps/rejected": -0.28331276774406433,
      "loss": 0.8986851572990417,
      "loss/core": 0.8608425259590149,
      "loss/preference_sft": 0.2750104069709778,
      "loss/reference_anchor": 0.7293528318405151,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.112533092498779,
      "rewards/margins": 2.645296096801758,
      "rewards/rejected": 1.4672367572784424,
      "ronpo/logit": 0.26452964544296265,
      "ronpo/residual": 0.25969162583351135,
      "ronpo/residual_abs": 0.3339794874191284,
      "ronpo/target": 0.004837975837290287,
      "ronpo/target_abs": 0.08931146562099457,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 850
    },
    {
      "drift/chosen_abs": 0.20746512711048126,
      "drift/rejected_abs": 0.07350940257310867,
      "epoch": 0.27175208581644816,
      "grad_norm": 2.96875,
      "learning_rate": 1.9841435672945938e-10,
      "logits/chosen": -2.3771004676818848,
      "logits/rejected": -2.3643927574157715,
      "logps/chosen": -0.27676576375961304,
      "logps/rejected": -0.2748029828071594,
      "loss": 0.4189775586128235,
      "loss/core": 0.40414971113204956,
      "loss/preference_sft": 0.2725796401500702,
      "loss/reference_anchor": 0.2692995071411133,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0746512413024902,
      "rewards/margins": 1.339883804321289,
      "rewards/rejected": 0.7347675561904907,
      "ronpo/logit": 0.13398835062980652,
      "ronpo/residual": 0.13738013803958893,
      "ronpo/residual_abs": 0.1904793679714203,
      "ronpo/target": -0.003391760867089033,
      "ronpo/target_abs": 0.0801364928483963,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 855
    },
    {
      "drift/chosen_abs": 0.18808570504188538,
      "drift/rejected_abs": 0.10459624230861664,
      "epoch": 0.27334127930075486,
      "grad_norm": 42.25,
      "learning_rate": 1.577110336711096e-10,
      "logits/chosen": -2.201777935028076,
      "logits/rejected": -2.127864360809326,
      "logps/chosen": -0.28073182702064514,
      "logps/rejected": -0.2788235545158386,
      "loss": 0.12705744802951813,
      "loss/core": 0.11867976188659668,
      "loss/preference_sft": 0.27348923683166504,
      "loss/reference_anchor": 0.14020445942878723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8808481693267822,
      "rewards/margins": 0.8413216471672058,
      "rewards/rejected": 1.0395262241363525,
      "ronpo/logit": 0.08413217961788177,
      "ronpo/residual": 0.07758622616529465,
      "ronpo/residual_abs": 0.14888854324817657,
      "ronpo/target": 0.006545948330312967,
      "ronpo/target_abs": 0.07704494148492813,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 860
    },
    {
      "drift/chosen_abs": 0.24855653941631317,
      "drift/rejected_abs": 0.12981536984443665,
      "epoch": 0.2749304727850616,
      "grad_norm": 0.859375,
      "learning_rate": 1.2164914073037048e-10,
      "logits/chosen": -2.090097427368164,
      "logits/rejected": -2.0350818634033203,
      "logps/chosen": -0.2797498106956482,
      "logps/rejected": -0.2887721359729767,
      "loss": 0.11714392900466919,
      "loss/core": 0.10699626058340073,
      "loss/preference_sft": 0.28749600052833557,
      "loss/reference_anchor": 0.1742037832736969,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.485067367553711,
      "rewards/margins": 1.1879299879074097,
      "rewards/rejected": 1.2971374988555908,
      "ronpo/logit": 0.11879298835992813,
      "ronpo/residual": 0.11563484370708466,
      "ronpo/residual_abs": 0.19152960181236267,
      "ronpo/target": 0.003158140927553177,
      "ronpo/target_abs": 0.08096446096897125,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 865
    },
    {
      "drift/chosen_abs": 0.4628352224826813,
      "drift/rejected_abs": 0.22026996314525604,
      "epoch": 0.2765196662693683,
      "grad_norm": 520.0,
      "learning_rate": 9.024223931035357e-11,
      "logits/chosen": -2.098940849304199,
      "logits/rejected": -2.087498188018799,
      "logps/chosen": -0.26976698637008667,
      "logps/rejected": -0.2759985327720642,
      "loss": 0.723773717880249,
      "loss/core": 0.6722049117088318,
      "loss/preference_sft": 0.2798956036567688,
      "loss/reference_anchor": 1.0033875703811646,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 4.626898765563965,
      "rewards/margins": 2.4265968799591064,
      "rewards/rejected": 2.2003021240234375,
      "ronpo/logit": 0.24265968799591064,
      "ronpo/residual": 0.24575605988502502,
      "ronpo/residual_abs": 0.30909624695777893,
      "ronpo/target": -0.0030963898170739412,
      "ronpo/target_abs": 0.08388674259185791,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 870
    },
    {
      "drift/chosen_abs": 0.3066100478172302,
      "drift/rejected_abs": 0.15433122217655182,
      "epoch": 0.278108859753675,
      "grad_norm": 5.90625,
      "learning_rate": 6.350214026223516e-11,
      "logits/chosen": -1.945692777633667,
      "logits/rejected": -1.9127715826034546,
      "logps/chosen": -0.2703152298927307,
      "logps/rejected": -0.27156907320022583,
      "loss": 0.22089354693889618,
      "loss/core": 0.20463630557060242,
      "loss/preference_sft": 0.2696455121040344,
      "loss/reference_anchor": 0.29818084836006165,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.064970016479492,
      "rewards/margins": 1.5222326517105103,
      "rewards/rejected": 1.5427372455596924,
      "ronpo/logit": 0.15222327411174774,
      "ronpo/residual": 0.1579734981060028,
      "ronpo/residual_abs": 0.20636561512947083,
      "ronpo/target": -0.005750220734626055,
      "ronpo/target_abs": 0.07495192438364029,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 875
    },
    {
      "drift/chosen_abs": 0.24454915523529053,
      "drift/rejected_abs": 0.09118019789457321,
      "epoch": 0.27969805323798175,
      "grad_norm": 6.96875,
      "learning_rate": 4.143889944367429e-11,
      "logits/chosen": -2.112548828125,
      "logits/rejected": -2.1549744606018066,
      "logps/chosen": -0.2780202031135559,
      "logps/rejected": -0.29079365730285645,
      "loss": 0.41406092047691345,
      "loss/core": 0.40033167600631714,
      "loss/preference_sft": 0.28780096769332886,
      "loss/reference_anchor": 0.24580499529838562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.4454915523529053,
      "rewards/margins": 1.5506150722503662,
      "rewards/rejected": 0.8948766589164734,
      "ronpo/logit": 0.1550614982843399,
      "ronpo/residual": 0.16445958614349365,
      "ronpo/residual_abs": 0.21083009243011475,
      "ronpo/target": -0.009398087859153748,
      "ronpo/target_abs": 0.0880463570356369,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 880
    },
    {
      "drift/chosen_abs": 0.2654206454753876,
      "drift/rejected_abs": 0.10342685878276825,
      "epoch": 0.28128724672228844,
      "grad_norm": 6.1875,
      "learning_rate": 2.4060813937225312e-11,
      "logits/chosen": -2.159634828567505,
      "logits/rejected": -2.221071243286133,
      "logps/chosen": -0.2973093092441559,
      "logps/rejected": -0.29612240195274353,
      "loss": 0.4108871519565582,
      "loss/core": 0.38317131996154785,
      "loss/preference_sft": 0.2979774475097656,
      "loss/reference_anchor": 0.5245189666748047,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.6539218425750732,
      "rewards/margins": 1.6225885152816772,
      "rewards/rejected": 1.0313332080841064,
      "ronpo/logit": 0.16225887835025787,
      "ronpo/residual": 0.16746975481510162,
      "ronpo/residual_abs": 0.21311867237091064,
      "ronpo/target": -0.005210875067859888,
      "ronpo/target_abs": 0.08115576207637787,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 885
    },
    {
      "drift/chosen_abs": 0.29945269227027893,
      "drift/rejected_abs": 0.12191145122051239,
      "epoch": 0.28287644020659514,
      "grad_norm": 3.625,
      "learning_rate": 1.1374418930135132e-11,
      "logits/chosen": -2.3053576946258545,
      "logits/rejected": -2.2626731395721436,
      "logps/chosen": -0.2816936671733856,
      "logps/rejected": -0.2805249094963074,
      "loss": 0.4204486906528473,
      "loss/core": 0.39468875527381897,
      "loss/preference_sft": 0.28686416149139404,
      "loss/reference_anchor": 0.4865127503871918,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9918599128723145,
      "rewards/margins": 1.7764670848846436,
      "rewards/rejected": 1.21539306640625,
      "ronpo/logit": 0.1776467114686966,
      "ronpo/residual": 0.1912662535905838,
      "ronpo/residual_abs": 0.23552899062633514,
      "ronpo/target": -0.013619527220726013,
      "ronpo/target_abs": 0.08702584356069565,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 890
    },
    {
      "drift/chosen_abs": 0.30740076303482056,
      "drift/rejected_abs": 0.12458650022745132,
      "epoch": 0.2844656336909019,
      "grad_norm": 11.375,
      "learning_rate": 3.3844852567285754e-12,
      "logits/chosen": -2.081117630004883,
      "logits/rejected": -2.069899559020996,
      "logps/chosen": -0.2847122251987457,
      "logps/rejected": -0.2946607172489166,
      "loss": 0.373470276594162,
      "loss/core": 0.3527851700782776,
      "loss/preference_sft": 0.28093528747558594,
      "loss/reference_anchor": 0.3856084942817688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.071903705596924,
      "rewards/margins": 1.9304262399673462,
      "rewards/rejected": 1.1414774656295776,
      "ronpo/logit": 0.19304262101650238,
      "ronpo/residual": 0.18837428092956543,
      "ronpo/residual_abs": 0.23437511920928955,
      "ronpo/target": 0.004668330308049917,
      "ronpo/target_abs": 0.07740633934736252,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 895
    },
    {
      "drift/chosen_abs": 0.15997305512428284,
      "drift/rejected_abs": 0.08303854614496231,
      "epoch": 0.2860548271752086,
      "grad_norm": 15.25,
      "learning_rate": 9.401760429905703e-14,
      "logits/chosen": -2.103471279144287,
      "logits/rejected": -2.086383819580078,
      "logps/chosen": -0.3180239498615265,
      "logps/rejected": -0.3187870681285858,
      "loss": 0.14068026840686798,
      "loss/core": 0.13307660818099976,
      "loss/preference_sft": 0.3193908929824829,
      "loss/reference_anchor": 0.12013395130634308,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.5992281436920166,
      "rewards/margins": 0.8927779197692871,
      "rewards/rejected": 0.7064501643180847,
      "ronpo/logit": 0.08927779644727707,
      "ronpo/residual": 0.08004127442836761,
      "ronpo/residual_abs": 0.140147402882576,
      "ronpo/target": 0.009236516430974007,
      "ronpo/target_abs": 0.06991586834192276,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 900
    },
    {
      "epoch": 0.2860548271752086,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 0.42856775469250147,
      "train_runtime": 7333.3683,
      "train_samples_per_second": 1.964,
      "train_steps_per_second": 0.123
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}