{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2860548271752086,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "drift/chosen_abs": 0.2889712452888489,
      "drift/rejected_abs": 0.17483893036842346,
      "epoch": 0.0015891934843067143,
      "grad_norm": 0.8203125,
      "learning_rate": 2.962962962962963e-09,
      "logits/chosen": -2.2284557819366455,
      "logits/rejected": -2.154376983642578,
      "logps/chosen": -0.2503115236759186,
      "logps/rejected": -0.2550564706325531,
      "loss": 0.26546722650527954,
      "loss/core": 0.23584802448749542,
      "loss/preference_sft": 0.2586488723754883,
      "loss/reference_anchor": 0.3690585196018219,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.8897125720977783,
      "rewards/margins": 1.1418126821517944,
      "rewards/rejected": 1.7478997707366943,
      "ronpo/logit": 0.11418129503726959,
      "ronpo/residual": 0.11854199320077896,
      "ronpo/residual_abs": 0.20326364040374756,
      "ronpo/target": -0.004360715392976999,
      "ronpo/target_abs": 0.03454750031232834,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 5
    },
    {
      "drift/chosen_abs": 0.25061291456222534,
      "drift/rejected_abs": 0.1725693792104721,
      "epoch": 0.0031783869686134287,
      "grad_norm": 5.34375,
      "learning_rate": 6.666666666666666e-09,
      "logits/chosen": -1.9815419912338257,
      "logits/rejected": -1.9222148656845093,
      "logps/chosen": -0.2649174630641937,
      "logps/rejected": -0.27258291840553284,
      "loss": 0.16829581558704376,
      "loss/core": 0.14190886914730072,
      "loss/preference_sft": 0.2717735469341278,
      "loss/reference_anchor": 0.3246484398841858,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.5059151649475098,
      "rewards/margins": 0.7813706398010254,
      "rewards/rejected": 1.7245442867279053,
      "ronpo/logit": 0.07813705503940582,
      "ronpo/residual": 0.06967471539974213,
      "ronpo/residual_abs": 0.15364500880241394,
      "ronpo/target": 0.008462338708341122,
      "ronpo/target_abs": 0.041159503161907196,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 10
    },
    {
      "drift/chosen_abs": 0.14762550592422485,
      "drift/rejected_abs": 0.08012887835502625,
      "epoch": 0.004767580452920143,
      "grad_norm": 7.15625,
      "learning_rate": 1.0370370370370369e-08,
      "logits/chosen": -2.2923481464385986,
      "logits/rejected": -2.2468485832214355,
      "logps/chosen": -0.2759386897087097,
      "logps/rejected": -0.2823220193386078,
      "loss": 0.04410820081830025,
      "loss/core": 0.03747672960162163,
      "loss/preference_sft": 0.27749139070510864,
      "loss/reference_anchor": 0.06067045405507088,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.4754669666290283,
      "rewards/margins": 0.675328254699707,
      "rewards/rejected": 0.8001387715339661,
      "ronpo/logit": 0.06753282248973846,
      "ronpo/residual": 0.08304200321435928,
      "ronpo/residual_abs": 0.10613694041967392,
      "ronpo/target": -0.015509175136685371,
      "ronpo/target_abs": 0.038143090903759,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 15
    },
    {
      "drift/chosen_abs": 0.14132478833198547,
      "drift/rejected_abs": 0.06987211853265762,
      "epoch": 0.006356773937226857,
      "grad_norm": 1.15625,
      "learning_rate": 1.4074074074074075e-08,
      "logits/chosen": -2.2929229736328125,
      "logits/rejected": -2.218808650970459,
      "logps/chosen": -0.27541348338127136,
      "logps/rejected": -0.27921879291534424,
      "loss": 0.02784859575331211,
      "loss/core": 0.02186846174299717,
      "loss/preference_sft": 0.27790480852127075,
      "loss/reference_anchor": 0.05194469541311264,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.4130252599716187,
      "rewards/margins": 0.7176119089126587,
      "rewards/rejected": 0.6954135894775391,
      "ronpo/logit": 0.07176118344068527,
      "ronpo/residual": 0.07096967101097107,
      "ronpo/residual_abs": 0.08470381796360016,
      "ronpo/target": 0.0007915133610367775,
      "ronpo/target_abs": 0.03379639983177185,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 20
    },
    {
      "drift/chosen_abs": 0.23398974537849426,
      "drift/rejected_abs": 0.07662718743085861,
      "epoch": 0.007945967421533572,
      "grad_norm": 1.1171875,
      "learning_rate": 1.7777777777777777e-08,
      "logits/chosen": -2.3544507026672363,
      "logits/rejected": -2.392819881439209,
      "logps/chosen": -0.24027124047279358,
      "logps/rejected": -0.24880602955818176,
      "loss": 0.3059253394603729,
      "loss/core": 0.2891264259815216,
      "loss/preference_sft": 0.2509022355079651,
      "loss/reference_anchor": 0.19889584183692932,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.3385119438171387,
      "rewards/margins": 1.572943925857544,
      "rewards/rejected": 0.765568196773529,
      "ronpo/logit": 0.15729442238807678,
      "ronpo/residual": 0.15887948870658875,
      "ronpo/residual_abs": 0.17786964774131775,
      "ronpo/target": -0.0015850653871893883,
      "ronpo/target_abs": 0.03792804479598999,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 25
    },
    {
      "drift/chosen_abs": 0.20628702640533447,
      "drift/rejected_abs": 0.08774978667497635,
      "epoch": 0.009535160905840286,
      "grad_norm": 316.0,
      "learning_rate": 2.148148148148148e-08,
      "logits/chosen": -2.2282731533050537,
      "logits/rejected": -2.1223556995391846,
      "logps/chosen": -0.2855835556983948,
      "logps/rejected": -0.2826509475708008,
      "loss": 0.18014441430568695,
      "loss/core": 0.16145095229148865,
      "loss/preference_sft": 0.2912216782569885,
      "loss/reference_anchor": 0.2201242446899414,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0624454021453857,
      "rewards/margins": 1.1857421398162842,
      "rewards/rejected": 0.876703143119812,
      "ronpo/logit": 0.11857421696186066,
      "ronpo/residual": 0.11068874597549438,
      "ronpo/residual_abs": 0.14063629508018494,
      "ronpo/target": 0.007885478436946869,
      "ronpo/target_abs": 0.0422462597489357,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 30
    },
    {
      "drift/chosen_abs": 0.20925363898277283,
      "drift/rejected_abs": 0.1305985450744629,
      "epoch": 0.011124354390147001,
      "grad_norm": 3.375,
      "learning_rate": 2.5185185185185182e-08,
      "logits/chosen": -2.122819185256958,
      "logits/rejected": -2.058803081512451,
      "logps/chosen": -0.27063214778900146,
      "logps/rejected": -0.28141161799430847,
      "loss": 0.0532744899392128,
      "loss/core": 0.03867238387465477,
      "loss/preference_sft": 0.2803190052509308,
      "loss/reference_anchor": 0.16666279733181,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0918078422546387,
      "rewards/margins": 0.7871139049530029,
      "rewards/rejected": 1.3046939373016357,
      "ronpo/logit": 0.07871139049530029,
      "ronpo/residual": 0.07295581698417664,
      "ronpo/residual_abs": 0.09958575665950775,
      "ronpo/target": 0.005755572579801083,
      "ronpo/target_abs": 0.036900341510772705,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 35
    },
    {
      "drift/chosen_abs": 0.13457752764225006,
      "drift/rejected_abs": 0.10919027030467987,
      "epoch": 0.012713547874453715,
      "grad_norm": 442.0,
      "learning_rate": 2.8888888888888884e-08,
      "logits/chosen": -2.0784153938293457,
      "logits/rejected": -2.0388412475585938,
      "logps/chosen": -0.31832799315452576,
      "logps/rejected": -0.32437992095947266,
      "loss": 0.23646362125873566,
      "loss/core": 0.22246842086315155,
      "loss/preference_sft": 0.32219210267066956,
      "loss/reference_anchor": 0.15438354015350342,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.3454421758651733,
      "rewards/margins": 0.2544054090976715,
      "rewards/rejected": 1.0910369157791138,
      "ronpo/logit": 0.025440547615289688,
      "ronpo/residual": 0.021605418995022774,
      "ronpo/residual_abs": 0.1361103653907776,
      "ronpo/target": 0.003835138399153948,
      "ronpo/target_abs": 0.034678228199481964,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 40
    },
    {
      "drift/chosen_abs": 0.2000904381275177,
      "drift/rejected_abs": 0.10357439517974854,
      "epoch": 0.014302741358760428,
      "grad_norm": 3.953125,
      "learning_rate": 3.2592592592592596e-08,
      "logits/chosen": -2.1482901573181152,
      "logits/rejected": -2.0773396492004395,
      "logps/chosen": -0.30437272787094116,
      "logps/rejected": -0.29646673798561096,
      "loss": 0.08828647434711456,
      "loss/core": 0.07516898214817047,
      "loss/preference_sft": 0.3076452314853668,
      "loss/reference_anchor": 0.14413544535636902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.0003647804260254,
      "rewards/margins": 0.966138482093811,
      "rewards/rejected": 1.0342261791229248,
      "ronpo/logit": 0.09661386907100677,
      "ronpo/residual": 0.0960509404540062,
      "ronpo/residual_abs": 0.12047521770000458,
      "ronpo/target": 0.0005629155784845352,
      "ronpo/target_abs": 0.0397389717400074,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 45
    },
    {
      "drift/chosen_abs": 0.209742471575737,
      "drift/rejected_abs": 0.17415867745876312,
      "epoch": 0.015891934843067144,
      "grad_norm": 11.875,
      "learning_rate": 3.62962962962963e-08,
      "logits/chosen": -2.2329165935516357,
      "logits/rejected": -2.2811379432678223,
      "logps/chosen": -0.38607966899871826,
      "logps/rejected": -0.37516289949417114,
      "loss": 0.505488932132721,
      "loss/core": 0.46689605712890625,
      "loss/preference_sft": 0.3818490207195282,
      "loss/reference_anchor": 0.4763876497745514,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.0969882011413574,
      "rewards/margins": 1.4074676036834717,
      "rewards/rejected": 0.6895202994346619,
      "ronpo/logit": 0.14074677228927612,
      "ronpo/residual": 0.13970384001731873,
      "ronpo/residual_abs": 0.2018175572156906,
      "ronpo/target": 0.0010429281974211335,
      "ronpo/target_abs": 0.039138179272413254,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 50
    },
    {
      "drift/chosen_abs": 0.2418314665555954,
      "drift/rejected_abs": 0.11427447944879532,
      "epoch": 0.01748112832737386,
      "grad_norm": 1.2421875,
      "learning_rate": 4e-08,
      "logits/chosen": -2.1331610679626465,
      "logits/rejected": -2.10676646232605,
      "logps/chosen": -0.3028566837310791,
      "logps/rejected": -0.29301467537879944,
      "loss": 0.2809545695781708,
      "loss/core": 0.26085159182548523,
      "loss/preference_sft": 0.291143000125885,
      "loss/reference_anchor": 0.2389260083436966,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.417487621307373,
      "rewards/margins": 1.2787439823150635,
      "rewards/rejected": 1.1387437582015991,
      "ronpo/logit": 0.12787440419197083,
      "ronpo/residual": 0.12766289710998535,
      "ronpo/residual_abs": 0.15509015321731567,
      "ronpo/target": 0.00021149823442101479,
      "ronpo/target_abs": 0.0375971794128418,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 55
    },
    {
      "drift/chosen_abs": 0.28653964400291443,
      "drift/rejected_abs": 0.07680980116128922,
      "epoch": 0.01907032181168057,
      "grad_norm": 7.15625,
      "learning_rate": 4.37037037037037e-08,
      "logits/chosen": -2.0188426971435547,
      "logits/rejected": -2.038294553756714,
      "logps/chosen": -0.2578909993171692,
      "logps/rejected": -0.26004260778427124,
      "loss": 0.7025126814842224,
      "loss/core": 0.6709312200546265,
      "loss/preference_sft": 0.25844430923461914,
      "loss/reference_anchor": 0.39524129033088684,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.8651163578033447,
      "rewards/margins": 2.112001419067383,
      "rewards/rejected": 0.7531147003173828,
      "ronpo/logit": 0.21120016276836395,
      "ronpo/residual": 0.2195129692554474,
      "ronpo/residual_abs": 0.2346762865781784,
      "ronpo/target": -0.00831279531121254,
      "ronpo/target_abs": 0.03656155616044998,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 60
    },
    {
      "drift/chosen_abs": 0.23444890975952148,
      "drift/rejected_abs": 0.08467339724302292,
      "epoch": 0.020659515295987287,
      "grad_norm": 163.0,
      "learning_rate": 4.7407407407407405e-08,
      "logits/chosen": -2.3911728858947754,
      "logits/rejected": -2.461418628692627,
      "logps/chosen": -0.2507760226726532,
      "logps/rejected": -0.24406008422374725,
      "loss": 0.24763154983520508,
      "loss/core": 0.23287668824195862,
      "loss/preference_sft": 0.25262296199798584,
      "loss/reference_anchor": 0.17146925628185272,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.344456672668457,
      "rewards/margins": 1.498246431350708,
      "rewards/rejected": 0.8462101221084595,
      "ronpo/logit": 0.14982464909553528,
      "ronpo/residual": 0.13953104615211487,
      "ronpo/residual_abs": 0.16128280758857727,
      "ronpo/target": 0.01029361505061388,
      "ronpo/target_abs": 0.03700987249612808,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 65
    },
    {
      "drift/chosen_abs": 0.21904170513153076,
      "drift/rejected_abs": 0.11182647943496704,
      "epoch": 0.022248708780294002,
      "grad_norm": 3.40625,
      "learning_rate": 5.111111111111111e-08,
      "logits/chosen": -2.1311838626861572,
      "logits/rejected": -2.13192081451416,
      "logps/chosen": -0.26317623257637024,
      "logps/rejected": -0.26813703775405884,
      "loss": 0.07477524131536484,
      "loss/core": 0.0641421526670456,
      "loss/preference_sft": 0.2720448076725006,
      "loss/reference_anchor": 0.11457005888223648,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.1900362968444824,
      "rewards/margins": 1.0720874071121216,
      "rewards/rejected": 1.1179488897323608,
      "ronpo/logit": 0.1072087436914444,
      "ronpo/residual": 0.1019325703382492,
      "ronpo/residual_abs": 0.12027925252914429,
      "ronpo/target": 0.005276172887533903,
      "ronpo/target_abs": 0.03518466651439667,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 70
    },
    {
      "drift/chosen_abs": 0.2813442051410675,
      "drift/rejected_abs": 0.14759781956672668,
      "epoch": 0.023837902264600714,
      "grad_norm": 6.03125,
      "learning_rate": 5.481481481481482e-08,
      "logits/chosen": -2.1067919731140137,
      "logits/rejected": -2.0403101444244385,
      "logps/chosen": -0.27957314252853394,
      "logps/rejected": -0.2719704210758209,
      "loss": 0.6640143394470215,
      "loss/core": 0.6304754614830017,
      "loss/preference_sft": 0.27398350834846497,
      "loss/reference_anchor": 0.41978755593299866,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.81268310546875,
      "rewards/margins": 1.3621594905853271,
      "rewards/rejected": 1.4505234956741333,
      "ronpo/logit": 0.1362159550189972,
      "ronpo/residual": 0.13798193633556366,
      "ronpo/residual_abs": 0.26703405380249023,
      "ronpo/target": -0.001765978755429387,
      "ronpo/target_abs": 0.04140448570251465,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 75
    },
    {
      "drift/chosen_abs": 0.32669883966445923,
      "drift/rejected_abs": 0.1386711448431015,
      "epoch": 0.02542709574890743,
      "grad_norm": 40.75,
      "learning_rate": 5.851851851851851e-08,
      "logits/chosen": -2.052741765975952,
      "logits/rejected": -2.021430730819702,
      "logps/chosen": -0.27059227228164673,
      "logps/rejected": -0.2733466327190399,
      "loss": 0.387042760848999,
      "loss/core": 0.3550611138343811,
      "loss/preference_sft": 0.26973801851272583,
      "loss/reference_anchor": 0.39944788813591003,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.2669882774353027,
      "rewards/margins": 1.8841441869735718,
      "rewards/rejected": 1.3828439712524414,
      "ronpo/logit": 0.18841443955898285,
      "ronpo/residual": 0.1888277232646942,
      "ronpo/residual_abs": 0.21059706807136536,
      "ronpo/target": -0.00041329022496938705,
      "ronpo/target_abs": 0.04022369906306267,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 80
    },
    {
      "drift/chosen_abs": 0.2280385047197342,
      "drift/rejected_abs": 0.07840272039175034,
      "epoch": 0.027016289233214145,
      "grad_norm": 1012.0,
      "learning_rate": 6.222222222222221e-08,
      "logits/chosen": -2.366596221923828,
      "logits/rejected": -2.309652805328369,
      "logps/chosen": -0.2941826283931732,
      "logps/rejected": -0.266676664352417,
      "loss": 0.5611018538475037,
      "loss/core": 0.5350865721702576,
      "loss/preference_sft": 0.2718455195426941,
      "loss/reference_anchor": 0.3196846544742584,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2803852558135986,
      "rewards/margins": 1.5015474557876587,
      "rewards/rejected": 0.7788375020027161,
      "ronpo/logit": 0.1501547396183014,
      "ronpo/residual": 0.1435343623161316,
      "ronpo/residual_abs": 0.17277644574642181,
      "ronpo/target": 0.006620384752750397,
      "ronpo/target_abs": 0.039146699011325836,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 85
    },
    {
      "drift/chosen_abs": 0.18973015248775482,
      "drift/rejected_abs": 0.16024281084537506,
      "epoch": 0.028605482717520857,
      "grad_norm": 1.21875,
      "learning_rate": 6.592592592592592e-08,
      "logits/chosen": -2.26116943359375,
      "logits/rejected": -2.2707791328430176,
      "logps/chosen": -0.28311672806739807,
      "logps/rejected": -0.27832671999931335,
      "loss": 0.27082979679107666,
      "loss/core": 0.25301414728164673,
      "loss/preference_sft": 0.28250330686569214,
      "loss/reference_anchor": 0.20929114520549774,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8972270488739014,
      "rewards/margins": 0.29858314990997314,
      "rewards/rejected": 1.5986440181732178,
      "ronpo/logit": 0.029858315363526344,
      "ronpo/residual": 0.026583939790725708,
      "ronpo/residual_abs": 0.1573331356048584,
      "ronpo/target": 0.003274365561082959,
      "ronpo/target_abs": 0.041410621255636215,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 90
    },
    {
      "drift/chosen_abs": 0.22587072849273682,
      "drift/rejected_abs": 0.1010076031088829,
      "epoch": 0.030194676201827572,
      "grad_norm": 3.453125,
      "learning_rate": 6.962962962962963e-08,
      "logits/chosen": -2.3380606174468994,
      "logits/rejected": -2.2585723400115967,
      "logps/chosen": -0.27271103858947754,
      "logps/rejected": -0.2697039246559143,
      "loss": 0.37328386306762695,
      "loss/core": 0.3553915023803711,
      "loss/preference_sft": 0.27376872301101685,
      "loss/reference_anchor": 0.21118827164173126,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.258707046508789,
      "rewards/margins": 1.2546396255493164,
      "rewards/rejected": 1.0040676593780518,
      "ronpo/logit": 0.12546396255493164,
      "ronpo/residual": 0.13282093405723572,
      "ronpo/residual_abs": 0.19285807013511658,
      "ronpo/target": -0.0073569887317717075,
      "ronpo/target_abs": 0.043073222041130066,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 95
    },
    {
      "drift/chosen_abs": 0.2537795305252075,
      "drift/rejected_abs": 0.10038461536169052,
      "epoch": 0.03178386968613429,
      "grad_norm": 9.9375,
      "learning_rate": 7.333333333333333e-08,
      "logits/chosen": -1.9689257144927979,
      "logits/rejected": -2.040370464324951,
      "logps/chosen": -0.29754379391670227,
      "logps/rejected": -0.288893461227417,
      "loss": 0.2434326410293579,
      "loss/core": 0.22620365023612976,
      "loss/preference_sft": 0.28982117772102356,
      "loss/reference_anchor": 0.2007375955581665,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.535590887069702,
      "rewards/margins": 1.5332473516464233,
      "rewards/rejected": 1.0023434162139893,
      "ronpo/logit": 0.15332476794719696,
      "ronpo/residual": 0.1544944941997528,
      "ronpo/residual_abs": 0.17103642225265503,
      "ronpo/target": -0.0011697321897372603,
      "ronpo/target_abs": 0.03966054320335388,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 100
    },
    {
      "drift/chosen_abs": 0.3896755576133728,
      "drift/rejected_abs": 0.12742675840854645,
      "epoch": 0.033373063170441,
      "grad_norm": 94.0,
      "learning_rate": 7.703703703703704e-08,
      "logits/chosen": -1.9289357662200928,
      "logits/rejected": -1.9362056255340576,
      "logps/chosen": -0.2808288335800171,
      "logps/rejected": -0.2832654118537903,
      "loss": 0.7134211659431458,
      "loss/core": 0.6654593348503113,
      "loss/preference_sft": 0.2797420024871826,
      "loss/reference_anchor": 0.6115163564682007,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.8960719108581543,
      "rewards/margins": 2.6388587951660156,
      "rewards/rejected": 1.2572133541107178,
      "ronpo/logit": 0.26388588547706604,
      "ronpo/residual": 0.2673845887184143,
      "ronpo/residual_abs": 0.28307345509529114,
      "ronpo/target": -0.003498706966638565,
      "ronpo/target_abs": 0.041351594030857086,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 105
    },
    {
      "drift/chosen_abs": 0.24370336532592773,
      "drift/rejected_abs": 0.12277977168560028,
      "epoch": 0.03496225665474772,
      "grad_norm": 8.5,
      "learning_rate": 8.074074074074075e-08,
      "logits/chosen": -2.243190288543701,
      "logits/rejected": -2.2032723426818848,
      "logps/chosen": -0.294159471988678,
      "logps/rejected": -0.28763648867607117,
      "loss": 0.12273113429546356,
      "loss/core": 0.10542932897806168,
      "loss/preference_sft": 0.2917981445789337,
      "loss/reference_anchor": 0.201510950922966,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.434004068374634,
      "rewards/margins": 1.2225492000579834,
      "rewards/rejected": 1.2114547491073608,
      "ronpo/logit": 0.12225492298603058,
      "ronpo/residual": 0.1237180233001709,
      "ronpo/residual_abs": 0.15315257012844086,
      "ronpo/target": -0.0014631092781201005,
      "ronpo/target_abs": 0.04341963678598404,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 110
    },
    {
      "drift/chosen_abs": 0.2751477062702179,
      "drift/rejected_abs": 0.13712719082832336,
      "epoch": 0.03655145013905443,
      "grad_norm": 2.046875,
      "learning_rate": 8.444444444444444e-08,
      "logits/chosen": -1.8840402364730835,
      "logits/rejected": -1.7751556634902954,
      "logps/chosen": -0.2792755961418152,
      "logps/rejected": -0.28249382972717285,
      "loss": 0.15595600008964539,
      "loss/core": 0.1379595398902893,
      "loss/preference_sft": 0.2846413254737854,
      "loss/reference_anchor": 0.21148864924907684,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.7486186027526855,
      "rewards/margins": 1.3827894926071167,
      "rewards/rejected": 1.3658287525177002,
      "ronpo/logit": 0.13827894628047943,
      "ronpo/residual": 0.14020434021949768,
      "ronpo/residual_abs": 0.15965864062309265,
      "ronpo/target": -0.0019253784557804465,
      "ronpo/target_abs": 0.03954272344708443,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 115
    },
    {
      "drift/chosen_abs": 0.25264251232147217,
      "drift/rejected_abs": 0.11706644296646118,
      "epoch": 0.03814064362336114,
      "grad_norm": 1.2578125,
      "learning_rate": 8.814814814814815e-08,
      "logits/chosen": -2.016573905944824,
      "logits/rejected": -2.064659357070923,
      "logps/chosen": -0.3026151955127716,
      "logps/rejected": -0.30011817812919617,
      "loss": 0.3021138608455658,
      "loss/core": 0.2761771082878113,
      "loss/preference_sft": 0.3143635392189026,
      "loss/reference_anchor": 0.3143865764141083,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5251619815826416,
      "rewards/margins": 1.356215000152588,
      "rewards/rejected": 1.1689469814300537,
      "ronpo/logit": 0.13562151789665222,
      "ronpo/residual": 0.13547687232494354,
      "ronpo/residual_abs": 0.1778053343296051,
      "ronpo/target": 0.00014464984997175634,
      "ronpo/target_abs": 0.03622669726610184,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 120
    },
    {
      "drift/chosen_abs": 0.12724295258522034,
      "drift/rejected_abs": 0.07093048840761185,
      "epoch": 0.03972983710766786,
      "grad_norm": 1.109375,
      "learning_rate": 9.185185185185185e-08,
      "logits/chosen": -2.2724061012268066,
      "logits/rejected": -2.1935276985168457,
      "logps/chosen": -0.27425864338874817,
      "logps/rejected": -0.2812421917915344,
      "loss": 0.04385209083557129,
      "loss/core": 0.0365266427397728,
      "loss/preference_sft": 0.27121520042419434,
      "loss/reference_anchor": 0.07055105268955231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.2717318534851074,
      "rewards/margins": 0.5637460947036743,
      "rewards/rejected": 0.7079858779907227,
      "ronpo/logit": 0.056374602019786835,
      "ronpo/residual": 0.060826338827610016,
      "ronpo/residual_abs": 0.08193003386259079,
      "ronpo/target": -0.004451734013855457,
      "ronpo/target_abs": 0.03687246888875961,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 125
    },
    {
      "drift/chosen_abs": 0.36936086416244507,
      "drift/rejected_abs": 0.15099933743476868,
      "epoch": 0.04131903059197457,
      "grad_norm": 16.125,
      "learning_rate": 9.555555555555556e-08,
      "logits/chosen": -2.049341917037964,
      "logits/rejected": -1.9780795574188232,
      "logps/chosen": -0.28084278106689453,
      "logps/rejected": -0.302662193775177,
      "loss": 0.52703458070755,
      "loss/core": 0.4721059203147888,
      "loss/preference_sft": 0.30051907896995544,
      "loss/reference_anchor": 0.7023288011550903,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.693608522415161,
      "rewards/margins": 2.1844735145568848,
      "rewards/rejected": 1.5091348886489868,
      "ronpo/logit": 0.21844735741615295,
      "ronpo/residual": 0.2173984795808792,
      "ronpo/residual_abs": 0.24278369545936584,
      "ronpo/target": 0.0010489022824913263,
      "ronpo/target_abs": 0.03291699290275574,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 130
    },
    {
      "drift/chosen_abs": 0.22806978225708008,
      "drift/rejected_abs": 0.17581363022327423,
      "epoch": 0.04290822407628129,
      "grad_norm": 93.5,
      "learning_rate": 9.925925925925925e-08,
      "logits/chosen": -2.155839681625366,
      "logits/rejected": -2.1424169540405273,
      "logps/chosen": -0.25515803694725037,
      "logps/rejected": -0.2733699679374695,
      "loss": 0.38603073358535767,
      "loss/core": 0.3621215224266052,
      "loss/preference_sft": 0.26424866914749146,
      "loss/reference_anchor": 0.2923644185066223,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2795376777648926,
      "rewards/margins": 0.5223998427391052,
      "rewards/rejected": 1.7571378946304321,
      "ronpo/logit": 0.052239976823329926,
      "ronpo/residual": 0.05240260437130928,
      "ronpo/residual_abs": 0.18414601683616638,
      "ronpo/target": -0.0001626206503715366,
      "ronpo/target_abs": 0.03484975919127464,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 135
    },
    {
      "drift/chosen_abs": 0.21248528361320496,
      "drift/rejected_abs": 0.1548730880022049,
      "epoch": 0.044497417560588004,
      "grad_norm": 37.25,
      "learning_rate": 9.999325429879214e-08,
      "logits/chosen": -2.066507339477539,
      "logits/rejected": -2.0359199047088623,
      "logps/chosen": -0.2865599989891052,
      "logps/rejected": -0.28677845001220703,
      "loss": 0.2560182213783264,
      "loss/core": 0.237252876162529,
      "loss/preference_sft": 0.2824710011482239,
      "loss/reference_anchor": 0.2219577133655548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1243197917938232,
      "rewards/margins": 0.5769208669662476,
      "rewards/rejected": 1.5473989248275757,
      "ronpo/logit": 0.05769209936261177,
      "ronpo/residual": 0.05259509012103081,
      "ronpo/residual_abs": 0.1705760955810547,
      "ronpo/target": 0.005097000394016504,
      "ronpo/target_abs": 0.0383358970284462,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 140
    },
    {
      "drift/chosen_abs": 0.31215378642082214,
      "drift/rejected_abs": 0.13634580373764038,
      "epoch": 0.04608661104489471,
      "grad_norm": 233.0,
      "learning_rate": 9.996585300715115e-08,
      "logits/chosen": -2.1033005714416504,
      "logits/rejected": -2.1405441761016846,
      "logps/chosen": -0.2735254764556885,
      "logps/rejected": -0.26641449332237244,
      "loss": 0.22382652759552002,
      "loss/core": 0.1932472586631775,
      "loss/preference_sft": 0.2740625739097595,
      "loss/reference_anchor": 0.38031694293022156,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.120529890060425,
      "rewards/margins": 1.759221076965332,
      "rewards/rejected": 1.3613090515136719,
      "ronpo/logit": 0.17592211067676544,
      "ronpo/residual": 0.18040737509727478,
      "ronpo/residual_abs": 0.19332192838191986,
      "ronpo/target": -0.004485286772251129,
      "ronpo/target_abs": 0.038091517984867096,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 145
    },
    {
      "drift/chosen_abs": 0.40075913071632385,
      "drift/rejected_abs": 0.11532590538263321,
      "epoch": 0.04767580452920143,
      "grad_norm": 2.671875,
      "learning_rate": 9.991738606227535e-08,
      "logits/chosen": -2.0292041301727295,
      "logits/rejected": -2.1317694187164307,
      "logps/chosen": -0.2969673275947571,
      "logps/rejected": -0.2732444405555725,
      "loss": 1.0155136585235596,
      "loss/core": 0.962049126625061,
      "loss/preference_sft": 0.29812875390052795,
      "loss/reference_anchor": 0.6830486059188843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 4.007282257080078,
      "rewards/margins": 2.855675220489502,
      "rewards/rejected": 1.1516071557998657,
      "ronpo/logit": 0.2855675220489502,
      "ronpo/residual": 0.2750109136104584,
      "ronpo/residual_abs": 0.29854530096054077,
      "ronpo/target": 0.010556596331298351,
      "ronpo/target_abs": 0.040050022304058075,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 150
    },
    {
      "drift/chosen_abs": 0.31126388907432556,
      "drift/rejected_abs": 0.10443758964538574,
      "epoch": 0.04926499801350814,
      "grad_norm": 4.46875,
      "learning_rate": 9.984787389787688e-08,
      "logits/chosen": -2.1672780513763428,
      "logits/rejected": -2.172288417816162,
      "logps/chosen": -0.26607850193977356,
      "logps/rejected": -0.26191627979278564,
      "loss": 0.6859047412872314,
      "loss/core": 0.644756555557251,
      "loss/preference_sft": 0.2656945586204529,
      "loss/reference_anchor": 0.5220734477043152,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.1122288703918457,
      "rewards/margins": 2.06986665725708,
      "rewards/rejected": 1.0423619747161865,
      "ronpo/logit": 0.206986665725708,
      "ronpo/residual": 0.20458738505840302,
      "ronpo/residual_abs": 0.2233704775571823,
      "ronpo/target": 0.002399270888417959,
      "ronpo/target_abs": 0.03868597000837326,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 155
    },
    {
      "drift/chosen_abs": 0.19957342743873596,
      "drift/rejected_abs": 0.06797181069850922,
      "epoch": 0.05085419149781486,
      "grad_norm": 836.0,
      "learning_rate": 9.975734582035323e-08,
      "logits/chosen": -2.2061939239501953,
      "logits/rejected": -2.2242698669433594,
      "logps/chosen": -0.32043007016181946,
      "logps/rejected": -0.273697167634964,
      "loss": 0.5350174307823181,
      "loss/core": 0.5092377066612244,
      "loss/preference_sft": 0.3236948847770691,
      "loss/reference_anchor": 0.31135934591293335,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.9948062896728516,
      "rewards/margins": 1.3165847063064575,
      "rewards/rejected": 0.6782217025756836,
      "ronpo/logit": 0.13165846467018127,
      "ronpo/residual": 0.12871192395687103,
      "ronpo/residual_abs": 0.15134011209011078,
      "ronpo/target": 0.00294652977026999,
      "ronpo/target_abs": 0.0420367605984211,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 160
    },
    {
      "drift/chosen_abs": 0.13104529678821564,
      "drift/rejected_abs": 0.051272135227918625,
      "epoch": 0.052443384982121574,
      "grad_norm": 3.875,
      "learning_rate": 9.964583999643173e-08,
      "logits/chosen": -2.1314780712127686,
      "logits/rejected": -2.0094780921936035,
      "logps/chosen": -0.2844526171684265,
      "logps/rejected": -0.2935580015182495,
      "loss": 0.10935626924037933,
      "loss/core": 0.1020495668053627,
      "loss/preference_sft": 0.2882479131221771,
      "loss/reference_anchor": 0.06859799474477768,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.3102877140045166,
      "rewards/margins": 0.8030896186828613,
      "rewards/rejected": 0.5071979761123657,
      "ronpo/logit": 0.08030896633863449,
      "ronpo/residual": 0.0773935616016388,
      "ronpo/residual_abs": 0.10444708168506622,
      "ronpo/target": 0.002915406133979559,
      "ronpo/target_abs": 0.03609101101756096,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 165
    },
    {
      "drift/chosen_abs": 0.31687039136886597,
      "drift/rejected_abs": 0.16696619987487793,
      "epoch": 0.05403257846642829,
      "grad_norm": 1.25,
      "learning_rate": 9.951340343707851e-08,
      "logits/chosen": -2.027431011199951,
      "logits/rejected": -2.020197868347168,
      "logps/chosen": -0.28902456164360046,
      "logps/rejected": -0.2761145830154419,
      "loss": 0.4374803602695465,
      "loss/core": 0.4037446975708008,
      "loss/preference_sft": 0.28865548968315125,
      "loss/reference_anchor": 0.42094317078590393,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.1682214736938477,
      "rewards/margins": 1.5027532577514648,
      "rewards/rejected": 1.6654682159423828,
      "ronpo/logit": 0.150275319814682,
      "ronpo/residual": 0.14720943570137024,
      "ronpo/residual_abs": 0.23024269938468933,
      "ronpo/target": 0.003065910190343857,
      "ronpo/target_abs": 0.039906926453113556,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 170
    },
    {
      "drift/chosen_abs": 0.19880618155002594,
      "drift/rejected_abs": 0.11131361871957779,
      "epoch": 0.055621771950735005,
      "grad_norm": 30.875,
      "learning_rate": 9.936009197767845e-08,
      "logits/chosen": -2.263164520263672,
      "logits/rejected": -2.1978354454040527,
      "logps/chosen": -0.2936115860939026,
      "logps/rejected": -0.30503323674201965,
      "loss": 0.06118471547961235,
      "loss/core": 0.0469072088599205,
      "loss/preference_sft": 0.2960822582244873,
      "loss/reference_anchor": 0.16075855493545532,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9874756336212158,
      "rewards/margins": 0.8753183484077454,
      "rewards/rejected": 1.1121572256088257,
      "ronpo/logit": 0.08753182739019394,
      "ronpo/residual": 0.09231725335121155,
      "ronpo/residual_abs": 0.11050446331501007,
      "ronpo/target": -0.0047854213044047356,
      "ronpo/target_abs": 0.040785495191812515,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 175
    },
    {
      "drift/chosen_abs": 0.47293806076049805,
      "drift/rejected_abs": 0.2267259806394577,
      "epoch": 0.057210965435041714,
      "grad_norm": 39.75,
      "learning_rate": 9.918597025449504e-08,
      "logits/chosen": -2.073373317718506,
      "logits/rejected": -2.0509047508239746,
      "logps/chosen": -0.2899821400642395,
      "logps/rejected": -0.281298965215683,
      "loss": 0.952112078666687,
      "loss/core": 0.8648629188537598,
      "loss/preference_sft": 0.2834712862968445,
      "loss/reference_anchor": 1.1349718570709229,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 4.7293806076049805,
      "rewards/margins": 2.462536334991455,
      "rewards/rejected": 2.2668442726135254,
      "ronpo/logit": 0.24625368416309357,
      "ronpo/residual": 0.2527698874473572,
      "ronpo/residual_abs": 0.27635571360588074,
      "ronpo/target": -0.006516219116747379,
      "ronpo/target_abs": 0.03632596880197525,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 180
    },
    {
      "drift/chosen_abs": 0.12262760102748871,
      "drift/rejected_abs": 0.06452114135026932,
      "epoch": 0.05880015891934843,
      "grad_norm": 4.6875,
      "learning_rate": 9.899111167741965e-08,
      "logits/chosen": -2.2256319522857666,
      "logits/rejected": -2.3286309242248535,
      "logps/chosen": -0.29475218057632446,
      "logps/rejected": -0.29190289974212646,
      "loss": 0.022488895803689957,
      "loss/core": 0.017534026876091957,
      "loss/preference_sft": 0.29180967807769775,
      "loss/reference_anchor": 0.03688395395874977,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.2256981134414673,
      "rewards/margins": 0.583113968372345,
      "rewards/rejected": 0.6425840854644775,
      "ronpo/logit": 0.05831141024827957,
      "ronpo/residual": 0.06310184299945831,
      "ronpo/residual_abs": 0.08577104657888412,
      "ronpo/target": -0.004790434148162603,
      "ronpo/target_abs": 0.04436204954981804,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 185
    },
    {
      "drift/chosen_abs": 0.12513764202594757,
      "drift/rejected_abs": 0.0551786944270134,
      "epoch": 0.060389352403655144,
      "grad_norm": 0.8359375,
      "learning_rate": 9.877559839902184e-08,
      "logits/chosen": -2.314668655395508,
      "logits/rejected": -2.410252332687378,
      "logps/chosen": -0.305042028427124,
      "logps/rejected": -0.30319944024086,
      "loss": 0.0467093251645565,
      "loss/core": 0.039730604737997055,
      "loss/preference_sft": 0.30456751585006714,
      "loss/reference_anchor": 0.06259283423423767,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.249873399734497,
      "rewards/margins": 0.6990891098976135,
      "rewards/rejected": 0.5507842302322388,
      "ronpo/logit": 0.06990890949964523,
      "ronpo/residual": 0.06662610918283463,
      "ronpo/residual_abs": 0.0962677150964737,
      "ronpo/target": 0.003282807767391205,
      "ronpo/target_abs": 0.04486598074436188,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 190
    },
    {
      "drift/chosen_abs": 0.21243241429328918,
      "drift/rejected_abs": 0.11867997795343399,
      "epoch": 0.06197854588796186,
      "grad_norm": 6.1875,
      "learning_rate": 9.853952127991371e-08,
      "logits/chosen": -2.0548830032348633,
      "logits/rejected": -2.0303750038146973,
      "logps/chosen": -0.27820366621017456,
      "logps/rejected": -0.27188804745674133,
      "loss": 0.056257784366607666,
      "loss/core": 0.04541615769267082,
      "loss/preference_sft": 0.2762068510055542,
      "loss/reference_anchor": 0.11693432182073593,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.123030185699463,
      "rewards/margins": 0.9386847615242004,
      "rewards/rejected": 1.1843453645706177,
      "ronpo/logit": 0.09386847168207169,
      "ronpo/residual": 0.09398861974477768,
      "ronpo/residual_abs": 0.11155357211828232,
      "ronpo/target": -0.00012014620006084442,
      "ronpo/target_abs": 0.03941798955202103,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 195
    },
    {
      "drift/chosen_abs": 0.2837020754814148,
      "drift/rejected_abs": 0.0895429477095604,
      "epoch": 0.06356773937226858,
      "grad_norm": 9.5625,
      "learning_rate": 9.828297985044313e-08,
      "logits/chosen": -2.0397047996520996,
      "logits/rejected": -2.1584408283233643,
      "logps/chosen": -0.2889028787612915,
      "logps/rejected": -0.2791716456413269,
      "loss": 0.4928821623325348,
      "loss/core": 0.4666885435581207,
      "loss/preference_sft": 0.28681278228759766,
      "loss/reference_anchor": 0.3205677270889282,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.836252212524414,
      "rewards/margins": 1.9445428848266602,
      "rewards/rejected": 0.8917096257209778,
      "ronpo/logit": 0.19445426762104034,
      "ronpo/residual": 0.19480250775814056,
      "ronpo/residual_abs": 0.21554896235466003,
      "ronpo/target": -0.000348227855283767,
      "ronpo/target_abs": 0.04315657541155815,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 200
    },
    {
      "drift/chosen_abs": 0.2943119406700134,
      "drift/rejected_abs": 0.10832907259464264,
      "epoch": 0.06515693285657528,
      "grad_norm": 101.0,
      "learning_rate": 9.800608226873141e-08,
      "logits/chosen": -2.000119686126709,
      "logits/rejected": -2.0814411640167236,
      "logps/chosen": -0.28162431716918945,
      "logps/rejected": -0.28688597679138184,
      "loss": 0.7683490514755249,
      "loss/core": 0.725469708442688,
      "loss/preference_sft": 0.2873081564903259,
      "loss/reference_anchor": 0.5429925322532654,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9406843185424805,
      "rewards/margins": 1.858192801475525,
      "rewards/rejected": 1.0824917554855347,
      "ronpo/logit": 0.18581928312778473,
      "ronpo/residual": 0.20173266530036926,
      "ronpo/residual_abs": 0.210129052400589,
      "ronpo/target": -0.015913408249616623,
      "ronpo/target_abs": 0.03376617655158043,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 205
    },
    {
      "drift/chosen_abs": 0.21819961071014404,
      "drift/rejected_abs": 0.05788310617208481,
      "epoch": 0.066746126340882,
      "grad_norm": 3.0625,
      "learning_rate": 9.770894527507391e-08,
      "logits/chosen": -2.2487425804138184,
      "logits/rejected": -2.167056083679199,
      "logps/chosen": -0.3088724613189697,
      "logps/rejected": -0.315146267414093,
      "loss": 1.0223528146743774,
      "loss/core": 0.9793564081192017,
      "loss/preference_sft": 0.31073203682899475,
      "loss/reference_anchor": 0.5422115325927734,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.181572675704956,
      "rewards/margins": 1.6049134731292725,
      "rewards/rejected": 0.5766594409942627,
      "ronpo/logit": 0.16049133241176605,
      "ronpo/residual": 0.16226737201213837,
      "ronpo/residual_abs": 0.1834581196308136,
      "ronpo/target": -0.001776007004082203,
      "ronpo/target_abs": 0.03983345627784729,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 210
    },
    {
      "drift/chosen_abs": 0.3269013464450836,
      "drift/rejected_abs": 0.164812833070755,
      "epoch": 0.06833531982518871,
      "grad_norm": 1.09375,
      "learning_rate": 9.739169414272217e-08,
      "logits/chosen": -2.167433500289917,
      "logits/rejected": -2.1554653644561768,
      "logps/chosen": -0.26557686924934387,
      "logps/rejected": -0.27201777696609497,
      "loss": 0.30894580483436584,
      "loss/core": 0.2645125985145569,
      "loss/preference_sft": 0.26722875237464905,
      "loss/reference_anchor": 0.5657202005386353,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 3.2687904834747314,
      "rewards/margins": 1.621460199356079,
      "rewards/rejected": 1.6473299264907837,
      "ronpo/logit": 0.16214601695537567,
      "ronpo/residual": 0.16244594752788544,
      "ronpo/residual_abs": 0.18437254428863525,
      "ronpo/target": -0.0002999236749019474,
      "ronpo/target_abs": 0.03696250170469284,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 215
    },
    {
      "drift/chosen_abs": 0.1197463721036911,
      "drift/rejected_abs": 0.08149294555187225,
      "epoch": 0.06992451330949544,
      "grad_norm": 0.423828125,
      "learning_rate": 9.705446262506858e-08,
      "logits/chosen": -2.2162628173828125,
      "logits/rejected": -2.0838351249694824,
      "logps/chosen": -0.30454206466674805,
      "logps/rejected": -0.2994697093963623,
      "loss": 0.053737662732601166,
      "loss/core": 0.04793507978320122,
      "loss/preference_sft": 0.3021828234195709,
      "loss/reference_anchor": 0.04714947193861008,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1946014165878296,
      "rewards/margins": 0.3831804096698761,
      "rewards/rejected": 0.8114210367202759,
      "ronpo/logit": 0.03831804171204567,
      "ronpo/residual": 0.04171460121870041,
      "ronpo/residual_abs": 0.09978470951318741,
      "ronpo/target": -0.0033965595066547394,
      "ronpo/target_abs": 0.03821447864174843,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 220
    },
    {
      "drift/chosen_abs": 0.23339900374412537,
      "drift/rejected_abs": 0.10263761132955551,
      "epoch": 0.07151370679380215,
      "grad_norm": 5.84375,
      "learning_rate": 9.669739289925576e-08,
      "logits/chosen": -2.178617000579834,
      "logits/rejected": -2.1953907012939453,
      "logps/chosen": -0.28538942337036133,
      "logps/rejected": -0.28581511974334717,
      "loss": 0.20499324798583984,
      "loss/core": 0.18968087434768677,
      "loss/preference_sft": 0.2798437476158142,
      "loss/reference_anchor": 0.17618073523044586,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.3339900970458984,
      "rewards/margins": 1.3076139688491821,
      "rewards/rejected": 1.0263762474060059,
      "ronpo/logit": 0.13076139986515045,
      "ronpo/residual": 0.12642139196395874,
      "ronpo/residual_abs": 0.14748407900333405,
      "ronpo/target": 0.004340013489127159,
      "ronpo/target_abs": 0.03319591283798218,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 225
    },
    {
      "drift/chosen_abs": 0.2666410207748413,
      "drift/rejected_abs": 0.09672902524471283,
      "epoch": 0.07310290027810885,
      "grad_norm": 2.625,
      "learning_rate": 9.632063550623463e-08,
      "logits/chosen": -2.241774082183838,
      "logits/rejected": -2.2601799964904785,
      "logps/chosen": -0.26070520281791687,
      "logps/rejected": -0.2603910565376282,
      "loss": 0.405656635761261,
      "loss/core": 0.3820309042930603,
      "loss/preference_sft": 0.2593463361263275,
      "loss/reference_anchor": 0.2890748381614685,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.666410207748413,
      "rewards/margins": 1.7003774642944336,
      "rewards/rejected": 0.9660326838493347,
      "ronpo/logit": 0.17003774642944336,
      "ronpo/residual": 0.17019498348236084,
      "ronpo/residual_abs": 0.18751393258571625,
      "ronpo/target": -0.00015722606622148305,
      "ronpo/target_abs": 0.038429904729127884,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 230
    },
    {
      "drift/chosen_abs": 0.45942026376724243,
      "drift/rejected_abs": 0.23705634474754333,
      "epoch": 0.07469209376241558,
      "grad_norm": 70.0,
      "learning_rate": 9.592434928729615e-08,
      "logits/chosen": -2.114424467086792,
      "logits/rejected": -2.1161932945251465,
      "logps/chosen": -0.28261953592300415,
      "logps/rejected": -0.28544551134109497,
      "loss": 1.219696283340454,
      "loss/core": 1.1286277770996094,
      "loss/preference_sft": 0.28843197226524353,
      "loss/reference_anchor": 1.1854050159454346,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 4.592301368713379,
      "rewards/margins": 2.304565191268921,
      "rewards/rejected": 2.2877368927001953,
      "ronpo/logit": 0.23045650124549866,
      "ronpo/residual": 0.23253145813941956,
      "ronpo/residual_abs": 0.3629453778266907,
      "ronpo/target": -0.0020749385003000498,
      "ronpo/target_abs": 0.032184846699237823,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 235
    },
    {
      "drift/chosen_abs": 0.29112711548805237,
      "drift/rejected_abs": 0.16287891566753387,
      "epoch": 0.07628128724672228,
      "grad_norm": 192.0,
      "learning_rate": 9.550870131710364e-08,
      "logits/chosen": -2.141789674758911,
      "logits/rejected": -2.2175676822662354,
      "logps/chosen": -0.2665143609046936,
      "logps/rejected": -0.2892531454563141,
      "loss": 0.23649156093597412,
      "loss/core": 0.19637542963027954,
      "loss/preference_sft": 0.28473249077796936,
      "loss/reference_anchor": 0.5064085721969604,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9112708568573,
      "rewards/margins": 1.2845176458358765,
      "rewards/rejected": 1.6267534494400024,
      "ronpo/logit": 0.12845174968242645,
      "ronpo/residual": 0.118413046002388,
      "ronpo/residual_abs": 0.14850665628910065,
      "ronpo/target": 0.010038711130619049,
      "ronpo/target_abs": 0.03962286561727524,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 240
    },
    {
      "drift/chosen_abs": 0.18798431754112244,
      "drift/rejected_abs": 0.07243553549051285,
      "epoch": 0.07787048073102901,
      "grad_norm": 1.609375,
      "learning_rate": 9.507386683325402e-08,
      "logits/chosen": -2.1808266639709473,
      "logits/rejected": -2.199118137359619,
      "logps/chosen": -0.2991955280303955,
      "logps/rejected": -0.30441370606422424,
      "loss": 0.27557721734046936,
      "loss/core": 0.26069125533103943,
      "loss/preference_sft": 0.30375024676322937,
      "loss/reference_anchor": 0.16810381412506104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.878678560256958,
      "rewards/margins": 1.1664841175079346,
      "rewards/rejected": 0.712194561958313,
      "ronpo/logit": 0.11664841324090958,
      "ronpo/residual": 0.1208236962556839,
      "ronpo/residual_abs": 0.14474524557590485,
      "ronpo/target": -0.004175290931016207,
      "ronpo/target_abs": 0.04190593212842941,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 245
    },
    {
      "drift/chosen_abs": 0.362449586391449,
      "drift/rejected_abs": 0.08568783104419708,
      "epoch": 0.07945967421533572,
      "grad_norm": 7.59375,
      "learning_rate": 9.462002916239739e-08,
      "logits/chosen": -2.11047625541687,
      "logits/rejected": -2.0861949920654297,
      "logps/chosen": -0.3092321753501892,
      "logps/rejected": -0.30976372957229614,
      "loss": 1.2317886352539062,
      "loss/core": 1.1781948804855347,
      "loss/preference_sft": 0.3098042607307434,
      "loss/reference_anchor": 0.6836021542549133,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.62377667427063,
      "rewards/margins": 2.7679665088653564,
      "rewards/rejected": 0.8558100461959839,
      "ronpo/logit": 0.2767966389656067,
      "ronpo/residual": 0.2732112407684326,
      "ronpo/residual_abs": 0.2978476881980896,
      "ronpo/target": 0.003585411235690117,
      "ronpo/target_abs": 0.035868819802999496,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 250
    },
    {
      "drift/chosen_abs": 0.44660884141921997,
      "drift/rejected_abs": 0.20091009140014648,
      "epoch": 0.08104886769964244,
      "grad_norm": 10.125,
      "learning_rate": 9.414737964294634e-08,
      "logits/chosen": -2.101787567138672,
      "logits/rejected": -2.1054489612579346,
      "logps/chosen": -0.26226210594177246,
      "logps/rejected": -0.27092769742012024,
      "loss": 0.7605728507041931,
      "loss/core": 0.6867702603340149,
      "loss/preference_sft": 0.2730024456977844,
      "loss/reference_anchor": 0.9567340612411499,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 4.464694976806641,
      "rewards/margins": 2.549316883087158,
      "rewards/rejected": 1.9153783321380615,
      "ronpo/logit": 0.25493165850639343,
      "ronpo/residual": 0.2584943175315857,
      "ronpo/residual_abs": 0.2778705954551697,
      "ronpo/target": -0.003562638070434332,
      "ronpo/target_abs": 0.04188482090830803,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 255
    },
    {
      "drift/chosen_abs": 0.3281967043876648,
      "drift/rejected_abs": 0.0860821008682251,
      "epoch": 0.08263806118394915,
      "grad_norm": 960.0,
      "learning_rate": 9.365611754440755e-08,
      "logits/chosen": -2.194645881652832,
      "logits/rejected": -2.1680915355682373,
      "logps/chosen": -0.28327953815460205,
      "logps/rejected": -0.2808752655982971,
      "loss": 0.8503106236457825,
      "loss/core": 0.8106344938278198,
      "loss/preference_sft": 0.29711759090423584,
      "loss/reference_anchor": 0.4993039667606354,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.2812724113464355,
      "rewards/margins": 2.437526226043701,
      "rewards/rejected": 0.8437461853027344,
      "ronpo/logit": 0.2437526285648346,
      "ronpo/residual": 0.24408352375030518,
      "ronpo/residual_abs": 0.26183831691741943,
      "ronpo/target": -0.0003309231251478195,
      "ronpo/target_abs": 0.038510847836732864,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 260
    },
    {
      "drift/chosen_abs": 0.1818397492170334,
      "drift/rejected_abs": 0.08128567785024643,
      "epoch": 0.08422725466825585,
      "grad_norm": 16.125,
      "learning_rate": 9.314644998336948e-08,
      "logits/chosen": -2.153672695159912,
      "logits/rejected": -2.1532981395721436,
      "logps/chosen": -0.28503522276878357,
      "logps/rejected": -0.2909298837184906,
      "loss": 0.07522353529930115,
      "loss/core": 0.06629156321287155,
      "loss/preference_sft": 0.2900465726852417,
      "loss/reference_anchor": 0.09008826315402985,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8183977603912354,
      "rewards/margins": 1.012382984161377,
      "rewards/rejected": 0.8060146570205688,
      "ronpo/logit": 0.10123831033706665,
      "ronpo/residual": 0.0954081267118454,
      "ronpo/residual_abs": 0.12208437919616699,
      "ronpo/target": 0.005830181296914816,
      "ronpo/target_abs": 0.03874696046113968,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 265
    },
    {
      "drift/chosen_abs": 0.2921431064605713,
      "drift/rejected_abs": 0.16023942828178406,
      "epoch": 0.08581644815256258,
      "grad_norm": 5.90625,
      "learning_rate": 9.261859183618188e-08,
      "logits/chosen": -2.220456123352051,
      "logits/rejected": -2.1795496940612793,
      "logps/chosen": -0.2800024747848511,
      "logps/rejected": -0.2899700403213501,
      "loss": 0.36583656072616577,
      "loss/core": 0.3111600875854492,
      "loss/preference_sft": 0.2803427577018738,
      "loss/reference_anchor": 0.700985312461853,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.9204893112182617,
      "rewards/margins": 1.3201406002044678,
      "rewards/rejected": 1.6003488302230835,
      "ronpo/logit": 0.13201405107975006,
      "ronpo/residual": 0.12298605591058731,
      "ronpo/residual_abs": 0.15286676585674286,
      "ronpo/target": 0.009028008207678795,
      "ronpo/target_abs": 0.044766865670681,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 270
    },
    {
      "drift/chosen_abs": 0.17901456356048584,
      "drift/rejected_abs": 0.10373431444168091,
      "epoch": 0.08740564163686929,
      "grad_norm": 7.75,
      "learning_rate": 9.207276564836366e-08,
      "logits/chosen": -2.273167133331299,
      "logits/rejected": -2.3467493057250977,
      "logps/chosen": -0.2789549231529236,
      "logps/rejected": -0.26800718903541565,
      "loss": 0.04449104890227318,
      "loss/core": 0.034896623343229294,
      "loss/preference_sft": 0.27731361985206604,
      "loss/reference_anchor": 0.10019437968730927,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.7901455163955688,
      "rewards/margins": 0.7541426420211792,
      "rewards/rejected": 1.0360027551651,
      "ronpo/logit": 0.0754142552614212,
      "ronpo/residual": 0.06837241351604462,
      "ronpo/residual_abs": 0.09739400446414948,
      "ronpo/target": 0.007041858043521643,
      "ronpo/target_abs": 0.038143135607242584,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 275
    },
    {
      "drift/chosen_abs": 0.16873064637184143,
      "drift/rejected_abs": 0.11091496795415878,
      "epoch": 0.08899483512117601,
      "grad_norm": 48.0,
      "learning_rate": 9.150920154077752e-08,
      "logits/chosen": -2.183501720428467,
      "logits/rejected": -2.1855404376983643,
      "logps/chosen": -0.2852725386619568,
      "logps/rejected": -0.2904953360557556,
      "loss": 0.09437291324138641,
      "loss/core": 0.08497822284698486,
      "loss/preference_sft": 0.2888687252998352,
      "loss/reference_anchor": 0.09637565910816193,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.686948537826538,
      "rewards/margins": 0.5791365504264832,
      "rewards/rejected": 1.1078121662139893,
      "ronpo/logit": 0.0579136498272419,
      "ronpo/residual": 0.05210072919726372,
      "ronpo/residual_abs": 0.12251182645559311,
      "ronpo/target": 0.0058129276148974895,
      "ronpo/target_abs": 0.0337434858083725,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 280
    },
    {
      "drift/chosen_abs": 0.4374244809150696,
      "drift/rejected_abs": 0.13985301554203033,
      "epoch": 0.09058402860548272,
      "grad_norm": 16.75,
      "learning_rate": 9.092813711261074e-08,
      "logits/chosen": -2.2230849266052246,
      "logits/rejected": -2.194876194000244,
      "logps/chosen": -0.2806859612464905,
      "logps/rejected": -0.29192352294921875,
      "loss": 1.5361340045928955,
      "loss/core": 1.4417104721069336,
      "loss/preference_sft": 0.2899562418460846,
      "loss/reference_anchor": 1.229984998703003,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.373291015625,
      "rewards/margins": 2.9773240089416504,
      "rewards/rejected": 1.3959667682647705,
      "ronpo/logit": 0.2977324426174164,
      "ronpo/residual": 0.30149295926094055,
      "ronpo/residual_abs": 0.32409870624542236,
      "ronpo/target": -0.0037605254910886288,
      "ronpo/target_abs": 0.03826469928026199,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 285
    },
    {
      "drift/chosen_abs": 0.2992640733718872,
      "drift/rejected_abs": 0.14071711897850037,
      "epoch": 0.09217322208978942,
      "grad_norm": 101.0,
      "learning_rate": 9.032981734120312e-08,
      "logits/chosen": -2.015190839767456,
      "logits/rejected": -2.089470624923706,
      "logps/chosen": -0.2724992632865906,
      "logps/rejected": -0.2815447449684143,
      "loss": 0.2769699692726135,
      "loss/core": 0.2548522353172302,
      "loss/preference_sft": 0.2765698730945587,
      "loss/reference_anchor": 0.26724615693092346,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.989751100540161,
      "rewards/margins": 1.5834410190582275,
      "rewards/rejected": 1.4063100814819336,
      "ronpo/logit": 0.1583441197872162,
      "ronpo/residual": 0.15625953674316406,
      "ronpo/residual_abs": 0.2027464210987091,
      "ronpo/target": 0.0020845707040280104,
      "ronpo/target_abs": 0.042959146201610565,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 290
    },
    {
      "drift/chosen_abs": 0.19468748569488525,
      "drift/rejected_abs": 0.07145005464553833,
      "epoch": 0.09376241557409615,
      "grad_norm": 193.0,
      "learning_rate": 8.97144944787643e-08,
      "logits/chosen": -2.1931087970733643,
      "logits/rejected": -2.186386823654175,
      "logps/chosen": -0.3015713095664978,
      "logps/rejected": -0.30587688088417053,
      "loss": 0.24030208587646484,
      "loss/core": 0.224404975771904,
      "loss/preference_sft": 0.29630112648010254,
      "loss/reference_anchor": 0.18233135342597961,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9454704523086548,
      "rewards/margins": 1.2360841035842896,
      "rewards/rejected": 0.70938640832901,
      "ronpo/logit": 0.12360841035842896,
      "ronpo/residual": 0.12222124636173248,
      "ronpo/residual_abs": 0.14251907169818878,
      "ronpo/target": 0.0013871697010472417,
      "ronpo/target_abs": 0.03497045487165451,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 295
    },
    {
      "drift/chosen_abs": 0.136820450425148,
      "drift/rejected_abs": 0.08163662254810333,
      "epoch": 0.09535160905840286,
      "grad_norm": 6.125,
      "learning_rate": 8.9082427946024e-08,
      "logits/chosen": -2.403992176055908,
      "logits/rejected": -2.408097743988037,
      "logps/chosen": -0.2954743504524231,
      "logps/rejected": -0.2864408791065216,
      "loss": 0.027770619839429855,
      "loss/core": 0.021785762161016464,
      "loss/preference_sft": 0.2909078299999237,
      "loss/reference_anchor": 0.05070732906460762,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.3670862913131714,
      "rewards/margins": 0.553402841091156,
      "rewards/rejected": 0.8136835098266602,
      "ronpo/logit": 0.05534028261899948,
      "ronpo/residual": 0.05457191914319992,
      "ronpo/residual_abs": 0.07968275249004364,
      "ronpo/target": 0.0007683668518438935,
      "ronpo/target_abs": 0.03892510011792183,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 300
    },
    {
      "drift/chosen_abs": 0.22264614701271057,
      "drift/rejected_abs": 0.10277211666107178,
      "epoch": 0.09694080254270958,
      "grad_norm": 2.34375,
      "learning_rate": 8.843388422285994e-08,
      "logits/chosen": -2.075179100036621,
      "logits/rejected": -2.102503538131714,
      "logps/chosen": -0.30451059341430664,
      "logps/rejected": -0.2927056550979614,
      "loss": 0.1892203986644745,
      "loss/core": 0.16778506338596344,
      "loss/preference_sft": 0.3040909171104431,
      "loss/reference_anchor": 0.2553955018520355,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2247345447540283,
      "rewards/margins": 1.1996135711669922,
      "rewards/rejected": 1.0251209735870361,
      "ronpo/logit": 0.11996134370565414,
      "ronpo/residual": 0.1212671548128128,
      "ronpo/residual_abs": 0.14174291491508484,
      "ronpo/target": -0.0013058021431788802,
      "ronpo/target_abs": 0.038282014429569244,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 305
    },
    {
      "drift/chosen_abs": 0.23042087256908417,
      "drift/rejected_abs": 0.1503649353981018,
      "epoch": 0.09852999602701629,
      "grad_norm": 211.0,
      "learning_rate": 8.776913673594967e-08,
      "logits/chosen": -2.0958151817321777,
      "logits/rejected": -1.9892704486846924,
      "logps/chosen": -0.30442753434181213,
      "logps/rejected": -0.30017727613449097,
      "loss": 0.08808089792728424,
      "loss/core": 0.0615558996796608,
      "loss/preference_sft": 0.3082145154476166,
      "loss/reference_anchor": 0.3228452503681183,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.3031153678894043,
      "rewards/margins": 0.8013383150100708,
      "rewards/rejected": 1.501776933670044,
      "ronpo/logit": 0.0801338404417038,
      "ronpo/residual": 0.08253569155931473,
      "ronpo/residual_abs": 0.11766223609447479,
      "ronpo/target": -0.0024018441326916218,
      "ronpo/target_abs": 0.03481123223900795,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 310
    },
    {
      "drift/chosen_abs": 0.21849361062049866,
      "drift/rejected_abs": 0.1704569309949875,
      "epoch": 0.10011918951132301,
      "grad_norm": 1.9609375,
      "learning_rate": 8.708846574349372e-08,
      "logits/chosen": -2.206904888153076,
      "logits/rejected": -2.2400927543640137,
      "logps/chosen": -0.28868699073791504,
      "logps/rejected": -0.2845986485481262,
      "loss": 0.1418466717004776,
      "loss/core": 0.08006531000137329,
      "loss/preference_sft": 0.28893551230430603,
      "loss/reference_anchor": 0.794857919216156,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.184345245361328,
      "rewards/margins": 0.48178085684776306,
      "rewards/rejected": 1.7025642395019531,
      "ronpo/logit": 0.04817808419466019,
      "ronpo/residual": 0.044129565358161926,
      "ronpo/residual_abs": 0.11013004928827286,
      "ronpo/target": 0.004048521164804697,
      "ronpo/target_abs": 0.03925003856420517,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 315
    },
    {
      "drift/chosen_abs": 0.3064383864402771,
      "drift/rejected_abs": 0.15249067544937134,
      "epoch": 0.10170838299562972,
      "grad_norm": 9.625,
      "learning_rate": 8.63921582170582e-08,
      "logits/chosen": -2.0485007762908936,
      "logits/rejected": -2.04766583442688,
      "logps/chosen": -0.2673102021217346,
      "logps/rejected": -0.2666773200035095,
      "loss": 0.43610453605651855,
      "loss/core": 0.3816451132297516,
      "loss/preference_sft": 0.2681375741958618,
      "loss/reference_anchor": 0.6993125081062317,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.062673807144165,
      "rewards/margins": 1.5397114753723145,
      "rewards/rejected": 1.5229620933532715,
      "ronpo/logit": 0.15397116541862488,
      "ronpo/residual": 0.15312956273555756,
      "ronpo/residual_abs": 0.18270666897296906,
      "ronpo/target": 0.0008415899937972426,
      "ronpo/target_abs": 0.0406792089343071,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 320
    },
    {
      "drift/chosen_abs": 0.2704039514064789,
      "drift/rejected_abs": 0.06453654170036316,
      "epoch": 0.10329757647993643,
      "grad_norm": 23.0,
      "learning_rate": 8.568050772058761e-08,
      "logits/chosen": -2.214332103729248,
      "logits/rejected": -2.2380669116973877,
      "logps/chosen": -0.3050305247306824,
      "logps/rejected": -0.2898855209350586,
      "loss": 1.148066759109497,
      "loss/core": 1.099792718887329,
      "loss/preference_sft": 0.30017027258872986,
      "loss/reference_anchor": 0.6136360168457031,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.703939437866211,
      "rewards/margins": 2.0673062801361084,
      "rewards/rejected": 0.6366328001022339,
      "ronpo/logit": 0.20673063397407532,
      "ronpo/residual": 0.20851190388202667,
      "ronpo/residual_abs": 0.2224424183368683,
      "ronpo/target": -0.001781263155862689,
      "ronpo/target_abs": 0.03680066391825676,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 325
    },
    {
      "drift/chosen_abs": 0.2501479387283325,
      "drift/rejected_abs": 0.12387964874505997,
      "epoch": 0.10488676996424315,
      "grad_norm": 12.5625,
      "learning_rate": 8.49538142866378e-08,
      "logits/chosen": -2.323371648788452,
      "logits/rejected": -2.3236114978790283,
      "logps/chosen": -0.26736950874328613,
      "logps/rejected": -0.2633174955844879,
      "loss": 0.5597383379936218,
      "loss/core": 0.532612681388855,
      "loss/preference_sft": 0.2744181752204895,
      "loss/reference_anchor": 0.3342326879501343,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.501479387283325,
      "rewards/margins": 1.2703591585159302,
      "rewards/rejected": 1.2311203479766846,
      "ronpo/logit": 0.12703590095043182,
      "ronpo/residual": 0.1348484754562378,
      "ronpo/residual_abs": 0.23397144675254822,
      "ronpo/target": -0.007812569849193096,
      "ronpo/target_abs": 0.04597053676843643,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 330
    },
    {
      "drift/chosen_abs": 0.44480544328689575,
      "drift/rejected_abs": 0.20965024828910828,
      "epoch": 0.10647596344854986,
      "grad_norm": 12.125,
      "learning_rate": 8.421238428988198e-08,
      "logits/chosen": -1.9913667440414429,
      "logits/rejected": -1.9402954578399658,
      "logps/chosen": -0.2954614460468292,
      "logps/rejected": -0.29534387588500977,
      "loss": 0.8044527173042297,
      "loss/core": 0.697630763053894,
      "loss/preference_sft": 0.2934873104095459,
      "loss/reference_anchor": 1.394944429397583,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 4.447211742401123,
      "rewards/margins": 2.354290246963501,
      "rewards/rejected": 2.0929219722747803,
      "ronpo/logit": 0.235429048538208,
      "ronpo/residual": 0.2442881315946579,
      "ronpo/residual_abs": 0.2787492871284485,
      "ronpo/target": -0.008859092369675636,
      "ronpo/target_abs": 0.036721840500831604,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 335
    },
    {
      "drift/chosen_abs": 0.4596560597419739,
      "drift/rejected_abs": 0.16454848647117615,
      "epoch": 0.10806515693285658,
      "grad_norm": 82.5,
      "learning_rate": 8.34565303179429e-08,
      "logits/chosen": -1.884810447692871,
      "logits/rejected": -1.9432003498077393,
      "logps/chosen": -0.28668686747550964,
      "logps/rejected": -0.2740832567214966,
      "loss": 1.0088319778442383,
      "loss/core": 0.9366035461425781,
      "loss/preference_sft": 0.28669866919517517,
      "loss/reference_anchor": 0.9343746304512024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.593608856201172,
      "rewards/margins": 2.9580533504486084,
      "rewards/rejected": 1.6355552673339844,
      "ronpo/logit": 0.29580533504486084,
      "ronpo/residual": 0.29322350025177,
      "ronpo/residual_abs": 0.311967670917511,
      "ronpo/target": 0.0025818462017923594,
      "ronpo/target_abs": 0.033389072865247726,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 340
    },
    {
      "drift/chosen_abs": 0.24590358138084412,
      "drift/rejected_abs": 0.11953750997781754,
      "epoch": 0.10965435041716329,
      "grad_norm": 59.5,
      "learning_rate": 8.268657103960556e-08,
      "logits/chosen": -2.178544521331787,
      "logits/rejected": -2.251857280731201,
      "logps/chosen": -0.272917777299881,
      "logps/rejected": -0.2786032259464264,
      "loss": 0.12808948755264282,
      "loss/core": 0.10189362615346909,
      "loss/preference_sft": 0.28778940439224243,
      "loss/reference_anchor": 0.3204990029335022,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.4582910537719727,
      "rewards/margins": 1.286404013633728,
      "rewards/rejected": 1.171886920928955,
      "ronpo/logit": 0.12864042818546295,
      "ronpo/residual": 0.13276901841163635,
      "ronpo/residual_abs": 0.1446714699268341,
      "ronpo/target": -0.004128606058657169,
      "ronpo/target_abs": 0.030540451407432556,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 345
    },
    {
      "drift/chosen_abs": 0.4136156439781189,
      "drift/rejected_abs": 0.16546812653541565,
      "epoch": 0.11124354390147001,
      "grad_norm": 1.4296875,
      "learning_rate": 8.190283107046611e-08,
      "logits/chosen": -2.2700629234313965,
      "logits/rejected": -2.299020767211914,
      "logps/chosen": -0.23404893279075623,
      "logps/rejected": -0.24618962407112122,
      "loss": 0.9944799542427063,
      "loss/core": 0.9109455347061157,
      "loss/preference_sft": 0.23421046137809753,
      "loss/reference_anchor": 1.0903706550598145,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.136050224304199,
      "rewards/margins": 2.4818317890167236,
      "rewards/rejected": 1.6542189121246338,
      "ronpo/logit": 0.2481832057237625,
      "ronpo/residual": 0.24511413276195526,
      "ronpo/residual_abs": 0.266287237405777,
      "ronpo/target": 0.0030690638814121485,
      "ronpo/target_abs": 0.03295261040329933,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 350
    },
    {
      "drift/chosen_abs": 0.28086286783218384,
      "drift/rejected_abs": 0.1028003841638565,
      "epoch": 0.11283273738577672,
      "grad_norm": 17.0,
      "learning_rate": 8.11056408360737e-08,
      "logits/chosen": -2.126852512359619,
      "logits/rejected": -2.0258126258850098,
      "logps/chosen": -0.2836364507675171,
      "logps/rejected": -0.27664485573768616,
      "loss": 0.4193928837776184,
      "loss/core": 0.39380452036857605,
      "loss/preference_sft": 0.2735845446586609,
      "loss/reference_anchor": 0.3138180375099182,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.8082804679870605,
      "rewards/margins": 1.7823970317840576,
      "rewards/rejected": 1.0258831977844238,
      "ronpo/logit": 0.17823971807956696,
      "ronpo/residual": 0.17218564450740814,
      "ronpo/residual_abs": 0.20453758537769318,
      "ronpo/target": 0.0060540675185620785,
      "ronpo/target_abs": 0.0407588966190815,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 355
    },
    {
      "drift/chosen_abs": 0.2176329344511032,
      "drift/rejected_abs": 0.08809185028076172,
      "epoch": 0.11442193087008343,
      "grad_norm": 10.875,
      "learning_rate": 8.029533643262255e-08,
      "logits/chosen": -2.2074317932128906,
      "logits/rejected": -2.2725961208343506,
      "logps/chosen": -0.267316609621048,
      "logps/rejected": -0.25893837213516235,
      "loss": 0.08880702406167984,
      "loss/core": 0.07977307587862015,
      "loss/preference_sft": 0.26773780584335327,
      "loss/reference_anchor": 0.09367872774600983,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.175424098968506,
      "rewards/margins": 1.2949514389038086,
      "rewards/rejected": 0.8804727792739868,
      "ronpo/logit": 0.12949514389038086,
      "ronpo/residual": 0.13094380497932434,
      "ronpo/residual_abs": 0.15374919772148132,
      "ronpo/target": -0.0014486557338386774,
      "ronpo/target_abs": 0.03519626706838608,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 360
    },
    {
      "drift/chosen_abs": 0.20784783363342285,
      "drift/rejected_abs": 0.12744882702827454,
      "epoch": 0.11601112435439015,
      "grad_norm": 1.9140625,
      "learning_rate": 7.947225948525369e-08,
      "logits/chosen": -2.0954489707946777,
      "logits/rejected": -2.1332972049713135,
      "logps/chosen": -0.28586500883102417,
      "logps/rejected": -0.32812875509262085,
      "loss": 0.07003101706504822,
      "loss/core": 0.04061419516801834,
      "loss/preference_sft": 0.2947535812854767,
      "loss/reference_anchor": 0.36274877190589905,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0766704082489014,
      "rewards/margins": 0.8259242177009583,
      "rewards/rejected": 1.2507461309432983,
      "ronpo/logit": 0.0825924351811409,
      "ronpo/residual": 0.07970172166824341,
      "ronpo/residual_abs": 0.10139290988445282,
      "ronpo/target": 0.002890714444220066,
      "ronpo/target_abs": 0.04324514418840408,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 365
    },
    {
      "drift/chosen_abs": 0.3182205855846405,
      "drift/rejected_abs": 0.21468524634838104,
      "epoch": 0.11760031783869686,
      "grad_norm": 39.25,
      "learning_rate": 7.863675700402526e-08,
      "logits/chosen": -2.134063720703125,
      "logits/rejected": -2.1670172214508057,
      "logps/chosen": -0.278795450925827,
      "logps/rejected": -0.28225570917129517,
      "loss": 0.19065257906913757,
      "loss/core": 0.10301892459392548,
      "loss/preference_sft": 0.28197628259658813,
      "loss/reference_anchor": 1.1402509212493896,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.1817617416381836,
      "rewards/margins": 1.0355478525161743,
      "rewards/rejected": 2.146214008331299,
      "ronpo/logit": 0.10355480015277863,
      "ronpo/residual": 0.09973715245723724,
      "ronpo/residual_abs": 0.13304369151592255,
      "ronpo/target": 0.003817648161202669,
      "ronpo/target_abs": 0.03572530299425125,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 370
    },
    {
      "drift/chosen_abs": 0.21340015530586243,
      "drift/rejected_abs": 0.0985214114189148,
      "epoch": 0.11918951132300358,
      "grad_norm": 21.125,
      "learning_rate": 7.778918123761286e-08,
      "logits/chosen": -2.27836012840271,
      "logits/rejected": -2.302485942840576,
      "logps/chosen": -0.29381102323532104,
      "logps/rejected": -0.2883267104625702,
      "loss": 0.1361909955739975,
      "loss/core": 0.12311957031488419,
      "loss/preference_sft": 0.2917711138725281,
      "loss/reference_anchor": 0.14510871469974518,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.133848190307617,
      "rewards/margins": 1.1486339569091797,
      "rewards/rejected": 0.9852142333984375,
      "ronpo/logit": 0.11486339569091797,
      "ronpo/residual": 0.1210290789604187,
      "ronpo/residual_abs": 0.14276602864265442,
      "ronpo/target": -0.006165677681565285,
      "ronpo/target_abs": 0.04001092165708542,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 375
    },
    {
      "drift/chosen_abs": 0.2751646339893341,
      "drift/rejected_abs": 0.09928236156702042,
      "epoch": 0.12077870480731029,
      "grad_norm": 0.63671875,
      "learning_rate": 7.692988952480112e-08,
      "logits/chosen": -2.1607887744903564,
      "logits/rejected": -2.1772239208221436,
      "logps/chosen": -0.2890273928642273,
      "logps/rejected": -0.2840579152107239,
      "loss": 0.5159703493118286,
      "loss/core": 0.4876245856285095,
      "loss/preference_sft": 0.2878762185573578,
      "loss/reference_anchor": 0.3491542637348175,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.7504799365997314,
      "rewards/margins": 1.7696754932403564,
      "rewards/rejected": 0.9808047413825989,
      "ronpo/logit": 0.1769675463438034,
      "ronpo/residual": 0.16951511800289154,
      "ronpo/residual_abs": 0.19311194121837616,
      "ronpo/target": 0.007452424615621567,
      "ronpo/target_abs": 0.039471663534641266,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 380
    },
    {
      "drift/chosen_abs": 0.25231146812438965,
      "drift/rejected_abs": 0.14849232137203217,
      "epoch": 0.12236789829161701,
      "grad_norm": 60.25,
      "learning_rate": 7.605924414382924e-08,
      "logits/chosen": -2.277235746383667,
      "logits/rejected": -2.2479469776153564,
      "logps/chosen": -0.2711496949195862,
      "logps/rejected": -0.26407307386398315,
      "loss": 0.17193853855133057,
      "loss/core": 0.15317845344543457,
      "loss/preference_sft": 0.26539701223373413,
      "loss/reference_anchor": 0.22359445691108704,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.5228137969970703,
      "rewards/margins": 1.0395729541778564,
      "rewards/rejected": 1.4832407236099243,
      "ronpo/logit": 0.10395729541778564,
      "ronpo/residual": 0.09792117774486542,
      "ronpo/residual_abs": 0.12750428915023804,
      "ronpo/target": 0.006036114878952503,
      "ronpo/target_abs": 0.042838118970394135,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 385
    },
    {
      "drift/chosen_abs": 0.21072201430797577,
      "drift/rejected_abs": 0.12239472568035126,
      "epoch": 0.12395709177592372,
      "grad_norm": 3.4375,
      "learning_rate": 7.517761215965428e-08,
      "logits/chosen": -2.078555107116699,
      "logits/rejected": -2.1201419830322266,
      "logps/chosen": -0.30479007959365845,
      "logps/rejected": -0.2949450612068176,
      "loss": 0.09117992967367172,
      "loss/core": 0.0706782341003418,
      "loss/preference_sft": 0.295664519071579,
      "loss/reference_anchor": 0.24378947913646698,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.1056594848632812,
      "rewards/margins": 0.8848453760147095,
      "rewards/rejected": 1.2208143472671509,
      "ronpo/logit": 0.08848454058170319,
      "ronpo/residual": 0.09790941327810287,
      "ronpo/residual_abs": 0.11511196941137314,
      "ronpo/target": -0.009424876421689987,
      "ronpo/target_abs": 0.038498423993587494,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 390
    },
    {
      "drift/chosen_abs": 0.21178297698497772,
      "drift/rejected_abs": 0.0658094510436058,
      "epoch": 0.12554628526023043,
      "grad_norm": 3.046875,
      "learning_rate": 7.428536526919602e-08,
      "logits/chosen": -2.158111095428467,
      "logits/rejected": -2.221972942352295,
      "logps/chosen": -0.31177249550819397,
      "logps/rejected": -0.30004364252090454,
      "loss": 0.3803747296333313,
      "loss/core": 0.3620082139968872,
      "loss/preference_sft": 0.306712806224823,
      "loss/reference_anchor": 0.21421615779399872,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.117314100265503,
      "rewards/margins": 1.459604024887085,
      "rewards/rejected": 0.657710075378418,
      "ronpo/logit": 0.14596042037010193,
      "ronpo/residual": 0.1454031765460968,
      "ronpo/residual_abs": 0.16442766785621643,
      "ronpo/target": 0.0005572382360696793,
      "ronpo/target_abs": 0.03522815182805061,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 395
    },
    {
      "drift/chosen_abs": 0.19120144844055176,
      "drift/rejected_abs": 0.08451732248067856,
      "epoch": 0.12713547874453715,
      "grad_norm": 9.5,
      "learning_rate": 7.338287964462934e-08,
      "logits/chosen": -2.2306790351867676,
      "logits/rejected": -2.3209800720214844,
      "logps/chosen": -0.25742968916893005,
      "logps/rejected": -0.2603406012058258,
      "loss": 0.07361692190170288,
      "loss/core": 0.06260836124420166,
      "loss/preference_sft": 0.25700175762176514,
      "loss/reference_anchor": 0.12108077108860016,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.9112262725830078,
      "rewards/margins": 1.0695370435714722,
      "rewards/rejected": 0.8416891098022461,
      "ronpo/logit": 0.1069537028670311,
      "ronpo/residual": 0.10586754232645035,
      "ronpo/residual_abs": 0.12485964596271515,
      "ronpo/target": 0.0010861654300242662,
      "ronpo/target_abs": 0.037036895751953125,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 400
    },
    {
      "drift/chosen_abs": 0.17024962604045868,
      "drift/rejected_abs": 0.14562097191810608,
      "epoch": 0.12872467222884387,
      "grad_norm": 1.8125,
      "learning_rate": 7.247053577478954e-08,
      "logits/chosen": -2.2906746864318848,
      "logits/rejected": -2.299346446990967,
      "logps/chosen": -0.29554539918899536,
      "logps/rejected": -0.2890940010547638,
      "loss": 0.2859688401222229,
      "loss/core": 0.2667706608772278,
      "loss/preference_sft": 0.2862167954444885,
      "loss/reference_anchor": 0.22735345363616943,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6998668909072876,
      "rewards/margins": 0.24389216303825378,
      "rewards/rejected": 1.4559749364852905,
      "ronpo/logit": 0.02438921108841896,
      "ronpo/residual": 0.024855410680174828,
      "ronpo/residual_abs": 0.1514817178249359,
      "ronpo/target": -0.0004661986604332924,
      "ronpo/target_abs": 0.03743233159184456,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 405
    },
    {
      "drift/chosen_abs": 0.1870613545179367,
      "drift/rejected_abs": 0.05845289304852486,
      "epoch": 0.13031386571315057,
      "grad_norm": 0.78515625,
      "learning_rate": 7.154871830475796e-08,
      "logits/chosen": -2.2287261486053467,
      "logits/rejected": -2.140763759613037,
      "logps/chosen": -0.2894555628299713,
      "logps/rejected": -0.287057101726532,
      "loss": 0.3428201377391815,
      "loss/core": 0.32726067304611206,
      "loss/preference_sft": 0.28788700699806213,
      "loss/reference_anchor": 0.1786712408065796,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8684613704681396,
      "rewards/margins": 1.3022751808166504,
      "rewards/rejected": 0.566186249256134,
      "ronpo/logit": 0.1302274912595749,
      "ronpo/residual": 0.13139872252941132,
      "ronpo/residual_abs": 0.15106776356697083,
      "ronpo/target": -0.0011712164850905538,
      "ronpo/target_abs": 0.035140570253133774,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 410
    },
    {
      "drift/chosen_abs": 0.12820783257484436,
      "drift/rejected_abs": 0.059625618159770966,
      "epoch": 0.1319030591974573,
      "grad_norm": 18.875,
      "learning_rate": 7.061781587369517e-08,
      "logits/chosen": -2.267620086669922,
      "logits/rejected": -2.297600269317627,
      "logps/chosen": -0.2848357558250427,
      "logps/rejected": -0.2741553485393524,
      "loss": 0.03822881355881691,
      "loss/core": 0.03276274353265762,
      "loss/preference_sft": 0.2775842547416687,
      "loss/reference_anchor": 0.04512251168489456,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.2819339036941528,
      "rewards/margins": 0.6913110017776489,
      "rewards/rejected": 0.5906227827072144,
      "ronpo/logit": 0.06913109868764877,
      "ronpo/residual": 0.06746384501457214,
      "ronpo/residual_abs": 0.09230469167232513,
      "ronpo/target": 0.0016672604251652956,
      "ronpo/target_abs": 0.03702988475561142,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 415
    },
    {
      "drift/chosen_abs": 0.39186540246009827,
      "drift/rejected_abs": 0.12389961630105972,
      "epoch": 0.133492252681764,
      "grad_norm": 2.9375,
      "learning_rate": 6.96782209509903e-08,
      "logits/chosen": -2.114809513092041,
      "logits/rejected": -2.2321271896362305,
      "logps/chosen": -0.2693604826927185,
      "logps/rejected": -0.2715781331062317,
      "loss": 0.8839341998100281,
      "loss/core": 0.8307901620864868,
      "loss/preference_sft": 0.2737378180027008,
      "loss/reference_anchor": 0.6812156438827515,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.918318271636963,
      "rewards/margins": 2.7466630935668945,
      "rewards/rejected": 1.1716548204421997,
      "ronpo/logit": 0.27466633915901184,
      "ronpo/residual": 0.26872462034225464,
      "ronpo/residual_abs": 0.286480188369751,
      "ronpo/target": 0.005941734649240971,
      "ronpo/target_abs": 0.039312396198511124,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 420
    },
    {
      "drift/chosen_abs": 0.4238465428352356,
      "drift/rejected_abs": 0.1388479471206665,
      "epoch": 0.1350814461660707,
      "grad_norm": 306.0,
      "learning_rate": 6.87303296707956e-08,
      "logits/chosen": -2.164503574371338,
      "logits/rejected": -2.157972812652588,
      "logps/chosen": -0.2988634407520294,
      "logps/rejected": -0.29640263319015503,
      "loss": 1.2289950847625732,
      "loss/core": 1.1507900953292847,
      "loss/preference_sft": 0.2907932698726654,
      "loss/reference_anchor": 1.0136525630950928,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 4.237428188323975,
      "rewards/margins": 2.8503975868225098,
      "rewards/rejected": 1.3870306015014648,
      "ronpo/logit": 0.2850397527217865,
      "ronpo/residual": 0.27526021003723145,
      "ronpo/residual_abs": 0.308517187833786,
      "ronpo/target": 0.009779555723071098,
      "ronpo/target_abs": 0.04520432651042938,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 425
    },
    {
      "drift/chosen_abs": 0.18785476684570312,
      "drift/rejected_abs": 0.09160666167736053,
      "epoch": 0.13667063965037743,
      "grad_norm": 11.9375,
      "learning_rate": 6.777454166501589e-08,
      "logits/chosen": -2.4303414821624756,
      "logits/rejected": -2.4165592193603516,
      "logps/chosen": -0.27914959192276,
      "logps/rejected": -0.27462783455848694,
      "loss": 0.09819485992193222,
      "loss/core": 0.08552583307027817,
      "loss/preference_sft": 0.2734072804450989,
      "loss/reference_anchor": 0.1415795087814331,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8771387338638306,
      "rewards/margins": 0.9617449641227722,
      "rewards/rejected": 0.9153939485549927,
      "ronpo/logit": 0.09617450088262558,
      "ronpo/residual": 0.0931866317987442,
      "ronpo/residual_abs": 0.11439953744411469,
      "ronpo/target": 0.002987860469147563,
      "ronpo/target_abs": 0.033472005277872086,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 430
    },
    {
      "drift/chosen_abs": 0.17324425280094147,
      "drift/rejected_abs": 0.09321893751621246,
      "epoch": 0.13825983313468415,
      "grad_norm": 11.875,
      "learning_rate": 6.681125989482337e-08,
      "logits/chosen": -2.1825103759765625,
      "logits/rejected": -2.1760473251342773,
      "logps/chosen": -0.27965161204338074,
      "logps/rejected": -0.2805442810058594,
      "loss": 0.04018530249595642,
      "loss/core": 0.03218042477965355,
      "loss/preference_sft": 0.2764553427696228,
      "loss/reference_anchor": 0.07908622920513153,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7314059734344482,
      "rewards/margins": 0.7999973297119141,
      "rewards/rejected": 0.9314085841178894,
      "ronpo/logit": 0.07999972999095917,
      "ronpo/residual": 0.0743856281042099,
      "ronpo/residual_abs": 0.1003790944814682,
      "ronpo/target": 0.005614105612039566,
      "ronpo/target_abs": 0.033698342740535736,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 435
    },
    {
      "drift/chosen_abs": 0.31887176632881165,
      "drift/rejected_abs": 0.1319849044084549,
      "epoch": 0.13984902661899087,
      "grad_norm": 3.984375,
      "learning_rate": 6.584089048076865e-08,
      "logits/chosen": -2.2736406326293945,
      "logits/rejected": -2.299166440963745,
      "logps/chosen": -0.2946208119392395,
      "logps/rejected": -0.2924998998641968,
      "loss": 0.7576056718826294,
      "loss/core": 0.6892427206039429,
      "loss/preference_sft": 0.299308180809021,
      "loss/reference_anchor": 0.8815743327140808,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.1881179809570312,
      "rewards/margins": 1.878112554550171,
      "rewards/rejected": 1.3100051879882812,
      "ronpo/logit": 0.1878112554550171,
      "ronpo/residual": 0.18988080322742462,
      "ronpo/residual_abs": 0.21202602982521057,
      "ronpo/target": -0.002069511916488409,
      "ronpo/target_abs": 0.04406188428401947,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 440
    },
    {
      "drift/chosen_abs": 0.19031694531440735,
      "drift/rejected_abs": 0.16893598437309265,
      "epoch": 0.14143822010329757,
      "grad_norm": 264.0,
      "learning_rate": 6.486384253156013e-08,
      "logits/chosen": -2.2305800914764404,
      "logits/rejected": -2.154871702194214,
      "logps/chosen": -0.28044503927230835,
      "logps/rejected": -0.2716797888278961,
      "loss": 0.24689729511737823,
      "loss/core": 0.22590681910514832,
      "loss/preference_sft": 0.2807045876979828,
      "loss/reference_anchor": 0.2518024444580078,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.9031444787979126,
      "rewards/margins": 0.2139008492231369,
      "rewards/rejected": 1.6892436742782593,
      "ronpo/logit": 0.021390080451965332,
      "ronpo/residual": 0.023730376735329628,
      "ronpo/residual_abs": 0.16236847639083862,
      "ronpo/target": -0.0023402937222272158,
      "ronpo/target_abs": 0.0374097041785717,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 445
    },
    {
      "drift/chosen_abs": 0.3165225088596344,
      "drift/rejected_abs": 0.10332963615655899,
      "epoch": 0.1430274135876043,
      "grad_norm": 32.25,
      "learning_rate": 6.388052797158323e-08,
      "logits/chosen": -2.2609755992889404,
      "logits/rejected": -2.325429677963257,
      "logps/chosen": -0.271506130695343,
      "logps/rejected": -0.2823403477668762,
      "loss": 0.7201793193817139,
      "loss/core": 0.6770936846733093,
      "loss/preference_sft": 0.27659398317337036,
      "loss/reference_anchor": 0.5468156933784485,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.1648669242858887,
      "rewards/margins": 2.1341538429260254,
      "rewards/rejected": 1.030713438987732,
      "ronpo/logit": 0.21341535449028015,
      "ronpo/residual": 0.21813836693763733,
      "ronpo/residual_abs": 0.23593993484973907,
      "ronpo/target": -0.004723007790744305,
      "ronpo/target_abs": 0.03459266573190689,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 450
    },
    {
      "drift/chosen_abs": 0.20675233006477356,
      "drift/rejected_abs": 0.09977269917726517,
      "epoch": 0.144616607071911,
      "grad_norm": 10.1875,
      "learning_rate": 6.289136136723267e-08,
      "logits/chosen": -2.3178977966308594,
      "logits/rejected": -2.3515288829803467,
      "logps/chosen": -0.28541308641433716,
      "logps/rejected": -0.2820412218570709,
      "loss": 0.10540688037872314,
      "loss/core": 0.09275539964437485,
      "loss/preference_sft": 0.2944668233394623,
      "loss/reference_anchor": 0.13923971354961395,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.067516326904297,
      "rewards/margins": 1.0707690715789795,
      "rewards/rejected": 0.9967470169067383,
      "ronpo/logit": 0.10707690566778183,
      "ronpo/residual": 0.10755382478237152,
      "ronpo/residual_abs": 0.13240396976470947,
      "ronpo/target": -0.0004769214428961277,
      "ronpo/target_abs": 0.03637883812189102,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 455
    },
    {
      "drift/chosen_abs": 0.37812691926956177,
      "drift/rejected_abs": 0.16481459140777588,
      "epoch": 0.1462058005562177,
      "grad_norm": 117.0,
      "learning_rate": 6.189675975213094e-08,
      "logits/chosen": -1.8068339824676514,
      "logits/rejected": -1.666780710220337,
      "logps/chosen": -0.28048020601272583,
      "logps/rejected": -0.29942306876182556,
      "loss": 0.5667847394943237,
      "loss/core": 0.5201860666275024,
      "loss/preference_sft": 0.29059094190597534,
      "loss/reference_anchor": 0.5922571420669556,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.775576114654541,
      "rewards/margins": 2.1349546909332275,
      "rewards/rejected": 1.6406214237213135,
      "ronpo/logit": 0.21349544823169708,
      "ronpo/residual": 0.21162322163581848,
      "ronpo/residual_abs": 0.234365776181221,
      "ronpo/target": 0.0018722775857895613,
      "ronpo/target_abs": 0.03759382292628288,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 460
    },
    {
      "drift/chosen_abs": 0.34445223212242126,
      "drift/rejected_abs": 0.23401422798633575,
      "epoch": 0.14779499404052443,
      "grad_norm": 253.0,
      "learning_rate": 6.089714245130639e-08,
      "logits/chosen": -2.095383405685425,
      "logits/rejected": -2.1152687072753906,
      "logps/chosen": -0.2696693539619446,
      "logps/rejected": -0.2734582722187042,
      "loss": 0.82310551404953,
      "loss/core": 0.7762866020202637,
      "loss/preference_sft": 0.2763036787509918,
      "loss/reference_anchor": 0.5966211557388306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.4437789916992188,
      "rewards/margins": 1.104857087135315,
      "rewards/rejected": 2.3389220237731934,
      "ronpo/logit": 0.11048571765422821,
      "ronpo/residual": 0.1046980768442154,
      "ronpo/residual_abs": 0.27617347240448,
      "ronpo/target": 0.005787642206996679,
      "ronpo/target_abs": 0.03396866098046303,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 465
    },
    {
      "drift/chosen_abs": 0.2995480000972748,
      "drift/rejected_abs": 0.11703862994909286,
      "epoch": 0.14938418752483115,
      "grad_norm": 3.09375,
      "learning_rate": 5.98929309044055e-08,
      "logits/chosen": -2.0731537342071533,
      "logits/rejected": -2.0429816246032715,
      "logps/chosen": -0.25429534912109375,
      "logps/rejected": -0.2596786320209503,
      "loss": 0.7873652577400208,
      "loss/core": 0.7461782693862915,
      "loss/preference_sft": 0.25899848341941833,
      "loss/reference_anchor": 0.5232595205307007,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.9948370456695557,
      "rewards/margins": 1.825717568397522,
      "rewards/rejected": 1.1691195964813232,
      "ronpo/logit": 0.18257175385951996,
      "ronpo/residual": 0.18372610211372375,
      "ronpo/residual_abs": 0.22811272740364075,
      "ronpo/target": -0.0011543564032763243,
      "ronpo/target_abs": 0.039658524096012115,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 470
    },
    {
      "drift/chosen_abs": 0.2512029707431793,
      "drift/rejected_abs": 0.09701138734817505,
      "epoch": 0.15097338100913787,
      "grad_norm": 7.75,
      "learning_rate": 5.888454848801344e-08,
      "logits/chosen": -2.195828676223755,
      "logits/rejected": -2.1313068866729736,
      "logps/chosen": -0.30464741587638855,
      "logps/rejected": -0.2989063858985901,
      "loss": 0.21697983145713806,
      "loss/core": 0.20200279355049133,
      "loss/preference_sft": 0.3056924343109131,
      "loss/reference_anchor": 0.169124573469162,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.510868787765503,
      "rewards/margins": 1.5423170328140259,
      "rewards/rejected": 0.9685519933700562,
      "ronpo/logit": 0.1542317122220993,
      "ronpo/residual": 0.1569804847240448,
      "ronpo/residual_abs": 0.1833471953868866,
      "ronpo/target": -0.0027487764600664377,
      "ronpo/target_abs": 0.035739701241254807,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 475
    },
    {
      "drift/chosen_abs": 0.221854567527771,
      "drift/rejected_abs": 0.10892774909734726,
      "epoch": 0.15256257449344457,
      "grad_norm": 1.4921875,
      "learning_rate": 5.787242033715808e-08,
      "logits/chosen": -2.1198408603668213,
      "logits/rejected": -2.14213490486145,
      "logps/chosen": -0.2731625437736511,
      "logps/rejected": -0.2686478793621063,
      "loss": 0.0852787047624588,
      "loss/core": 0.07367287576198578,
      "loss/preference_sft": 0.26938527822494507,
      "loss/reference_anchor": 0.12780579924583435,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.216320753097534,
      "rewards/margins": 1.1356593370437622,
      "rewards/rejected": 1.0806615352630615,
      "ronpo/logit": 0.11356594413518906,
      "ronpo/residual": 0.12406400591135025,
      "ronpo/residual_abs": 0.13756707310676575,
      "ronpo/target": -0.010498052462935448,
      "ronpo/target_abs": 0.03645600751042366,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 480
    },
    {
      "drift/chosen_abs": 0.29766806960105896,
      "drift/rejected_abs": 0.1601838320493698,
      "epoch": 0.1541517679777513,
      "grad_norm": 6.1875,
      "learning_rate": 5.685697316607273e-08,
      "logits/chosen": -1.901150107383728,
      "logits/rejected": -2.023740530014038,
      "logps/chosen": -0.2827039659023285,
      "logps/rejected": -0.28020596504211426,
      "loss": 0.13552992045879364,
      "loss/core": 0.10949452966451645,
      "loss/preference_sft": 0.2870265245437622,
      "loss/reference_anchor": 0.31843602657318115,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.975731134414673,
      "rewards/margins": 1.3740638494491577,
      "rewards/rejected": 1.601667046546936,
      "ronpo/logit": 0.1374063938856125,
      "ronpo/residual": 0.13112321496009827,
      "ronpo/residual_abs": 0.15812107920646667,
      "ronpo/target": 0.006283201277256012,
      "ronpo/target_abs": 0.04036698862910271,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 485
    },
    {
      "drift/chosen_abs": 0.3679271936416626,
      "drift/rejected_abs": 0.15137556195259094,
      "epoch": 0.15574096146205801,
      "grad_norm": 362.0,
      "learning_rate": 5.583863508829281e-08,
      "logits/chosen": -2.1041316986083984,
      "logits/rejected": -2.0852389335632324,
      "logps/chosen": -0.28241246938705444,
      "logps/rejected": -0.282585084438324,
      "loss": 0.681788980960846,
      "loss/core": 0.6289718747138977,
      "loss/preference_sft": 0.2821527421474457,
      "loss/reference_anchor": 0.6760126948356628,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.6771881580352783,
      "rewards/margins": 2.1641111373901367,
      "rewards/rejected": 1.5130772590637207,
      "ronpo/logit": 0.21641111373901367,
      "ronpo/residual": 0.21629562973976135,
      "ronpo/residual_abs": 0.23846352100372314,
      "ronpo/target": 0.00011549964256118983,
      "ronpo/target_abs": 0.039183907210826874,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 490
    },
    {
      "drift/chosen_abs": 0.22022119164466858,
      "drift/rejected_abs": 0.18502850830554962,
      "epoch": 0.1573301549463647,
      "grad_norm": 264.0,
      "learning_rate": 5.4817835436163e-08,
      "logits/chosen": -2.1266822814941406,
      "logits/rejected": -2.09769344329834,
      "logps/chosen": -0.286654531955719,
      "logps/rejected": -0.27344611287117004,
      "loss": 1.2056705951690674,
      "loss/core": 1.1531113386154175,
      "loss/preference_sft": 0.28491732478141785,
      "loss/reference_anchor": 0.672301709651947,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.2011239528656006,
      "rewards/margins": 0.3535767197608948,
      "rewards/rejected": 1.8475472927093506,
      "ronpo/logit": 0.03535768389701843,
      "ronpo/residual": 0.03558214753866196,
      "ronpo/residual_abs": 0.2800794839859009,
      "ronpo/target": -0.00022447053925134242,
      "ronpo/target_abs": 0.04289742931723595,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 495
    },
    {
      "drift/chosen_abs": 0.24428431689739227,
      "drift/rejected_abs": 0.14757463335990906,
      "epoch": 0.15891934843067143,
      "grad_norm": 4.09375,
      "learning_rate": 5.379500457983005e-08,
      "logits/chosen": -2.233088493347168,
      "logits/rejected": -2.089561939239502,
      "logps/chosen": -0.28068411350250244,
      "logps/rejected": -0.27640584111213684,
      "loss": 0.2265290468931198,
      "loss/core": 0.203072190284729,
      "loss/preference_sft": 0.2757698893547058,
      "loss/reference_anchor": 0.2851807475090027,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.442843437194824,
      "rewards/margins": 0.9873887896537781,
      "rewards/rejected": 1.4554545879364014,
      "ronpo/logit": 0.09873887896537781,
      "ronpo/residual": 0.10049834102392197,
      "ronpo/residual_abs": 0.17442360520362854,
      "ronpo/target": -0.0017594577511772513,
      "ronpo/target_abs": 0.03850649669766426,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 500
    },
    {
      "drift/chosen_abs": 0.1610635221004486,
      "drift/rejected_abs": 0.1057731881737709,
      "epoch": 0.16050854191497815,
      "grad_norm": 4.28125,
      "learning_rate": 5.27705737457985e-08,
      "logits/chosen": -2.282769203186035,
      "logits/rejected": -2.203336715698242,
      "logps/chosen": -0.29518648982048035,
      "logps/rejected": -0.28152212500572205,
      "loss": 0.027087271213531494,
      "loss/core": 0.01963396742939949,
      "loss/preference_sft": 0.2921666204929352,
      "loss/reference_anchor": 0.07016074657440186,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6077709197998047,
      "rewards/margins": 0.5503221154212952,
      "rewards/rejected": 1.0574488639831543,
      "ronpo/logit": 0.055032216012477875,
      "ronpo/residual": 0.04476800560951233,
      "ronpo/residual_abs": 0.08265403658151627,
      "ronpo/target": 0.010264214128255844,
      "ronpo/target_abs": 0.03978971391916275,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 505
    },
    {
      "drift/chosen_abs": 0.31371009349823,
      "drift/rejected_abs": 0.16116222739219666,
      "epoch": 0.16209773539928488,
      "grad_norm": 2.484375,
      "learning_rate": 5.1744974835125056e-08,
      "logits/chosen": -1.9395443201065063,
      "logits/rejected": -1.9623817205429077,
      "logps/chosen": -0.263325572013855,
      "logps/rejected": -0.26283055543899536,
      "loss": 0.43391937017440796,
      "loss/core": 0.3860779404640198,
      "loss/preference_sft": 0.26411715149879456,
      "loss/reference_anchor": 0.6114745140075684,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.135848045349121,
      "rewards/margins": 1.5270367860794067,
      "rewards/rejected": 1.608811378479004,
      "ronpo/logit": 0.1527036726474762,
      "ronpo/residual": 0.1507427990436554,
      "ronpo/residual_abs": 0.1738193929195404,
      "ronpo/target": 0.00196089968085289,
      "ronpo/target_abs": 0.034039102494716644,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 510
    },
    {
      "drift/chosen_abs": 0.2592666745185852,
      "drift/rejected_abs": 0.1546706259250641,
      "epoch": 0.16368692888359157,
      "grad_norm": 5.0,
      "learning_rate": 5.071864024132867e-08,
      "logits/chosen": -2.1409928798675537,
      "logits/rejected": -2.2105953693389893,
      "logps/chosen": -0.2879509925842285,
      "logps/rejected": -0.2745327353477478,
      "loss": 0.34892356395721436,
      "loss/core": 0.3256063461303711,
      "loss/preference_sft": 0.2804018259048462,
      "loss/reference_anchor": 0.2828558087348938,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.5926671028137207,
      "rewards/margins": 1.0476982593536377,
      "rewards/rejected": 1.544968843460083,
      "ronpo/logit": 0.10476982593536377,
      "ronpo/residual": 0.10065307468175888,
      "ronpo/residual_abs": 0.19120723009109497,
      "ronpo/target": 0.0041167414747178555,
      "ronpo/target_abs": 0.037326134741306305,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 515
    },
    {
      "drift/chosen_abs": 0.21726028621196747,
      "drift/rejected_abs": 0.10538153350353241,
      "epoch": 0.1652761223678983,
      "grad_norm": 2.859375,
      "learning_rate": 4.969200266809307e-08,
      "logits/chosen": -2.1490025520324707,
      "logits/rejected": -2.16178560256958,
      "logps/chosen": -0.29243892431259155,
      "logps/rejected": -0.30090299248695374,
      "loss": 0.26550185680389404,
      "loss/core": 0.22479858994483948,
      "loss/preference_sft": 0.2937026917934418,
      "loss/reference_anchor": 0.5133408904075623,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.171968460083008,
      "rewards/margins": 1.124643087387085,
      "rewards/rejected": 1.0473252534866333,
      "ronpo/logit": 0.1124643087387085,
      "ronpo/residual": 0.10653400421142578,
      "ronpo/residual_abs": 0.131363645195961,
      "ronpo/target": 0.0059303115122020245,
      "ronpo/target_abs": 0.04164915904402733,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 520
    },
    {
      "drift/chosen_abs": 0.21974170207977295,
      "drift/rejected_abs": 0.06504969298839569,
      "epoch": 0.16686531585220502,
      "grad_norm": 7.75,
      "learning_rate": 4.866549494683838e-08,
      "logits/chosen": -1.9057044982910156,
      "logits/rejected": -2.0494656562805176,
      "logps/chosen": -0.31208372116088867,
      "logps/rejected": -0.3202230632305145,
      "loss": 0.267322301864624,
      "loss/core": 0.25114768743515015,
      "loss/preference_sft": 0.3229205012321472,
      "loss/reference_anchor": 0.18336977064609528,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.1963610649108887,
      "rewards/margins": 1.5482251644134521,
      "rewards/rejected": 0.6481360197067261,
      "ronpo/logit": 0.15482252836227417,
      "ronpo/residual": 0.16023744642734528,
      "ronpo/residual_abs": 0.17627668380737305,
      "ronpo/target": -0.005414927843958139,
      "ronpo/target_abs": 0.04029358923435211,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 525
    },
    {
      "drift/chosen_abs": 0.15339070558547974,
      "drift/rejected_abs": 0.06500252336263657,
      "epoch": 0.1684545093365117,
      "grad_norm": 1.15625,
      "learning_rate": 4.763954985423904e-08,
      "logits/chosen": -2.323418378829956,
      "logits/rejected": -2.318234920501709,
      "logps/chosen": -0.2666859030723572,
      "logps/rejected": -0.2610042989253998,
      "loss": 0.056935809552669525,
      "loss/core": 0.05055410414934158,
      "loss/preference_sft": 0.2673361599445343,
      "loss/reference_anchor": 0.0583559088408947,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.533581256866455,
      "rewards/margins": 0.8862428665161133,
      "rewards/rejected": 0.6473384499549866,
      "ronpo/logit": 0.08862429112195969,
      "ronpo/residual": 0.09020157158374786,
      "ronpo/residual_abs": 0.1063581109046936,
      "ronpo/target": -0.001577280811034143,
      "ronpo/target_abs": 0.03666796162724495,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 530
    },
    {
      "drift/chosen_abs": 0.3485838770866394,
      "drift/rejected_abs": 0.14534269273281097,
      "epoch": 0.17004370282081843,
      "grad_norm": 2.875,
      "learning_rate": 4.6614599929764626e-08,
      "logits/chosen": -2.2419562339782715,
      "logits/rejected": -2.2564449310302734,
      "logps/chosen": -0.3141189515590668,
      "logps/rejected": -0.32258644700050354,
      "loss": 1.5565026998519897,
      "loss/core": 1.473420262336731,
      "loss/preference_sft": 0.31737521290779114,
      "loss/reference_anchor": 1.0760281085968018,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.48555326461792,
      "rewards/margins": 2.219147205352783,
      "rewards/rejected": 1.2664062976837158,
      "ronpo/logit": 0.22191467881202698,
      "ronpo/residual": 0.2292250096797943,
      "ronpo/residual_abs": 0.3092333674430847,
      "ronpo/target": -0.007310344371944666,
      "ronpo/target_abs": 0.04150359705090523,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 535
    },
    {
      "drift/chosen_abs": 0.24426977336406708,
      "drift/rejected_abs": 0.10654947906732559,
      "epoch": 0.17163289630512515,
      "grad_norm": 8.375,
      "learning_rate": 4.5591077293320914e-08,
      "logits/chosen": -2.2026126384735107,
      "logits/rejected": -2.239305019378662,
      "logps/chosen": -0.2799997925758362,
      "logps/rejected": -0.2913915812969208,
      "loss": 0.22537103295326233,
      "loss/core": 0.19563640654087067,
      "loss/preference_sft": 0.28852710127830505,
      "loss/reference_anchor": 0.36760905385017395,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.4418580532073975,
      "rewards/margins": 1.3771570920944214,
      "rewards/rejected": 1.0647010803222656,
      "ronpo/logit": 0.13771569728851318,
      "ronpo/residual": 0.1375226229429245,
      "ronpo/residual_abs": 0.1619495302438736,
      "ronpo/target": 0.00019309911294840276,
      "ronpo/target_abs": 0.03357410058379173,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 540
    },
    {
      "drift/chosen_abs": 0.39887771010398865,
      "drift/rejected_abs": 0.2444618195295334,
      "epoch": 0.17322208978943188,
      "grad_norm": 5.25,
      "learning_rate": 4.45694134630676e-08,
      "logits/chosen": -1.9924676418304443,
      "logits/rejected": -2.0167553424835205,
      "logps/chosen": -0.2603081166744232,
      "logps/rejected": -0.2806777060031891,
      "loss": 0.48962488770484924,
      "loss/core": 0.42624610662460327,
      "loss/preference_sft": 0.27903109788894653,
      "loss/reference_anchor": 0.8171467781066895,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.987790584564209,
      "rewards/margins": 1.5475997924804688,
      "rewards/rejected": 2.4401907920837402,
      "ronpo/logit": 0.1547599732875824,
      "ronpo/residual": 0.15789082646369934,
      "ronpo/residual_abs": 0.23713842034339905,
      "ronpo/target": -0.003130866913124919,
      "ronpo/target_abs": 0.04018576815724373,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 545
    },
    {
      "drift/chosen_abs": 0.2760270833969116,
      "drift/rejected_abs": 0.1387709677219391,
      "epoch": 0.17481128327373857,
      "grad_norm": 1.25,
      "learning_rate": 4.355003917348984e-08,
      "logits/chosen": -2.1436705589294434,
      "logits/rejected": -2.1003220081329346,
      "logps/chosen": -0.25462886691093445,
      "logps/rejected": -0.2643481194972992,
      "loss": 0.3101690411567688,
      "loss/core": 0.28291812539100647,
      "loss/preference_sft": 0.26238495111465454,
      "loss/reference_anchor": 0.3371073007583618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.759854793548584,
      "rewards/margins": 1.3732728958129883,
      "rewards/rejected": 1.3865820169448853,
      "ronpo/logit": 0.13732729852199554,
      "ronpo/residual": 0.1491035521030426,
      "ronpo/residual_abs": 0.17806217074394226,
      "ronpo/target": -0.011776268482208252,
      "ronpo/target_abs": 0.04159698635339737,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 550
    },
    {
      "drift/chosen_abs": 0.12518154084682465,
      "drift/rejected_abs": 0.05519000440835953,
      "epoch": 0.1764004767580453,
      "grad_norm": 0.412109375,
      "learning_rate": 4.253338419380016e-08,
      "logits/chosen": -2.2161974906921387,
      "logits/rejected": -2.2592287063598633,
      "logps/chosen": -0.3115261197090149,
      "logps/rejected": -0.3088235855102539,
      "loss": 0.06282468140125275,
      "loss/core": 0.05494839698076248,
      "loss/preference_sft": 0.30944591760635376,
      "loss/reference_anchor": 0.07407257705926895,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2518155574798584,
      "rewards/margins": 0.7002865076065063,
      "rewards/rejected": 0.551529049873352,
      "ronpo/logit": 0.0700286477804184,
      "ronpo/residual": 0.0636088103055954,
      "ronpo/residual_abs": 0.09568959474563599,
      "ronpo/target": 0.006419846322387457,
      "ronpo/target_abs": 0.04170956462621689,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 555
    },
    {
      "drift/chosen_abs": 0.2650905251502991,
      "drift/rejected_abs": 0.1225300058722496,
      "epoch": 0.17798967024235202,
      "grad_norm": 17.0,
      "learning_rate": 4.151987714674736e-08,
      "logits/chosen": -2.084272861480713,
      "logits/rejected": -2.0913736820220947,
      "logps/chosen": -0.2723826766014099,
      "logps/rejected": -0.27883923053741455,
      "loss": 0.2584993243217468,
      "loss/core": 0.23772597312927246,
      "loss/preference_sft": 0.27814561128616333,
      "loss/reference_anchor": 0.24916362762451172,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.6494510173797607,
      "rewards/margins": 1.4253307580947876,
      "rewards/rejected": 1.224120020866394,
      "ronpo/logit": 0.1425330638885498,
      "ronpo/residual": 0.14170929789543152,
      "ronpo/residual_abs": 0.15773284435272217,
      "ronpo/target": 0.0008237810689024627,
      "ronpo/target_abs": 0.032286323606967926,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 560
    },
    {
      "drift/chosen_abs": 0.2599169611930847,
      "drift/rejected_abs": 0.11472652852535248,
      "epoch": 0.1795788637266587,
      "grad_norm": 29.875,
      "learning_rate": 4.0509945327908704e-08,
      "logits/chosen": -2.0401737689971924,
      "logits/rejected": -2.0661861896514893,
      "logps/chosen": -0.34084010124206543,
      "logps/rejected": -0.3358941674232483,
      "loss": 0.36931371688842773,
      "loss/core": 0.33606022596359253,
      "loss/preference_sft": 0.34215277433395386,
      "loss/reference_anchor": 0.40916499495506287,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.597136974334717,
      "rewards/margins": 1.4537303447723389,
      "rewards/rejected": 1.1434067487716675,
      "ronpo/logit": 0.14537303149700165,
      "ronpo/residual": 0.14311483502388,
      "ronpo/residual_abs": 0.16634781658649445,
      "ronpo/target": 0.0022582041565328836,
      "ronpo/target_abs": 0.036671485751867294,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 565
    },
    {
      "drift/chosen_abs": 0.4108946919441223,
      "drift/rejected_abs": 0.10000451654195786,
      "epoch": 0.18116805721096543,
      "grad_norm": 12.375,
      "learning_rate": 3.950401452554171e-08,
      "logits/chosen": -2.1480000019073486,
      "logits/rejected": -2.1182470321655273,
      "logps/chosen": -0.2765151560306549,
      "logps/rejected": -0.2645667493343353,
      "loss": 1.3812847137451172,
      "loss/core": 1.3190586566925049,
      "loss/preference_sft": 0.27181077003479004,
      "loss/reference_anchor": 0.8025016784667969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.108946800231934,
      "rewards/margins": 3.11163330078125,
      "rewards/rejected": 0.9973136186599731,
      "ronpo/logit": 0.3111633360385895,
      "ronpo/residual": 0.30616316199302673,
      "ronpo/residual_abs": 0.3383898138999939,
      "ronpo/target": 0.005000181496143341,
      "ronpo/target_abs": 0.046630989760160446,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 570
    },
    {
      "drift/chosen_abs": 0.19218239188194275,
      "drift/rejected_abs": 0.09658966213464737,
      "epoch": 0.18275725069527216,
      "grad_norm": 3.421875,
      "learning_rate": 3.850250884107133e-08,
      "logits/chosen": -2.0075910091400146,
      "logits/rejected": -2.094909429550171,
      "logps/chosen": -0.28321409225463867,
      "logps/rejected": -0.27906012535095215,
      "loss": 0.05575397610664368,
      "loss/core": 0.04724646359682083,
      "loss/preference_sft": 0.2887270748615265,
      "loss/reference_anchor": 0.0845608115196228,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9186960458755493,
      "rewards/margins": 0.9540292620658875,
      "rewards/rejected": 0.9646669626235962,
      "ronpo/logit": 0.09540291875600815,
      "ronpo/residual": 0.0999816432595253,
      "ronpo/residual_abs": 0.1167878657579422,
      "ronpo/target": -0.004578724503517151,
      "ronpo/target_abs": 0.03717956319451332,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 575
    },
    {
      "drift/chosen_abs": 0.36307868361473083,
      "drift/rejected_abs": 0.16689428687095642,
      "epoch": 0.18434644417957885,
      "grad_norm": 2.125,
      "learning_rate": 3.750585051028845e-08,
      "logits/chosen": -2.0031397342681885,
      "logits/rejected": -2.033116102218628,
      "logps/chosen": -0.29011231660842896,
      "logps/rejected": -0.2679058611392975,
      "loss": 0.5739777684211731,
      "loss/core": 0.5245698690414429,
      "loss/preference_sft": 0.2851426303386688,
      "loss/reference_anchor": 0.630257248878479,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.629528045654297,
      "rewards/margins": 1.9614536762237549,
      "rewards/rejected": 1.6680740118026733,
      "ronpo/logit": 0.19614538550376892,
      "ronpo/residual": 0.19045701622962952,
      "ronpo/residual_abs": 0.23201608657836914,
      "ronpo/target": 0.005688379053026438,
      "ronpo/target_abs": 0.03336022049188614,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 580
    },
    {
      "drift/chosen_abs": 0.23189207911491394,
      "drift/rejected_abs": 0.09468026459217072,
      "epoch": 0.18593563766388557,
      "grad_norm": 1.671875,
      "learning_rate": 3.651445972533487e-08,
      "logits/chosen": -1.9576046466827393,
      "logits/rejected": -1.9823410511016846,
      "logps/chosen": -0.2863709330558777,
      "logps/rejected": -0.2869415879249573,
      "loss": 0.1575651466846466,
      "loss/core": 0.14273057878017426,
      "loss/preference_sft": 0.295357882976532,
      "loss/reference_anchor": 0.1682581901550293,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3186957836151123,
      "rewards/margins": 1.3729966878890991,
      "rewards/rejected": 0.9456988573074341,
      "ronpo/logit": 0.13729968667030334,
      "ronpo/residual": 0.1349247694015503,
      "ronpo/residual_abs": 0.15127669274806976,
      "ronpo/target": 0.0023749221581965685,
      "ronpo/target_abs": 0.03486529737710953,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 585
    },
    {
      "drift/chosen_abs": 0.2609372138977051,
      "drift/rejected_abs": 0.12208591401576996,
      "epoch": 0.1875248311481923,
      "grad_norm": 1.328125,
      "learning_rate": 3.552875445754975e-08,
      "logits/chosen": -2.116716146469116,
      "logits/rejected": -2.0834579467773438,
      "logps/chosen": -0.3057488203048706,
      "logps/rejected": -0.3151412904262543,
      "loss": 0.6112656593322754,
      "loss/core": 0.5733734965324402,
      "loss/preference_sft": 0.31387200951576233,
      "loss/reference_anchor": 0.47384127974510193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.6066231727600098,
      "rewards/margins": 1.3866499662399292,
      "rewards/rejected": 1.219973087310791,
      "ronpo/logit": 0.13866500556468964,
      "ronpo/residual": 0.14339695870876312,
      "ronpo/residual_abs": 0.17332182824611664,
      "ronpo/target": -0.004731960128992796,
      "ronpo/target_abs": 0.03508470207452774,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 590
    },
    {
      "drift/chosen_abs": 0.23165372014045715,
      "drift/rejected_abs": 0.11231007426977158,
      "epoch": 0.18911402463249902,
      "grad_norm": 5.34375,
      "learning_rate": 3.4549150281252633e-08,
      "logits/chosen": -1.9744529724121094,
      "logits/rejected": -1.9331066608428955,
      "logps/chosen": -0.2898443341255188,
      "logps/rejected": -0.29342323541641235,
      "loss": 0.20584657788276672,
      "loss/core": 0.18938076496124268,
      "loss/preference_sft": 0.2938799262046814,
      "loss/reference_anchor": 0.1901559680700302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.314818859100342,
      "rewards/margins": 1.1970219612121582,
      "rewards/rejected": 1.1177968978881836,
      "ronpo/logit": 0.11970218271017075,
      "ronpo/residual": 0.1114286333322525,
      "ronpo/residual_abs": 0.1585887372493744,
      "ronpo/target": 0.008273562416434288,
      "ronpo/target_abs": 0.04056030511856079,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 595
    },
    {
      "drift/chosen_abs": 0.2051103115081787,
      "drift/rejected_abs": 0.1440320909023285,
      "epoch": 0.1907032181168057,
      "grad_norm": 145.0,
      "learning_rate": 3.3576060198536815e-08,
      "logits/chosen": -2.132131576538086,
      "logits/rejected": -2.1050548553466797,
      "logps/chosen": -0.29798927903175354,
      "logps/rejected": -0.318352609872818,
      "loss": 0.4632243514060974,
      "loss/core": 0.43989643454551697,
      "loss/preference_sft": 0.3055892586708069,
      "loss/reference_anchor": 0.28048065304756165,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.049271821975708,
      "rewards/margins": 0.6105486154556274,
      "rewards/rejected": 1.4387229681015015,
      "ronpo/logit": 0.06105484813451767,
      "ronpo/residual": 0.062055326998233795,
      "ronpo/residual_abs": 0.2071889191865921,
      "ronpo/target": -0.0010004760697484016,
      "ronpo/target_abs": 0.03393607214093208,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 600
    },
    {
      "drift/chosen_abs": 0.34726980328559875,
      "drift/rejected_abs": 0.15757516026496887,
      "epoch": 0.19229241160111243,
      "grad_norm": 1.3515625,
      "learning_rate": 3.260989446514726e-08,
      "logits/chosen": -2.0277671813964844,
      "logits/rejected": -1.95284104347229,
      "logps/chosen": -0.28237101435661316,
      "logps/rejected": -0.2795073688030243,
      "loss": 0.4946117401123047,
      "loss/core": 0.4488951563835144,
      "loss/preference_sft": 0.2740669250488281,
      "loss/reference_anchor": 0.5821473598480225,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.4638190269470215,
      "rewards/margins": 1.898168921470642,
      "rewards/rejected": 1.565650224685669,
      "ronpo/logit": 0.1898169070482254,
      "ronpo/residual": 0.18911583721637726,
      "ronpo/residual_abs": 0.21403014659881592,
      "ronpo/target": 0.0007010673289187253,
      "ronpo/target_abs": 0.0408640094101429,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 605
    },
    {
      "drift/chosen_abs": 0.3361540138721466,
      "drift/rejected_abs": 0.1532098352909088,
      "epoch": 0.19388160508541916,
      "grad_norm": 93.0,
      "learning_rate": 3.165106041751639e-08,
      "logits/chosen": -2.0205764770507812,
      "logits/rejected": -1.9713436365127563,
      "logps/chosen": -0.30869030952453613,
      "logps/rejected": -0.2928310036659241,
      "loss": 0.36041101813316345,
      "loss/core": 0.3217954933643341,
      "loss/preference_sft": 0.314534455537796,
      "loss/reference_anchor": 0.4834206998348236,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.3594353199005127,
      "rewards/margins": 1.8287744522094727,
      "rewards/rejected": 1.5306605100631714,
      "ronpo/logit": 0.18287745118141174,
      "ronpo/residual": 0.18577490746974945,
      "ronpo/residual_abs": 0.20601661503314972,
      "ronpo/target": -0.0028974597807973623,
      "ronpo/target_abs": 0.04099640995264053,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 610
    },
    {
      "drift/chosen_abs": 0.2551515996456146,
      "drift/rejected_abs": 0.11366691440343857,
      "epoch": 0.19547079856972585,
      "grad_norm": 128.0,
      "learning_rate": 3.069996230103066e-08,
      "logits/chosen": -2.1977291107177734,
      "logits/rejected": -2.159012794494629,
      "logps/chosen": -0.2876770496368408,
      "logps/rejected": -0.28747281432151794,
      "loss": 0.3087400197982788,
      "loss/core": 0.2813792824745178,
      "loss/preference_sft": 0.2934519648551941,
      "loss/reference_anchor": 0.3354647755622864,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.551334857940674,
      "rewards/margins": 1.4261997938156128,
      "rewards/rejected": 1.1251349449157715,
      "ronpo/logit": 0.1426199972629547,
      "ronpo/residual": 0.14335501194000244,
      "ronpo/residual_abs": 0.15863969922065735,
      "ronpo/target": -0.0007350404048338532,
      "ronpo/target_abs": 0.03268890082836151,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 615
    },
    {
      "drift/chosen_abs": 0.22230342030525208,
      "drift/rejected_abs": 0.10894560813903809,
      "epoch": 0.19705999205403257,
      "grad_norm": 12.75,
      "learning_rate": 2.9757001099600225e-08,
      "logits/chosen": -2.0835347175598145,
      "logits/rejected": -2.018907070159912,
      "logps/chosen": -0.29156553745269775,
      "logps/rejected": -0.28759321570396423,
      "loss": 0.10223954916000366,
      "loss/core": 0.08924255520105362,
      "loss/preference_sft": 0.2917032241821289,
      "loss/reference_anchor": 0.1441229283809662,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2221884727478027,
      "rewards/margins": 1.1332981586456299,
      "rewards/rejected": 1.0888900756835938,
      "ronpo/logit": 0.11332981288433075,
      "ronpo/residual": 0.11472058296203613,
      "ronpo/residual_abs": 0.13595080375671387,
      "ronpo/target": -0.0013907629763707519,
      "ronpo/target_abs": 0.03300943225622177,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 620
    },
    {
      "drift/chosen_abs": 0.2458353340625763,
      "drift/rejected_abs": 0.09283015877008438,
      "epoch": 0.1986491855383393,
      "grad_norm": 44.5,
      "learning_rate": 2.8822574366603803e-08,
      "logits/chosen": -2.0361926555633545,
      "logits/rejected": -2.0415730476379395,
      "logps/chosen": -0.2964986562728882,
      "logps/rejected": -0.298392653465271,
      "loss": 0.33931902050971985,
      "loss/core": 0.3200526833534241,
      "loss/preference_sft": 0.298342764377594,
      "loss/reference_anchor": 0.22704991698265076,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4581804275512695,
      "rewards/margins": 1.5353305339813232,
      "rewards/rejected": 0.9228495359420776,
      "ronpo/logit": 0.15353310108184814,
      "ronpo/residual": 0.1616130769252777,
      "ronpo/residual_abs": 0.1806202232837677,
      "ronpo/target": -0.008080018684267998,
      "ronpo/target_abs": 0.03993826359510422,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 625
    },
    {
      "drift/chosen_abs": 0.23397347331047058,
      "drift/rejected_abs": 0.08942471444606781,
      "epoch": 0.20023837902264602,
      "grad_norm": 5.6875,
      "learning_rate": 2.78970760572797e-08,
      "logits/chosen": -2.3254988193511963,
      "logits/rejected": -2.4149765968322754,
      "logps/chosen": -0.2674204409122467,
      "logps/rejected": -0.26369109749794006,
      "loss": 0.31197839975357056,
      "loss/core": 0.2868371903896332,
      "loss/preference_sft": 0.266465961933136,
      "loss/reference_anchor": 0.30856937170028687,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.3387513160705566,
      "rewards/margins": 1.4476630687713623,
      "rewards/rejected": 0.8910881876945496,
      "ronpo/logit": 0.14476630091667175,
      "ronpo/residual": 0.13926377892494202,
      "ronpo/residual_abs": 0.16467837989330292,
      "ronpo/target": 0.005502545274794102,
      "ronpo/target_abs": 0.03496618941426277,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 630
    },
    {
      "drift/chosen_abs": 0.2901550531387329,
      "drift/rejected_abs": 0.10847359895706177,
      "epoch": 0.2018275725069527,
      "grad_norm": 16.625,
      "learning_rate": 2.6980896362634047e-08,
      "logits/chosen": -2.1083121299743652,
      "logits/rejected": -2.1480813026428223,
      "logps/chosen": -0.26491206884384155,
      "logps/rejected": -0.2721540033817291,
      "loss": 0.3880026042461395,
      "loss/core": 0.36257725954055786,
      "loss/preference_sft": 0.26688578724861145,
      "loss/reference_anchor": 0.3123162090778351,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.9005064964294434,
      "rewards/margins": 1.8258981704711914,
      "rewards/rejected": 1.074608325958252,
      "ronpo/logit": 0.1825898140668869,
      "ronpo/residual": 0.17556194961071014,
      "ronpo/residual_abs": 0.20522348582744598,
      "ronpo/target": 0.007027854211628437,
      "ronpo/target_abs": 0.03924590349197388,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 635
    },
    {
      "drift/chosen_abs": 0.3232152760028839,
      "drift/rejected_abs": 0.09826619923114777,
      "epoch": 0.20341676599125944,
      "grad_norm": 1.40625,
      "learning_rate": 2.6074421544935676e-08,
      "logits/chosen": -2.015664577484131,
      "logits/rejected": -2.0078439712524414,
      "logps/chosen": -0.31468555331230164,
      "logps/rejected": -0.3170844614505768,
      "loss": 0.7409719824790955,
      "loss/core": 0.6974954605102539,
      "loss/preference_sft": 0.31186458468437195,
      "loss/reference_anchor": 0.5485001802444458,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2312824726104736,
      "rewards/margins": 2.2840328216552734,
      "rewards/rejected": 0.9472495913505554,
      "ronpo/logit": 0.2284032553434372,
      "ronpo/residual": 0.2323722094297409,
      "ronpo/residual_abs": 0.2483997792005539,
      "ronpo/target": -0.003968966659158468,
      "ronpo/target_abs": 0.038829077035188675,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 640
    },
    {
      "drift/chosen_abs": 0.2661309540271759,
      "drift/rejected_abs": 0.11417138576507568,
      "epoch": 0.20500595947556616,
      "grad_norm": 8.1875,
      "learning_rate": 2.5178033774867692e-08,
      "logits/chosen": -2.1209521293640137,
      "logits/rejected": -2.1270642280578613,
      "logps/chosen": -0.30802121758461,
      "logps/rejected": -0.2950921952724457,
      "loss": 0.33442920446395874,
      "loss/core": 0.30818670988082886,
      "loss/preference_sft": 0.3144807517528534,
      "loss/reference_anchor": 0.31845149397850037,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.659245014190674,
      "rewards/margins": 1.5209836959838867,
      "rewards/rejected": 1.1382615566253662,
      "ronpo/logit": 0.1520983725786209,
      "ronpo/residual": 0.14826013147830963,
      "ronpo/residual_abs": 0.1766631305217743,
      "ronpo/target": 0.0038382462225854397,
      "ronpo/target_abs": 0.04023699089884758,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 645
    },
    {
      "drift/chosen_abs": 0.1514950692653656,
      "drift/rejected_abs": 0.0910639688372612,
      "epoch": 0.20659515295987285,
      "grad_norm": 1.4765625,
      "learning_rate": 2.4292110970403884e-08,
      "logits/chosen": -2.2066166400909424,
      "logits/rejected": -2.1343297958374023,
      "logps/chosen": -0.3011590242385864,
      "logps/rejected": -0.29367610812187195,
      "loss": 0.05876779556274414,
      "loss/core": 0.05132526159286499,
      "loss/preference_sft": 0.3029544949531555,
      "loss/reference_anchor": 0.06893852353096008,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.5139214992523193,
      "rewards/margins": 0.6618244051933289,
      "rewards/rejected": 0.85209721326828,
      "ronpo/logit": 0.066182442009449,
      "ronpo/residual": 0.06901034712791443,
      "ronpo/residual_abs": 0.09437278658151627,
      "ronpo/target": -0.002827908843755722,
      "ronpo/target_abs": 0.037444792687892914,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 650
    },
    {
      "drift/chosen_abs": 0.20499472320079803,
      "drift/rejected_abs": 0.064644455909729,
      "epoch": 0.20818434644417957,
      "grad_norm": 127.0,
      "learning_rate": 2.3417026637478186e-08,
      "logits/chosen": -2.277536392211914,
      "logits/rejected": -2.283423900604248,
      "logps/chosen": -0.2831169366836548,
      "logps/rejected": -0.2986695170402527,
      "loss": 0.38664403557777405,
      "loss/core": 0.367914617061615,
      "loss/preference_sft": 0.29018551111221313,
      "loss/reference_anchor": 0.2207069844007492,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.0499470233917236,
      "rewards/margins": 1.4070502519607544,
      "rewards/rejected": 0.6428970098495483,
      "ronpo/logit": 0.14070503413677216,
      "ronpo/residual": 0.1532699167728424,
      "ronpo/residual_abs": 0.1704498678445816,
      "ronpo/target": -0.012564892880618572,
      "ronpo/target_abs": 0.04341302812099457,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 655
    },
    {
      "drift/chosen_abs": 0.17548860609531403,
      "drift/rejected_abs": 0.07433202862739563,
      "epoch": 0.2097735399284863,
      "grad_norm": 6.71875,
      "learning_rate": 2.255314971251419e-08,
      "logits/chosen": -2.251873731613159,
      "logits/rejected": -2.295532703399658,
      "logps/chosen": -0.29541832208633423,
      "logps/rejected": -0.2891073524951935,
      "loss": 0.1282232701778412,
      "loss/core": 0.11798721551895142,
      "loss/preference_sft": 0.2941397428512573,
      "loss/reference_anchor": 0.10706689208745956,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.754166841506958,
      "rewards/margins": 1.0147831439971924,
      "rewards/rejected": 0.7393836975097656,
      "ronpo/logit": 0.10147831588983536,
      "ronpo/residual": 0.10077782720327377,
      "ronpo/residual_abs": 0.1172768622636795,
      "ronpo/target": 0.0007004888029769063,
      "ronpo/target_abs": 0.03470372408628464,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 660
    },
    {
      "drift/chosen_abs": 0.2351798266172409,
      "drift/rejected_abs": 0.1356552243232727,
      "epoch": 0.21136273341279302,
      "grad_norm": 0.65625,
      "learning_rate": 2.1700844406881107e-08,
      "logits/chosen": -2.0752432346343994,
      "logits/rejected": -2.019031524658203,
      "logps/chosen": -0.28208786249160767,
      "logps/rejected": -0.28667864203453064,
      "loss": 0.23971502482891083,
      "loss/core": 0.21660105884075165,
      "loss/preference_sft": 0.28722888231277466,
      "loss/reference_anchor": 0.27946338057518005,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.348977565765381,
      "rewards/margins": 0.9931402206420898,
      "rewards/rejected": 1.3558372259140015,
      "ronpo/logit": 0.09931402653455734,
      "ronpo/residual": 0.10590650886297226,
      "ronpo/residual_abs": 0.13825678825378418,
      "ronpo/target": -0.006592473480850458,
      "ronpo/target_abs": 0.037365201860666275,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 665
    },
    {
      "drift/chosen_abs": 0.20990140736103058,
      "drift/rejected_abs": 0.07856923341751099,
      "epoch": 0.2129519268970997,
      "grad_norm": 5.5,
      "learning_rate": 2.0860470053341957e-08,
      "logits/chosen": -2.3458199501037598,
      "logits/rejected": -2.3533999919891357,
      "logps/chosen": -0.2676357328891754,
      "logps/rejected": -0.27186718583106995,
      "loss": 0.21298956871032715,
      "loss/core": 0.19874659180641174,
      "loss/preference_sft": 0.2663695812225342,
      "loss/reference_anchor": 0.16326895356178284,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.097968101501465,
      "rewards/margins": 1.3122758865356445,
      "rewards/rejected": 0.7856923341751099,
      "ronpo/logit": 0.13122756779193878,
      "ronpo/residual": 0.1469726860523224,
      "ronpo/residual_abs": 0.15434160828590393,
      "ronpo/target": -0.01574511080980301,
      "ronpo/target_abs": 0.03544539958238602,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 670
    },
    {
      "drift/chosen_abs": 0.4016236364841461,
      "drift/rejected_abs": 0.07960326224565506,
      "epoch": 0.21454112038140644,
      "grad_norm": 2.234375,
      "learning_rate": 2.0032380954558548e-08,
      "logits/chosen": -1.986501693725586,
      "logits/rejected": -2.0995402336120605,
      "logps/chosen": -0.27848881483078003,
      "logps/rejected": -0.271128386259079,
      "loss": 1.4639503955841064,
      "loss/core": 1.401967167854309,
      "loss/preference_sft": 0.2735333740711212,
      "loss/reference_anchor": 0.7990926504135132,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 4.014946460723877,
      "rewards/margins": 3.2203209400177,
      "rewards/rejected": 0.7946258783340454,
      "ronpo/logit": 0.32203203439712524,
      "ronpo/residual": 0.31797724962234497,
      "ronpo/residual_abs": 0.3429466485977173,
      "ronpo/target": 0.0040548210963606834,
      "ronpo/target_abs": 0.04477760195732117,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 675
    },
    {
      "drift/chosen_abs": 0.1950986236333847,
      "drift/rejected_abs": 0.1172579675912857,
      "epoch": 0.21613031386571316,
      "grad_norm": 2.203125,
      "learning_rate": 1.9216926233717085e-08,
      "logits/chosen": -1.9547698497772217,
      "logits/rejected": -1.949459433555603,
      "logps/chosen": -0.28763753175735474,
      "logps/rejected": -0.2842276394367218,
      "loss": 0.06276907026767731,
      "loss/core": 0.04688558354973793,
      "loss/preference_sft": 0.2863856256008148,
      "loss/reference_anchor": 0.18314120173454285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9503726959228516,
      "rewards/margins": 0.7782739400863647,
      "rewards/rejected": 1.172098994255066,
      "ronpo/logit": 0.07782740890979767,
      "ronpo/residual": 0.07493593543767929,
      "ronpo/residual_abs": 0.10520467907190323,
      "ronpo/target": 0.0028914690483361483,
      "ronpo/target_abs": 0.042150795459747314,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 680
    },
    {
      "drift/chosen_abs": 0.1819314956665039,
      "drift/rejected_abs": 0.09456787258386612,
      "epoch": 0.21771950735001985,
      "grad_norm": 3.0,
      "learning_rate": 1.8414449687337464e-08,
      "logits/chosen": -1.972158432006836,
      "logits/rejected": -2.0605552196502686,
      "logps/chosen": -0.2929169535636902,
      "logps/rejected": -0.2997816503047943,
      "loss": 0.052760761231184006,
      "loss/core": 0.044637665152549744,
      "loss/preference_sft": 0.3041604459285736,
      "loss/reference_anchor": 0.07789192348718643,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8181995153427124,
      "rewards/margins": 0.875436007976532,
      "rewards/rejected": 0.9427635073661804,
      "ronpo/logit": 0.08754359930753708,
      "ronpo/residual": 0.08810020983219147,
      "ronpo/residual_abs": 0.10490294545888901,
      "ronpo/target": -0.0005566048203036189,
      "ronpo/target_abs": 0.038404613733291626,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 685
    },
    {
      "drift/chosen_abs": 0.45035824179649353,
      "drift/rejected_abs": 0.18560758233070374,
      "epoch": 0.21930870083432658,
      "grad_norm": 440.0,
      "learning_rate": 1.7625289640328318e-08,
      "logits/chosen": -1.9293177127838135,
      "logits/rejected": -1.9729528427124023,
      "logps/chosen": -0.2784627079963684,
      "logps/rejected": -0.27157917618751526,
      "loss": 0.891852080821991,
      "loss/core": 0.807598888874054,
      "loss/preference_sft": 0.2851676642894745,
      "loss/reference_anchor": 1.0948574542999268,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 4.503391265869141,
      "rewards/margins": 2.649190664291382,
      "rewards/rejected": 1.8542003631591797,
      "ronpo/logit": 0.26491910219192505,
      "ronpo/residual": 0.2634193003177643,
      "ronpo/residual_abs": 0.2890183627605438,
      "ronpo/target": 0.001499784062616527,
      "ronpo/target_abs": 0.0398874506354332,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 690
    },
    {
      "drift/chosen_abs": 0.1834675371646881,
      "drift/rejected_abs": 0.11008524894714355,
      "epoch": 0.2208978943186333,
      "grad_norm": 12.375,
      "learning_rate": 1.6849778803349007e-08,
      "logits/chosen": -2.160452127456665,
      "logits/rejected": -2.1469032764434814,
      "logps/chosen": -0.29817405343055725,
      "logps/rejected": -0.3005991578102112,
      "loss": 0.10696051269769669,
      "loss/core": 0.09560181200504303,
      "loss/preference_sft": 0.29798537492752075,
      "loss/reference_anchor": 0.12165065854787827,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8344100713729858,
      "rewards/margins": 0.7348702549934387,
      "rewards/rejected": 1.099540114402771,
      "ronpo/logit": 0.07348702102899551,
      "ronpo/residual": 0.06854233145713806,
      "ronpo/residual_abs": 0.13825374841690063,
      "ronpo/target": 0.004944689106196165,
      "ronpo/target_abs": 0.03506334498524666,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 695
    },
    {
      "drift/chosen_abs": 0.22251728177070618,
      "drift/rejected_abs": 0.139739990234375,
      "epoch": 0.22248708780294002,
      "grad_norm": 6.21875,
      "learning_rate": 1.60882441325383e-08,
      "logits/chosen": -2.216174364089966,
      "logits/rejected": -2.175872564315796,
      "logps/chosen": -0.3221741318702698,
      "logps/rejected": -0.3115740418434143,
      "loss": 0.43813127279281616,
      "loss/core": 0.4061037003993988,
      "loss/preference_sft": 0.3150758147239685,
      "loss/reference_anchor": 0.3955269455909729,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.2245700359344482,
      "rewards/margins": 0.8280360102653503,
      "rewards/rejected": 1.3965343236923218,
      "ronpo/logit": 0.08280360698699951,
      "ronpo/residual": 0.08249805122613907,
      "ronpo/residual_abs": 0.17446820437908173,
      "ronpo/target": 0.0003055560518987477,
      "ronpo/target_abs": 0.0334026925265789,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 700
    },
    {
      "drift/chosen_abs": 0.184719055891037,
      "drift/rejected_abs": 0.08408048748970032,
      "epoch": 0.22407628128724671,
      "grad_norm": 3.109375,
      "learning_rate": 1.5341006691669512e-08,
      "logits/chosen": -2.2611207962036133,
      "logits/rejected": -2.366459369659424,
      "logps/chosen": -0.2667378783226013,
      "logps/rejected": -0.2706339359283447,
      "loss": 0.07560595124959946,
      "loss/core": 0.06591415405273438,
      "loss/preference_sft": 0.27826228737831116,
      "loss/reference_anchor": 0.10139782726764679,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8469181060791016,
      "rewards/margins": 1.010367512702942,
      "rewards/rejected": 0.8365505933761597,
      "ronpo/logit": 0.10103676468133926,
      "ronpo/residual": 0.10294114053249359,
      "ronpo/residual_abs": 0.12307702004909515,
      "ronpo/target": -0.0019043807405978441,
      "ronpo/target_abs": 0.041786063462495804,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 705
    },
    {
      "drift/chosen_abs": 0.2891550660133362,
      "drift/rejected_abs": 0.10379669815301895,
      "epoch": 0.22566547477155344,
      "grad_norm": 101.0,
      "learning_rate": 1.4608381516789548e-08,
      "logits/chosen": -2.1673738956451416,
      "logits/rejected": -2.180238962173462,
      "logps/chosen": -0.29929423332214355,
      "logps/rejected": -0.2966969609260559,
      "loss": 0.7068883180618286,
      "loss/core": 0.6746903657913208,
      "loss/preference_sft": 0.29829683899879456,
      "loss/reference_anchor": 0.3994755446910858,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.891120433807373,
      "rewards/margins": 1.9227606058120728,
      "rewards/rejected": 0.9683598279953003,
      "ronpo/logit": 0.19227604568004608,
      "ronpo/residual": 0.1905229389667511,
      "ronpo/residual_abs": 0.2223322093486786,
      "ronpo/target": 0.0017531251069158316,
      "ronpo/target_abs": 0.03450788930058479,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 710
    },
    {
      "drift/chosen_abs": 0.1664462685585022,
      "drift/rejected_abs": 0.09171423316001892,
      "epoch": 0.22725466825586016,
      "grad_norm": 1.1796875,
      "learning_rate": 1.3890677483399537e-08,
      "logits/chosen": -2.047231912612915,
      "logits/rejected": -1.9673421382904053,
      "logps/chosen": -0.30263659358024597,
      "logps/rejected": -0.3006385564804077,
      "loss": 0.06795786321163177,
      "loss/core": 0.05951591208577156,
      "loss/preference_sft": 0.3053247034549713,
      "loss/reference_anchor": 0.08202681690454483,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6637260913848877,
      "rewards/margins": 0.747805118560791,
      "rewards/rejected": 0.9159210324287415,
      "ronpo/logit": 0.07478051632642746,
      "ronpo/residual": 0.06953813135623932,
      "ronpo/residual_abs": 0.11027568578720093,
      "ronpo/target": 0.0052423798479139805,
      "ronpo/target_abs": 0.0369865819811821,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 715
    },
    {
      "drift/chosen_abs": 0.3428126871585846,
      "drift/rejected_abs": 0.14502094686031342,
      "epoch": 0.22884386174016685,
      "grad_norm": 524.0,
      "learning_rate": 1.3188197176232486e-08,
      "logits/chosen": -1.9479389190673828,
      "logits/rejected": -2.027177095413208,
      "logps/chosen": -0.2941284775733948,
      "logps/rejected": -0.285388320684433,
      "loss": 0.583694577217102,
      "loss/core": 0.5399834513664246,
      "loss/preference_sft": 0.2799127697944641,
      "loss/reference_anchor": 0.5548235774040222,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.4279160499572754,
      "rewards/margins": 2.029214859008789,
      "rewards/rejected": 1.3987010717391968,
      "ronpo/logit": 0.20292147994041443,
      "ronpo/residual": 0.19507944583892822,
      "ronpo/residual_abs": 0.21936511993408203,
      "ronpo/target": 0.007842039689421654,
      "ronpo/target_abs": 0.037268415093421936,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 720
    },
    {
      "drift/chosen_abs": 0.2962958812713623,
      "drift/rejected_abs": 0.06851255893707275,
      "epoch": 0.23043305522447358,
      "grad_norm": 3.765625,
      "learning_rate": 1.250123676168332e-08,
      "logits/chosen": -2.0833048820495605,
      "logits/rejected": -2.0245532989501953,
      "logps/chosen": -0.28989580273628235,
      "logps/rejected": -0.2917415499687195,
      "loss": 1.0986053943634033,
      "loss/core": 1.0533392429351807,
      "loss/preference_sft": 0.2916197180747986,
      "loss/reference_anchor": 0.5743886828422546,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.961975574493408,
      "rewards/margins": 2.2812411785125732,
      "rewards/rejected": 0.6807340383529663,
      "ronpo/logit": 0.22812414169311523,
      "ronpo/residual": 0.21930313110351562,
      "ronpo/residual_abs": 0.2511158585548401,
      "ronpo/target": 0.008821023628115654,
      "ronpo/target_abs": 0.042353756725788116,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 725
    },
    {
      "drift/chosen_abs": 0.31698840856552124,
      "drift/rejected_abs": 0.09349516034126282,
      "epoch": 0.2320222487087803,
      "grad_norm": 2416.0,
      "learning_rate": 1.183008586294485e-08,
      "logits/chosen": -2.178093910217285,
      "logits/rejected": -2.21374249458313,
      "logps/chosen": -0.29720231890678406,
      "logps/rejected": -0.2978862226009369,
      "loss": 0.9873377084732056,
      "loss/core": 0.9382851719856262,
      "loss/preference_sft": 0.30639463663101196,
      "loss/reference_anchor": 0.6233946084976196,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.169236183166504,
      "rewards/margins": 2.236426830291748,
      "rewards/rejected": 0.9328093528747559,
      "ronpo/logit": 0.22364270687103271,
      "ronpo/residual": 0.21640853583812714,
      "ronpo/residual_abs": 0.24321117997169495,
      "ronpo/target": 0.0072341784834861755,
      "ronpo/target_abs": 0.039287909865379333,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 730
    },
    {
      "drift/chosen_abs": 0.4120539724826813,
      "drift/rejected_abs": 0.15305277705192566,
      "epoch": 0.23361144219308702,
      "grad_norm": 10.625,
      "learning_rate": 1.1175027437902328e-08,
      "logits/chosen": -1.8680089712142944,
      "logits/rejected": -1.9213451147079468,
      "logps/chosen": -0.31041333079338074,
      "logps/rejected": -0.282566100358963,
      "loss": 1.3381201028823853,
      "loss/core": 1.2714483737945557,
      "loss/preference_sft": 0.3085525631904602,
      "loss/reference_anchor": 0.8580993413925171,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.118987083435059,
      "rewards/margins": 2.5983529090881348,
      "rewards/rejected": 1.5206340551376343,
      "ronpo/logit": 0.25983530282974243,
      "ronpo/residual": 0.2567339241504669,
      "ronpo/residual_abs": 0.3305421769618988,
      "ronpo/target": 0.0031014089472591877,
      "ronpo/target_abs": 0.046786196529865265,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 735
    },
    {
      "drift/chosen_abs": 0.4067060947418213,
      "drift/rejected_abs": 0.1747940182685852,
      "epoch": 0.23520063567739372,
      "grad_norm": 6.90625,
      "learning_rate": 1.0536337659838091e-08,
      "logits/chosen": -2.0639398097991943,
      "logits/rejected": -2.054687976837158,
      "logps/chosen": -0.2756916880607605,
      "logps/rejected": -0.27250200510025024,
      "loss": 0.8732901811599731,
      "loss/core": 0.7945246696472168,
      "loss/preference_sft": 0.2825769782066345,
      "loss/reference_anchor": 1.0219491720199585,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 4.066915988922119,
      "rewards/margins": 2.3204641342163086,
      "rewards/rejected": 1.746451735496521,
      "ronpo/logit": 0.23204641044139862,
      "ronpo/residual": 0.2248552292585373,
      "ronpo/residual_abs": 0.265652596950531,
      "ronpo/target": 0.00719115138053894,
      "ronpo/target_abs": 0.04038388282060623,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 740
    },
    {
      "drift/chosen_abs": 0.279170423746109,
      "drift/rejected_abs": 0.14095792174339294,
      "epoch": 0.23678982916170044,
      "grad_norm": 1.8046875,
      "learning_rate": 9.914285800996669e-09,
      "logits/chosen": -1.965667486190796,
      "logits/rejected": -1.9407310485839844,
      "logps/chosen": -0.3105481266975403,
      "logps/rejected": -0.29601940512657166,
      "loss": 0.42376190423965454,
      "loss/core": 0.39586013555526733,
      "loss/preference_sft": 0.3031192421913147,
      "loss/reference_anchor": 0.34171098470687866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.7881577014923096,
      "rewards/margins": 1.4106953144073486,
      "rewards/rejected": 1.3774619102478027,
      "ronpo/logit": 0.14106956124305725,
      "ronpo/residual": 0.14506323635578156,
      "ronpo/residual_abs": 0.18387925624847412,
      "ronpo/target": -0.003993672784417868,
      "ronpo/target_abs": 0.03346020728349686,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 745
    },
    {
      "drift/chosen_abs": 0.4928775727748871,
      "drift/rejected_abs": 0.08850506693124771,
      "epoch": 0.23837902264600716,
      "grad_norm": 27.375,
      "learning_rate": 9.309134119059337e-09,
      "logits/chosen": -1.8723657131195068,
      "logits/rejected": -2.0245261192321777,
      "logps/chosen": -0.3265787661075592,
      "logps/rejected": -0.3039904534816742,
      "loss": 1.7299203872680664,
      "loss/core": 1.652858018875122,
      "loss/preference_sft": 0.308999240398407,
      "loss/reference_anchor": 0.9966007471084595,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.92842960357666,
      "rewards/margins": 4.047325134277344,
      "rewards/rejected": 0.8811044692993164,
      "ronpo/logit": 0.40473252534866333,
      "ronpo/residual": 0.40347546339035034,
      "ronpo/residual_abs": 0.4258064329624176,
      "ronpo/target": 0.0012570355320349336,
      "ronpo/target_abs": 0.04142013564705849,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 750
    },
    {
      "drift/chosen_abs": 0.5538117289543152,
      "drift/rejected_abs": 0.2532739043235779,
      "epoch": 0.23996821613031386,
      "grad_norm": 169.0,
      "learning_rate": 8.721137746576019e-09,
      "logits/chosen": -1.7783946990966797,
      "logits/rejected": -1.7093887329101562,
      "logps/chosen": -0.2701883912086487,
      "logps/rejected": -0.2783645987510681,
      "loss": 1.8371918201446533,
      "loss/core": 1.743137001991272,
      "loss/preference_sft": 0.2584080994129181,
      "loss/reference_anchor": 1.2282252311706543,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 5.537991046905518,
      "rewards/margins": 3.011918544769287,
      "rewards/rejected": 2.5260722637176514,
      "ronpo/logit": 0.30119186639785767,
      "ronpo/residual": 0.29457274079322815,
      "ronpo/residual_abs": 0.46518081426620483,
      "ronpo/target": 0.006619105581194162,
      "ronpo/target_abs": 0.036873988807201385,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 755
    },
    {
      "drift/chosen_abs": 0.2569332718849182,
      "drift/rejected_abs": 0.10224013030529022,
      "epoch": 0.24155740961462058,
      "grad_norm": 1.2421875,
      "learning_rate": 8.150544583401114e-09,
      "logits/chosen": -2.202519655227661,
      "logits/rejected": -2.2522640228271484,
      "logps/chosen": -0.2607022821903229,
      "logps/rejected": -0.26129353046417236,
      "loss": 0.41069111227989197,
      "loss/core": 0.38572731614112854,
      "loss/preference_sft": 0.2723042368888855,
      "loss/reference_anchor": 0.3056199550628662,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5683581829071045,
      "rewards/margins": 1.5499927997589111,
      "rewards/rejected": 1.0183653831481934,
      "ronpo/logit": 0.1549992859363556,
      "ronpo/residual": 0.1515064835548401,
      "ronpo/residual_abs": 0.17911091446876526,
      "ronpo/target": 0.0034928210079669952,
      "ronpo/target_abs": 0.0378664955496788,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 760
    },
    {
      "drift/chosen_abs": 0.16791877150535583,
      "drift/rejected_abs": 0.09147568047046661,
      "epoch": 0.2431466030989273,
      "grad_norm": 0.97265625,
      "learning_rate": 7.597595192178703e-09,
      "logits/chosen": -2.397317409515381,
      "logits/rejected": -2.3352389335632324,
      "logps/chosen": -0.28569984436035156,
      "logps/rejected": -0.28979137539863586,
      "loss": 0.044909216463565826,
      "loss/core": 0.03475845605134964,
      "loss/preference_sft": 0.28664320707321167,
      "loss/reference_anchor": 0.1066790446639061,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6786880493164062,
      "rewards/margins": 0.7643090486526489,
      "rewards/rejected": 0.9143790006637573,
      "ronpo/logit": 0.07643090933561325,
      "ronpo/residual": 0.08657925575971603,
      "ronpo/residual_abs": 0.10033098608255386,
      "ronpo/target": -0.010148352012038231,
      "ronpo/target_abs": 0.03190162032842636,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 765
    },
    {
      "drift/chosen_abs": 0.32959985733032227,
      "drift/rejected_abs": 0.10112398862838745,
      "epoch": 0.24473579658323402,
      "grad_norm": 55.5,
      "learning_rate": 7.062522696921108e-09,
      "logits/chosen": -1.964592695236206,
      "logits/rejected": -2.0949795246124268,
      "logps/chosen": -0.2564390003681183,
      "logps/rejected": -0.27129215002059937,
      "loss": 0.5951908826828003,
      "loss/core": 0.565562903881073,
      "loss/preference_sft": 0.26717469096183777,
      "loss/reference_anchor": 0.36832180619239807,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.2949090003967285,
      "rewards/margins": 2.3579647541046143,
      "rewards/rejected": 0.9369438290596008,
      "ronpo/logit": 0.2357964962720871,
      "ronpo/residual": 0.24124550819396973,
      "ronpo/residual_abs": 0.25939813256263733,
      "ronpo/target": -0.005449019372463226,
      "ronpo/target_abs": 0.04333696514368057,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 770
    },
    {
      "drift/chosen_abs": 0.3441210985183716,
      "drift/rejected_abs": 0.1433735191822052,
      "epoch": 0.24632499006754072,
      "grad_norm": 1.078125,
      "learning_rate": 6.545552684723582e-09,
      "logits/chosen": -1.8267816305160522,
      "logits/rejected": -1.8162485361099243,
      "logps/chosen": -0.2950648367404938,
      "logps/rejected": -0.2963963449001312,
      "loss": 0.5573657155036926,
      "loss/core": 0.5240105390548706,
      "loss/preference_sft": 0.29786160588264465,
      "loss/reference_anchor": 0.4149499833583832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.4393134117126465,
      "rewards/margins": 2.0451102256774902,
      "rewards/rejected": 1.3942031860351562,
      "ronpo/logit": 0.20451104640960693,
      "ronpo/residual": 0.2089034765958786,
      "ronpo/residual_abs": 0.2280438393354416,
      "ronpo/target": -0.004392450209707022,
      "ronpo/target_abs": 0.04173804819583893,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 775
    },
    {
      "drift/chosen_abs": 0.349498987197876,
      "drift/rejected_abs": 0.1552140712738037,
      "epoch": 0.24791418355184744,
      "grad_norm": 3.828125,
      "learning_rate": 6.046903110656504e-09,
      "logits/chosen": -2.1567647457122803,
      "logits/rejected": -2.180453062057495,
      "logps/chosen": -0.2706982493400574,
      "logps/rejected": -0.2793269753456116,
      "loss": 0.35834938287734985,
      "loss/core": 0.32353121042251587,
      "loss/preference_sft": 0.27363473176956177,
      "loss/reference_anchor": 0.436878502368927,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.4948108196258545,
      "rewards/margins": 1.943900465965271,
      "rewards/rejected": 1.5509105920791626,
      "ronpo/logit": 0.19439007341861725,
      "ronpo/residual": 0.19187742471694946,
      "ronpo/residual_abs": 0.2496909350156784,
      "ronpo/target": 0.0025126230902969837,
      "ronpo/target_abs": 0.04215142875909805,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 780
    },
    {
      "drift/chosen_abs": 0.47929421067237854,
      "drift/rejected_abs": 0.1080431342124939,
      "epoch": 0.24950337703615416,
      "grad_norm": 3.71875,
      "learning_rate": 5.566784205875363e-09,
      "logits/chosen": -2.1040244102478027,
      "logits/rejected": -2.2148659229278564,
      "logps/chosen": -0.26970845460891724,
      "logps/rejected": -0.2620154917240143,
      "loss": 1.8097639083862305,
      "loss/core": 1.7370777130126953,
      "loss/preference_sft": 0.26922696828842163,
      "loss/reference_anchor": 0.9422265887260437,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.785176753997803,
      "rewards/margins": 3.7156035900115967,
      "rewards/rejected": 1.0695734024047852,
      "ronpo/logit": 0.37156039476394653,
      "ronpo/residual": 0.3729218542575836,
      "ronpo/residual_abs": 0.39052265882492065,
      "ronpo/target": -0.0013614622876048088,
      "ronpo/target_abs": 0.04210304841399193,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 785
    },
    {
      "drift/chosen_abs": 0.35931769013404846,
      "drift/rejected_abs": 0.12135148048400879,
      "epoch": 0.25109257052046086,
      "grad_norm": 199.0,
      "learning_rate": 5.105398388987098e-09,
      "logits/chosen": -2.007981538772583,
      "logits/rejected": -2.027435302734375,
      "logps/chosen": -0.31240516901016235,
      "logps/rejected": -0.28974097967147827,
      "loss": 0.5043392181396484,
      "loss/core": 0.4675632119178772,
      "loss/preference_sft": 0.30789914727211,
      "loss/reference_anchor": 0.45955753326416016,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.5931477546691895,
      "rewards/margins": 2.408783435821533,
      "rewards/rejected": 1.1843643188476562,
      "ronpo/logit": 0.2408783733844757,
      "ronpo/residual": 0.24351993203163147,
      "ronpo/residual_abs": 0.27939122915267944,
      "ronpo/target": -0.002641576575115323,
      "ronpo/target_abs": 0.039345476776361465,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 790
    },
    {
      "drift/chosen_abs": 0.24088767170906067,
      "drift/rejected_abs": 0.1129360944032669,
      "epoch": 0.2526817640047676,
      "grad_norm": 1.171875,
      "learning_rate": 4.662940180710167e-09,
      "logits/chosen": -2.1340980529785156,
      "logits/rejected": -2.16754150390625,
      "logps/chosen": -0.2590358257293701,
      "logps/rejected": -0.2528657913208008,
      "loss": 0.1206575408577919,
      "loss/core": 0.10890237987041473,
      "loss/preference_sft": 0.2507992386817932,
      "loss/reference_anchor": 0.13165560364723206,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.4075381755828857,
      "rewards/margins": 1.279028296470642,
      "rewards/rejected": 1.128509759902954,
      "ronpo/logit": 0.1279028356075287,
      "ronpo/residual": 0.13108345866203308,
      "ronpo/residual_abs": 0.1609242856502533,
      "ronpo/target": -0.0031806365586817265,
      "ronpo/target_abs": 0.03893408179283142,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 795
    },
    {
      "drift/chosen_abs": 0.28383442759513855,
      "drift/rejected_abs": 0.1589200645685196,
      "epoch": 0.2542709574890743,
      "grad_norm": 20.125,
      "learning_rate": 4.23959612186448e-09,
      "logits/chosen": -2.1699929237365723,
      "logits/rejected": -2.2424261569976807,
      "logps/chosen": -0.26959216594696045,
      "logps/rejected": -0.27034133672714233,
      "loss": 0.2392566204071045,
      "loss/core": 0.17778518795967102,
      "loss/preference_sft": 0.25929346680641174,
      "loss/reference_anchor": 0.7936893701553345,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8376173973083496,
      "rewards/margins": 1.2496715784072876,
      "rewards/rejected": 1.5879456996917725,
      "ronpo/logit": 0.12496715784072876,
      "ronpo/residual": 0.1267654150724411,
      "ronpo/residual_abs": 0.14537087082862854,
      "ronpo/target": -0.0017982644494622946,
      "ronpo/target_abs": 0.03246055170893669,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 800
    },
    {
      "drift/chosen_abs": 0.31993216276168823,
      "drift/rejected_abs": 0.13845376670360565,
      "epoch": 0.255860150973381,
      "grad_norm": 195.0,
      "learning_rate": 3.835544694725529e-09,
      "logits/chosen": -2.2753825187683105,
      "logits/rejected": -2.2379002571105957,
      "logps/chosen": -0.3026265501976013,
      "logps/rejected": -0.30721110105514526,
      "loss": 0.9451104998588562,
      "loss/core": 0.8910640478134155,
      "loss/preference_sft": 0.29863858222961426,
      "loss/reference_anchor": 0.6907570958137512,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.199321746826172,
      "rewards/margins": 1.8681728839874268,
      "rewards/rejected": 1.3311488628387451,
      "ronpo/logit": 0.1868172585964203,
      "ronpo/residual": 0.18839898705482483,
      "ronpo/residual_abs": 0.2753552794456482,
      "ronpo/target": -0.0015817094827070832,
      "ronpo/target_abs": 0.036414168775081635,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 805
    },
    {
      "drift/chosen_abs": 0.2113758623600006,
      "drift/rejected_abs": 0.11016102135181427,
      "epoch": 0.25744934445768775,
      "grad_norm": 7.9375,
      "learning_rate": 3.4509562477761658e-09,
      "logits/chosen": -2.0627362728118896,
      "logits/rejected": -2.113102912902832,
      "logps/chosen": -0.283230185508728,
      "logps/rejected": -0.287577748298645,
      "loss": 0.04678564518690109,
      "loss/core": 0.03707611560821533,
      "loss/preference_sft": 0.2837401032447815,
      "loss/reference_anchor": 0.10108641535043716,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.1131529808044434,
      "rewards/margins": 1.0126079320907593,
      "rewards/rejected": 1.1005451679229736,
      "ronpo/logit": 0.10126079618930817,
      "ronpo/residual": 0.08919169753789902,
      "ronpo/residual_abs": 0.11383958160877228,
      "ronpo/target": 0.012069085612893105,
      "ronpo/target_abs": 0.03552011400461197,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 810
    },
    {
      "drift/chosen_abs": 0.2591978907585144,
      "drift/rejected_abs": 0.1618344634771347,
      "epoch": 0.25903853794199444,
      "grad_norm": 12.4375,
      "learning_rate": 3.0859929238874314e-09,
      "logits/chosen": -2.1082515716552734,
      "logits/rejected": -2.030113697052002,
      "logps/chosen": -0.26747792959213257,
      "logps/rejected": -0.27874407172203064,
      "loss": 0.21905355155467987,
      "loss/core": 0.2024170458316803,
      "loss/preference_sft": 0.26561182737350464,
      "loss/reference_anchor": 0.19525930285453796,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.5913424491882324,
      "rewards/margins": 0.9835828542709351,
      "rewards/rejected": 1.6077591180801392,
      "ronpo/logit": 0.09835829585790634,
      "ronpo/residual": 0.08990249782800674,
      "ronpo/residual_abs": 0.19450879096984863,
      "ronpo/target": 0.008455811068415642,
      "ronpo/target_abs": 0.04190158098936081,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 815
    },
    {
      "drift/chosen_abs": 0.19986757636070251,
      "drift/rejected_abs": 0.11452219635248184,
      "epoch": 0.26062773142630113,
      "grad_norm": 2.359375,
      "learning_rate": 2.7408085919590262e-09,
      "logits/chosen": -2.022050380706787,
      "logits/rejected": -1.9783365726470947,
      "logps/chosen": -0.2758370041847229,
      "logps/rejected": -0.28798961639404297,
      "loss": 0.06202256679534912,
      "loss/core": 0.047978322952985764,
      "loss/preference_sft": 0.2731260657310486,
      "loss/reference_anchor": 0.1599440574645996,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9943485260009766,
      "rewards/margins": 0.8497527241706848,
      "rewards/rejected": 1.1445958614349365,
      "ronpo/logit": 0.08497526496648788,
      "ronpo/residual": 0.08352629095315933,
      "ronpo/residual_abs": 0.11770425736904144,
      "ronpo/target": 0.0014489861205220222,
      "ronpo/target_abs": 0.03912581130862236,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 820
    },
    {
      "drift/chosen_abs": 0.287466824054718,
      "drift/rejected_abs": 0.18906255066394806,
      "epoch": 0.2622169249106079,
      "grad_norm": 80.5,
      "learning_rate": 2.415548782047999e-09,
      "logits/chosen": -2.213947057723999,
      "logits/rejected": -2.160534620285034,
      "logps/chosen": -0.299244225025177,
      "logps/rejected": -0.3014959692955017,
      "loss": 0.3660229444503784,
      "loss/core": 0.301073282957077,
      "loss/preference_sft": 0.29996412992477417,
      "loss/reference_anchor": 0.8359990119934082,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.8729865550994873,
      "rewards/margins": 0.9903990030288696,
      "rewards/rejected": 1.8825876712799072,
      "ronpo/logit": 0.09903990477323532,
      "ronpo/residual": 0.09887038916349411,
      "ronpo/residual_abs": 0.1854637861251831,
      "ronpo/target": 0.00016951440193224698,
      "ronpo/target_abs": 0.035706378519535065,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 825
    },
    {
      "drift/chosen_abs": 0.18484026193618774,
      "drift/rejected_abs": 0.11339914798736572,
      "epoch": 0.2638061183949146,
      "grad_norm": 31.875,
      "learning_rate": 2.110350624013124e-09,
      "logits/chosen": -2.233262538909912,
      "logits/rejected": -2.234431266784668,
      "logps/chosen": -0.277521014213562,
      "logps/rejected": -0.2729399502277374,
      "loss": 0.12444338947534561,
      "loss/core": 0.11208049207925797,
      "loss/preference_sft": 0.2777172923088074,
      "loss/reference_anchor": 0.1370667666196823,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8484026193618774,
      "rewards/margins": 0.7153452038764954,
      "rewards/rejected": 1.1330575942993164,
      "ronpo/logit": 0.07153451442718506,
      "ronpo/residual": 0.06523828208446503,
      "ronpo/residual_abs": 0.13810399174690247,
      "ronpo/target": 0.006296238861978054,
      "ronpo/target_abs": 0.03897542133927345,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 830
    },
    {
      "drift/chosen_abs": 0.40631216764450073,
      "drift/rejected_abs": 0.1610126495361328,
      "epoch": 0.2653953118792213,
      "grad_norm": 34.5,
      "learning_rate": 1.8253427897008456e-09,
      "logits/chosen": -2.191138982772827,
      "logits/rejected": -2.1562600135803223,
      "logps/chosen": -0.24265797436237335,
      "logps/rejected": -0.24613361060619354,
      "loss": 1.0934888124465942,
      "loss/core": 1.040398359298706,
      "loss/preference_sft": 0.24096707999706268,
      "loss/reference_anchor": 0.6837762594223022,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 4.062660217285156,
      "rewards/margins": 2.45281982421875,
      "rewards/rejected": 1.6098406314849854,
      "ronpo/logit": 0.24528197944164276,
      "ronpo/residual": 0.23903052508831024,
      "ronpo/residual_abs": 0.3242782950401306,
      "ronpo/target": 0.006251484155654907,
      "ronpo/target_abs": 0.036385711282491684,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 835
    },
    {
      "drift/chosen_abs": 0.28632503747940063,
      "drift/rejected_abs": 0.13661116361618042,
      "epoch": 0.266984505363528,
      "grad_norm": 0.92578125,
      "learning_rate": 1.5606454386970668e-09,
      "logits/chosen": -2.362611770629883,
      "logits/rejected": -2.4255595207214355,
      "logps/chosen": -0.2671854496002197,
      "logps/rejected": -0.2649426758289337,
      "loss": 0.36431366205215454,
      "loss/core": 0.3328004479408264,
      "loss/preference_sft": 0.2735097408294678,
      "loss/reference_anchor": 0.3928253948688507,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.863193988800049,
      "rewards/margins": 1.4984618425369263,
      "rewards/rejected": 1.364732027053833,
      "ronpo/logit": 0.1498461812734604,
      "ronpo/residual": 0.1546889841556549,
      "ronpo/residual_abs": 0.17629854381084442,
      "ronpo/target": -0.0048428005538880825,
      "ronpo/target_abs": 0.04114602878689766,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 840
    },
    {
      "drift/chosen_abs": 0.29827648401260376,
      "drift/rejected_abs": 0.18521063029766083,
      "epoch": 0.2685736988478347,
      "grad_norm": 4.96875,
      "learning_rate": 1.316370167667763e-09,
      "logits/chosen": -2.1433889865875244,
      "logits/rejected": -2.135906457901001,
      "logps/chosen": -0.24772045016288757,
      "logps/rejected": -0.23420384526252747,
      "loss": 0.09277145564556122,
      "loss/core": 0.06800076365470886,
      "loss/preference_sft": 0.24155735969543457,
      "loss/reference_anchor": 0.3061201870441437,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.98088002204895,
      "rewards/margins": 1.129408597946167,
      "rewards/rejected": 1.8514715433120728,
      "ronpo/logit": 0.11294084787368774,
      "ronpo/residual": 0.10954286903142929,
      "ronpo/residual_abs": 0.1346508264541626,
      "ronpo/target": 0.0033979781437665224,
      "ronpo/target_abs": 0.04030502587556839,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 845
    },
    {
      "drift/chosen_abs": 0.41069334745407104,
      "drift/rejected_abs": 0.14589200913906097,
      "epoch": 0.2701628923321414,
      "grad_norm": 80.0,
      "learning_rate": 1.0926199633097156e-09,
      "logits/chosen": -1.9553956985473633,
      "logits/rejected": -1.8889896869659424,
      "logps/chosen": -0.27275902032852173,
      "logps/rejected": -0.28432151675224304,
      "loss": 0.9306114315986633,
      "loss/core": 0.8736518621444702,
      "loss/preference_sft": 0.2677767872810364,
      "loss/reference_anchor": 0.7326820492744446,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 4.104775428771973,
      "rewards/margins": 2.6476259231567383,
      "rewards/rejected": 1.4571493864059448,
      "ronpo/logit": 0.2647625803947449,
      "ronpo/residual": 0.2660098969936371,
      "ronpo/residual_abs": 0.2944030165672302,
      "ronpo/target": -0.0012473245151340961,
      "ronpo/target_abs": 0.03741658851504326,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 850
    },
    {
      "drift/chosen_abs": 0.2068370282649994,
      "drift/rejected_abs": 0.07252339273691177,
      "epoch": 0.27175208581644816,
      "grad_norm": 2.078125,
      "learning_rate": 8.894891589312448e-10,
      "logits/chosen": -2.378898859024048,
      "logits/rejected": -2.3654732704162598,
      "logps/chosen": -0.2773938775062561,
      "logps/rejected": -0.2757563591003418,
      "loss": 0.41834402084350586,
      "loss/core": 0.39636608958244324,
      "loss/preference_sft": 0.27365297079086304,
      "loss/reference_anchor": 0.26567381620407104,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0683701038360596,
      "rewards/margins": 1.3431364297866821,
      "rewards/rejected": 0.7252338528633118,
      "ronpo/logit": 0.13431362807750702,
      "ronpo/residual": 0.13792987167835236,
      "ronpo/residual_abs": 0.1564667969942093,
      "ronpo/target": -0.0036162561737000942,
      "ronpo/target_abs": 0.03731636330485344,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 855
    },
    {
      "drift/chosen_abs": 0.18851238489151,
      "drift/rejected_abs": 0.10406199842691422,
      "epoch": 0.27334127930075486,
      "grad_norm": 32.5,
      "learning_rate": 7.070633946811345e-10,
      "logits/chosen": -2.203418254852295,
      "logits/rejected": -2.129629373550415,
      "logps/chosen": -0.2803042531013489,
      "logps/rejected": -0.2796124815940857,
      "loss": 0.12177155166864395,
      "loss/core": 0.10930196195840836,
      "loss/preference_sft": 0.2833760976791382,
      "loss/reference_anchor": 0.13792333006858826,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8851238489151,
      "rewards/margins": 0.8534868359565735,
      "rewards/rejected": 1.0316369533538818,
      "ronpo/logit": 0.08534868061542511,
      "ronpo/residual": 0.08152299374341965,
      "ronpo/residual_abs": 0.1145586222410202,
      "ronpo/target": 0.0038256910629570484,
      "ronpo/target_abs": 0.03399581462144852,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 860
    },
    {
      "drift/chosen_abs": 0.24835388362407684,
      "drift/rejected_abs": 0.12930510938167572,
      "epoch": 0.2749304727850616,
      "grad_norm": 0.5703125,
      "learning_rate": 5.454195814427021e-10,
      "logits/chosen": -2.090212821960449,
      "logits/rejected": -2.035008668899536,
      "logps/chosen": -0.2799471318721771,
      "logps/rejected": -0.2892903685569763,
      "loss": 0.11395768821239471,
      "loss/core": 0.09873252362012863,
      "loss/preference_sft": 0.2899934649467468,
      "loss/reference_anchor": 0.1740027666091919,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.4830942153930664,
      "rewards/margins": 1.1911389827728271,
      "rewards/rejected": 1.2919554710388184,
      "ronpo/logit": 0.11911387741565704,
      "ronpo/residual": 0.11887941509485245,
      "ronpo/residual_abs": 0.15069282054901123,
      "ronpo/target": 0.00023447340936399996,
      "ronpo/target_abs": 0.03890164569020271,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 865
    },
    {
      "drift/chosen_abs": 0.4616221487522125,
      "drift/rejected_abs": 0.2202831208705902,
      "epoch": 0.2765196662693683,
      "grad_norm": 520.0,
      "learning_rate": 4.0462586840810387e-10,
      "logits/chosen": -2.099531888961792,
      "logits/rejected": -2.0872442722320557,
      "logps/chosen": -0.27098220586776733,
      "logps/rejected": -0.2758156359195709,
      "loss": 0.7392908930778503,
      "loss/core": 0.6623992919921875,
      "loss/preference_sft": 0.2761059105396271,
      "loss/reference_anchor": 0.9976102709770203,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.614747047424316,
      "rewards/margins": 2.412616014480591,
      "rewards/rejected": 2.2021312713623047,
      "ronpo/logit": 0.2412615716457367,
      "ronpo/residual": 0.2500116527080536,
      "ronpo/residual_abs": 0.27789297699928284,
      "ronpo/target": -0.008750082924962044,
      "ronpo/target_abs": 0.0406760573387146,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 870
    },
    {
      "drift/chosen_abs": 0.30633243918418884,
      "drift/rejected_abs": 0.15432590246200562,
      "epoch": 0.278108859753675,
      "grad_norm": 5.1875,
      "learning_rate": 2.847416143465342e-10,
      "logits/chosen": -1.943389892578125,
      "logits/rejected": -1.9119516611099243,
      "logps/chosen": -0.2705642282962799,
      "logps/rejected": -0.27163758873939514,
      "loss": 0.2206476628780365,
      "loss/core": 0.19633831083774567,
      "loss/preference_sft": 0.27449434995651245,
      "loss/reference_anchor": 0.2966754138469696,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.0624802112579346,
      "rewards/margins": 1.5204280614852905,
      "rewards/rejected": 1.542051911354065,
      "ronpo/logit": 0.15204280614852905,
      "ronpo/residual": 0.1534896343946457,
      "ronpo/residual_abs": 0.17870879173278809,
      "ronpo/target": -0.001446827664040029,
      "ronpo/target_abs": 0.04036080092191696,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 875
    },
    {
      "drift/chosen_abs": 0.244231179356575,
      "drift/rejected_abs": 0.09062740206718445,
      "epoch": 0.27969805323798175,
      "grad_norm": 5.5,
      "learning_rate": 1.8581736257852754e-10,
      "logits/chosen": -2.1149797439575195,
      "logits/rejected": -2.1567158699035645,
      "logps/chosen": -0.27833813428878784,
      "logps/rejected": -0.28993064165115356,
      "loss": 0.40490633249282837,
      "loss/core": 0.38430437445640564,
      "loss/preference_sft": 0.29224926233291626,
      "loss/reference_anchor": 0.24546775221824646,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.4423117637634277,
      "rewards/margins": 1.538805365562439,
      "rewards/rejected": 0.9035066366195679,
      "ronpo/logit": 0.1538805216550827,
      "ronpo/residual": 0.14947783946990967,
      "ronpo/residual_abs": 0.1738136261701584,
      "ronpo/target": 0.004402679391205311,
      "ronpo/target_abs": 0.04329409450292587,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 880
    },
    {
      "drift/chosen_abs": 0.2656122148036957,
      "drift/rejected_abs": 0.10401421785354614,
      "epoch": 0.28128724672228844,
      "grad_norm": 7.0625,
      "learning_rate": 1.0789481966683722e-10,
      "logits/chosen": -2.159553050994873,
      "logits/rejected": -2.2222096920013428,
      "logps/chosen": -0.29716652631759644,
      "logps/rejected": -0.2955102324485779,
      "loss": 0.41419491171836853,
      "loss/core": 0.3724747598171234,
      "loss/preference_sft": 0.29870742559432983,
      "loss/reference_anchor": 0.5263984203338623,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.6553502082824707,
      "rewards/margins": 1.617895483970642,
      "rewards/rejected": 1.037454605102539,
      "ronpo/logit": 0.16178953647613525,
      "ronpo/residual": 0.16292984783649445,
      "ronpo/residual_abs": 0.17799483239650726,
      "ronpo/target": -0.0011402855161577463,
      "ronpo/target_abs": 0.03544146940112114,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 885
    },
    {
      "drift/chosen_abs": 0.2998327910900116,
      "drift/rejected_abs": 0.12200510501861572,
      "epoch": 0.28287644020659514,
      "grad_norm": 3.515625,
      "learning_rate": 5.1006837832928694e-11,
      "logits/chosen": -2.3049538135528564,
      "logits/rejected": -2.263355255126953,
      "logps/chosen": -0.28130239248275757,
      "logps/rejected": -0.28048938512802124,
      "loss": 0.4184916913509369,
      "loss/core": 0.37995871901512146,
      "loss/preference_sft": 0.28656134009361267,
      "loss/reference_anchor": 0.48511725664138794,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.9957728385925293,
      "rewards/margins": 1.780024766921997,
      "rewards/rejected": 1.2157481908798218,
      "ronpo/logit": 0.1780024766921997,
      "ronpo/residual": 0.1812599003314972,
      "ronpo/residual_abs": 0.19650830328464508,
      "ronpo/target": -0.003257411066442728,
      "ronpo/target_abs": 0.035617318004369736,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 890
    },
    {
      "drift/chosen_abs": 0.3059307932853699,
      "drift/rejected_abs": 0.1232098937034607,
      "epoch": 0.2844656336909019,
      "grad_norm": 13.5,
      "learning_rate": 1.517740110641985e-11,
      "logits/chosen": -2.080798625946045,
      "logits/rejected": -2.070003032684326,
      "logps/chosen": -0.28609567880630493,
      "logps/rejected": -0.2946876883506775,
      "loss": 0.3638497292995453,
      "loss/core": 0.3331601321697235,
      "loss/preference_sft": 0.3008359372615814,
      "loss/reference_anchor": 0.37911102175712585,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.0580692291259766,
      "rewards/margins": 1.9168615341186523,
      "rewards/rejected": 1.1412078142166138,
      "ronpo/logit": 0.19168615341186523,
      "ronpo/residual": 0.18204104900360107,
      "ronpo/residual_abs": 0.21228711307048798,
      "ronpo/target": 0.009645095095038414,
      "ronpo/target_abs": 0.044705357402563095,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 895
    },
    {
      "drift/chosen_abs": 0.15993809700012207,
      "drift/rejected_abs": 0.0836057960987091,
      "epoch": 0.2860548271752086,
      "grad_norm": 15.4375,
      "learning_rate": 4.2161521340289277e-13,
      "logits/chosen": -2.1035187244415283,
      "logits/rejected": -2.086210012435913,
      "logps/chosen": -0.31812211871147156,
      "logps/rejected": -0.3182290196418762,
      "loss": 0.13807110488414764,
      "loss/core": 0.12665197253227234,
      "loss/preference_sft": 0.32046961784362793,
      "loss/reference_anchor": 0.1202082484960556,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.598246455192566,
      "rewards/margins": 0.88621586561203,
      "rewards/rejected": 0.7120307087898254,
      "ronpo/logit": 0.0886215791106224,
      "ronpo/residual": 0.08629345148801804,
      "ronpo/residual_abs": 0.11131948232650757,
      "ronpo/target": 0.0023281420581042767,
      "ronpo/target_abs": 0.036599740386009216,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 900
    },
    {
      "epoch": 0.2860548271752086,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 0.430257171690464,
      "train_runtime": 7318.8237,
      "train_samples_per_second": 1.968,
      "train_steps_per_second": 0.123
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}