{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8599068434252956,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "drift/chosen_abs": 0.3420991003513336,
      "drift/rejected_abs": 0.1417631208896637,
      "epoch": 0.004777260241251642,
      "grad_norm": 374.0,
      "learning_rate": 1.111111111111111e-08,
      "logits/chosen": -2.2763760089874268,
      "logits/rejected": -2.2397658824920654,
      "logps/chosen": -0.3073771595954895,
      "logps/rejected": -0.3057326674461365,
      "loss": 2.170994281768799,
      "loss/core": 2.1488795280456543,
      "loss/preference_sft": 0.3073771595954895,
      "loss/reference_anchor": 1.075006365776062,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.418382167816162,
      "rewards/margins": 2.0053153038024902,
      "rewards/rejected": 1.4130665063858032,
      "step": 5
    },
    {
      "drift/chosen_abs": 0.29207316040992737,
      "drift/rejected_abs": 0.1325351893901825,
      "epoch": 0.009554520482503284,
      "grad_norm": 11.125,
      "learning_rate": 2.5e-08,
      "logits/chosen": -2.2057507038116455,
      "logits/rejected": -2.197091579437256,
      "logps/chosen": -0.2905746102333069,
      "logps/rejected": -0.28117018938064575,
      "loss": 0.5789883136749268,
      "loss/core": 0.5726724863052368,
      "loss/preference_sft": 0.2905746102333069,
      "loss/reference_anchor": 0.2867390513420105,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.920731782913208,
      "rewards/margins": 1.5960335731506348,
      "rewards/rejected": 1.3246982097625732,
      "step": 10
    },
    {
      "drift/chosen_abs": 0.1926683634519577,
      "drift/rejected_abs": 0.08847218751907349,
      "epoch": 0.014331780723754926,
      "grad_norm": 2.265625,
      "learning_rate": 3.888888888888889e-08,
      "logits/chosen": -1.8458455801010132,
      "logits/rejected": -1.9005146026611328,
      "logps/chosen": -0.2896016538143158,
      "logps/rejected": -0.28710517287254333,
      "loss": 0.23165909945964813,
      "loss/core": 0.22878754138946533,
      "loss/preference_sft": 0.2896016538143158,
      "loss/reference_anchor": 0.11461815983057022,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.9264246225357056,
      "rewards/margins": 1.057593584060669,
      "rewards/rejected": 0.8688309788703918,
      "step": 15
    },
    {
      "drift/chosen_abs": 0.1641753613948822,
      "drift/rejected_abs": 0.05660538747906685,
      "epoch": 0.01910904096500657,
      "grad_norm": 4.96875,
      "learning_rate": 5.2777777777777776e-08,
      "logits/chosen": -2.1686434745788574,
      "logits/rejected": -2.0960006713867188,
      "logps/chosen": -0.2862441837787628,
      "logps/rejected": -0.28391966223716736,
      "loss": 0.15841466188430786,
      "loss/core": 0.15627533197402954,
      "loss/preference_sft": 0.2862441837787628,
      "loss/reference_anchor": 0.07834186404943466,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6407321691513062,
      "rewards/margins": 1.0884038209915161,
      "rewards/rejected": 0.5523284077644348,
      "step": 20
    },
    {
      "drift/chosen_abs": 0.18150915205478668,
      "drift/rejected_abs": 0.11916539818048477,
      "epoch": 0.02388630120625821,
      "grad_norm": 73.0,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -2.2032320499420166,
      "logits/rejected": -2.2200589179992676,
      "logps/chosen": -0.2700212597846985,
      "logps/rejected": -0.2805071473121643,
      "loss": 0.263420045375824,
      "loss/core": 0.2602730393409729,
      "loss/preference_sft": 0.2700212597846985,
      "loss/reference_anchor": 0.13034658133983612,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8135448694229126,
      "rewards/margins": 0.6393396258354187,
      "rewards/rejected": 1.1742051839828491,
      "step": 25
    },
    {
      "drift/chosen_abs": 0.2730526924133301,
      "drift/rejected_abs": 0.1805466115474701,
      "epoch": 0.028663561447509853,
      "grad_norm": 156.0,
      "learning_rate": 8.055555555555555e-08,
      "logits/chosen": -2.1016995906829834,
      "logits/rejected": -2.0634474754333496,
      "logps/chosen": -0.31322628259658813,
      "logps/rejected": -0.27209383249282837,
      "loss": 1.1556752920150757,
      "loss/core": 1.1436045169830322,
      "loss/preference_sft": 0.31322628259658813,
      "loss/reference_anchor": 0.5722212791442871,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.7279961109161377,
      "rewards/margins": 0.923298180103302,
      "rewards/rejected": 1.8046979904174805,
      "step": 30
    },
    {
      "drift/chosen_abs": 0.18750497698783875,
      "drift/rejected_abs": 0.07988496869802475,
      "epoch": 0.033440821688761495,
      "grad_norm": 3.453125,
      "learning_rate": 9.444444444444444e-08,
      "logits/chosen": -2.1232945919036865,
      "logits/rejected": -2.225569248199463,
      "logps/chosen": -0.2830365300178528,
      "logps/rejected": -0.26490843296051025,
      "loss": 0.17748409509658813,
      "loss/core": 0.17516210675239563,
      "loss/preference_sft": 0.2830365300178528,
      "loss/reference_anchor": 0.08779536187648773,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8736076354980469,
      "rewards/margins": 1.09036123752594,
      "rewards/rejected": 0.7832463979721069,
      "step": 35
    },
    {
      "drift/chosen_abs": 0.23411647975444794,
      "drift/rejected_abs": 0.09753605723381042,
      "epoch": 0.03821808193001314,
      "grad_norm": 13.5,
      "learning_rate": 1.0833333333333334e-07,
      "logits/chosen": -2.1147046089172363,
      "logits/rejected": -2.191413164138794,
      "logps/chosen": -0.29881948232650757,
      "logps/rejected": -0.3003283143043518,
      "loss": 0.31825491786003113,
      "loss/core": 0.3145065903663635,
      "loss/preference_sft": 0.29881948232650757,
      "loss/reference_anchor": 0.1575334519147873,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.3403539657592773,
      "rewards/margins": 1.3710139989852905,
      "rewards/rejected": 0.9693400263786316,
      "step": 40
    },
    {
      "drift/chosen_abs": 0.28226155042648315,
      "drift/rejected_abs": 0.18430650234222412,
      "epoch": 0.04299534217126478,
      "grad_norm": 69.5,
      "learning_rate": 1.2222222222222222e-07,
      "logits/chosen": -2.0363850593566895,
      "logits/rejected": -2.076094150543213,
      "logps/chosen": -0.28957605361938477,
      "logps/rejected": -0.2919236719608307,
      "loss": 1.3649934530258179,
      "loss/core": 1.3508943319320679,
      "loss/preference_sft": 0.28957605361938477,
      "loss/reference_anchor": 0.6759929060935974,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.8192150592803955,
      "rewards/margins": 0.9820273518562317,
      "rewards/rejected": 1.8371880054473877,
      "step": 45
    },
    {
      "drift/chosen_abs": 0.1817009598016739,
      "drift/rejected_abs": 0.1377423107624054,
      "epoch": 0.04777260241251642,
      "grad_norm": 0.984375,
      "learning_rate": 1.3611111111111108e-07,
      "logits/chosen": -2.1037697792053223,
      "logits/rejected": -2.192720890045166,
      "logps/chosen": -0.2859702706336975,
      "logps/rejected": -0.2898501455783844,
      "loss": 0.35004186630249023,
      "loss/core": 0.34600991010665894,
      "loss/preference_sft": 0.2859702706336975,
      "loss/reference_anchor": 0.17300085723400116,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.816737413406372,
      "rewards/margins": 0.44307082891464233,
      "rewards/rejected": 1.3736662864685059,
      "step": 50
    },
    {
      "drift/chosen_abs": 0.1675940454006195,
      "drift/rejected_abs": 0.0922044888138771,
      "epoch": 0.05254986265376806,
      "grad_norm": 3.890625,
      "learning_rate": 1.5e-07,
      "logits/chosen": -2.1252381801605225,
      "logits/rejected": -2.2449936866760254,
      "logps/chosen": -0.2968815565109253,
      "logps/rejected": -0.293079674243927,
      "loss": 0.11641957610845566,
      "loss/core": 0.11467579752206802,
      "loss/preference_sft": 0.2968815565109253,
      "loss/reference_anchor": 0.05750115588307381,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6759402751922607,
      "rewards/margins": 0.7550500631332397,
      "rewards/rejected": 0.9208904504776001,
      "step": 55
    },
    {
      "drift/chosen_abs": 0.14563629031181335,
      "drift/rejected_abs": 0.1081518679857254,
      "epoch": 0.057327122895019705,
      "grad_norm": 62.25,
      "learning_rate": 1.6388888888888888e-07,
      "logits/chosen": -2.1089062690734863,
      "logits/rejected": -2.1458160877227783,
      "logps/chosen": -0.29047664999961853,
      "logps/rejected": -0.2865060865879059,
      "loss": 0.29185742139816284,
      "loss/core": 0.2883908748626709,
      "loss/preference_sft": 0.29047664999961853,
      "loss/reference_anchor": 0.14428026974201202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.4559776782989502,
      "rewards/margins": 0.3751867413520813,
      "rewards/rejected": 1.0807908773422241,
      "step": 60
    },
    {
      "drift/chosen_abs": 0.2445906698703766,
      "drift/rejected_abs": 0.09041162580251694,
      "epoch": 0.06210438313627135,
      "grad_norm": 154.0,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -2.1424062252044678,
      "logits/rejected": -2.2173452377319336,
      "logps/chosen": -0.3165227174758911,
      "logps/rejected": -0.310057133436203,
      "loss": 0.7591328620910645,
      "loss/core": 0.7509864568710327,
      "loss/preference_sft": 0.3165227174758911,
      "loss/reference_anchor": 0.37566640973091125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.4453353881835938,
      "rewards/margins": 1.5434644222259521,
      "rewards/rejected": 0.901870846748352,
      "step": 65
    },
    {
      "drift/chosen_abs": 0.23211221396923065,
      "drift/rejected_abs": 0.10898170620203018,
      "epoch": 0.06688164337752299,
      "grad_norm": 21.5,
      "learning_rate": 1.9166666666666668e-07,
      "logits/chosen": -2.1081697940826416,
      "logits/rejected": -2.1647961139678955,
      "logps/chosen": -0.3016880452632904,
      "logps/rejected": -0.3038561940193176,
      "loss": 0.4961710572242737,
      "loss/core": 0.4906535744667053,
      "loss/preference_sft": 0.3016880452632904,
      "loss/reference_anchor": 0.24570658802986145,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.32112193107605,
      "rewards/margins": 1.3848812580108643,
      "rewards/rejected": 0.9362408518791199,
      "step": 70
    },
    {
      "drift/chosen_abs": 0.15495255589485168,
      "drift/rejected_abs": 0.10253290832042694,
      "epoch": 0.07165890361877464,
      "grad_norm": 3.71875,
      "learning_rate": 2.0555555555555553e-07,
      "logits/chosen": -2.0683295726776123,
      "logits/rejected": -2.1414763927459717,
      "logps/chosen": -0.28989818692207336,
      "logps/rejected": -0.29636505246162415,
      "loss": 0.21972903609275818,
      "loss/core": 0.21697552502155304,
      "loss/preference_sft": 0.28989818692207336,
      "loss/reference_anchor": 0.10868575423955917,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.5473926067352295,
      "rewards/margins": 0.5245345830917358,
      "rewards/rejected": 1.0228580236434937,
      "step": 75
    },
    {
      "drift/chosen_abs": 0.12912170588970184,
      "drift/rejected_abs": 0.07073847949504852,
      "epoch": 0.07643616386002627,
      "grad_norm": 3.546875,
      "learning_rate": 2.1944444444444442e-07,
      "logits/chosen": -2.271545648574829,
      "logits/rejected": -2.3474278450012207,
      "logps/chosen": -0.30917978286743164,
      "logps/rejected": -0.30712777376174927,
      "loss": 0.09423994272947311,
      "loss/core": 0.09269268810749054,
      "loss/preference_sft": 0.30917978286743164,
      "loss/reference_anchor": 0.046444639563560486,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2905423641204834,
      "rewards/margins": 0.5846601724624634,
      "rewards/rejected": 0.7058821320533752,
      "step": 80
    },
    {
      "drift/chosen_abs": 0.2436620444059372,
      "drift/rejected_abs": 0.1702345758676529,
      "epoch": 0.08121342410127792,
      "grad_norm": 1.859375,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -2.1165645122528076,
      "logits/rejected": -2.1273322105407715,
      "logps/chosen": -0.2930624485015869,
      "logps/rejected": -0.28557881712913513,
      "loss": 0.9216262698173523,
      "loss/core": 0.9119135141372681,
      "loss/preference_sft": 0.2930624485015869,
      "loss/reference_anchor": 0.45633044838905334,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.4357974529266357,
      "rewards/margins": 0.7339332103729248,
      "rewards/rejected": 1.7018646001815796,
      "step": 85
    },
    {
      "drift/chosen_abs": 0.2897084355354309,
      "drift/rejected_abs": 0.06650175154209137,
      "epoch": 0.08599068434252956,
      "grad_norm": 21.375,
      "learning_rate": 2.4722222222222224e-07,
      "logits/chosen": -2.0020716190338135,
      "logits/rejected": -2.18646502494812,
      "logps/chosen": -0.30623993277549744,
      "logps/rejected": -0.3014351725578308,
      "loss": 0.6364718675613403,
      "loss/core": 0.6295538544654846,
      "loss/preference_sft": 0.30623993277549744,
      "loss/reference_anchor": 0.3152781128883362,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.8956007957458496,
      "rewards/margins": 2.232067584991455,
      "rewards/rejected": 0.6635335087776184,
      "step": 90
    },
    {
      "drift/chosen_abs": 0.21713528037071228,
      "drift/rejected_abs": 0.13760428130626678,
      "epoch": 0.09076794458378121,
      "grad_norm": 113.5,
      "learning_rate": 2.4998495746616845e-07,
      "logits/chosen": -2.0369200706481934,
      "logits/rejected": -1.9513944387435913,
      "logps/chosen": -0.2811819612979889,
      "logps/rejected": -0.30305975675582886,
      "loss": 0.6681066751480103,
      "loss/core": 0.6609320640563965,
      "loss/preference_sft": 0.2811819612979889,
      "loss/reference_anchor": 0.33061572909355164,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.170912981033325,
      "rewards/margins": 0.7965704798698425,
      "rewards/rejected": 1.3743422031402588,
      "step": 95
    },
    {
      "drift/chosen_abs": 0.2975863218307495,
      "drift/rejected_abs": 0.1862352043390274,
      "epoch": 0.09554520482503284,
      "grad_norm": 1.671875,
      "learning_rate": 2.4992385337738696e-07,
      "logits/chosen": -2.008779525756836,
      "logits/rejected": -2.0400052070617676,
      "logps/chosen": -0.28240451216697693,
      "logps/rejected": -0.31364208459854126,
      "loss": 0.9967119097709656,
      "loss/core": 0.9862802624702454,
      "loss/preference_sft": 0.28240451216697693,
      "loss/reference_anchor": 0.4933498501777649,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9755539894104004,
      "rewards/margins": 1.446618676185608,
      "rewards/rejected": 1.5289356708526611,
      "step": 100
    },
    {
      "drift/chosen_abs": 0.31854820251464844,
      "drift/rejected_abs": 0.13878673315048218,
      "epoch": 0.10032246506628449,
      "grad_norm": 4.375,
      "learning_rate": 2.4981577053636144e-07,
      "logits/chosen": -1.934293508529663,
      "logits/rejected": -1.9871383905410767,
      "logps/chosen": -0.2761833667755127,
      "logps/rejected": -0.26434868574142456,
      "loss": 0.7714275121688843,
      "loss/core": 0.7632346153259277,
      "loss/preference_sft": 0.2761833667755127,
      "loss/reference_anchor": 0.38203054666519165,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.1841020584106445,
      "rewards/margins": 1.7970466613769531,
      "rewards/rejected": 1.3870553970336914,
      "step": 105
    },
    {
      "drift/chosen_abs": 0.21965603530406952,
      "drift/rejected_abs": 0.10388318449258804,
      "epoch": 0.10509972530753613,
      "grad_norm": 5.09375,
      "learning_rate": 2.496607495886281e-07,
      "logits/chosen": -2.1238505840301514,
      "logits/rejected": -2.097701072692871,
      "logps/chosen": -0.2954608201980591,
      "logps/rejected": -0.3057326674461365,
      "loss": 0.650016188621521,
      "loss/core": 0.6429935097694397,
      "loss/preference_sft": 0.2954608201980591,
      "loss/reference_anchor": 0.3215869665145874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1926186084747314,
      "rewards/margins": 1.1593815088272095,
      "rewards/rejected": 1.0332369804382324,
      "step": 110
    },
    {
      "drift/chosen_abs": 0.3507579267024994,
      "drift/rejected_abs": 0.12545308470726013,
      "epoch": 0.10987698554878778,
      "grad_norm": 536.0,
      "learning_rate": 2.4945884883122553e-07,
      "logits/chosen": -2.0215022563934326,
      "logits/rejected": -2.0700583457946777,
      "logps/chosen": -0.28947877883911133,
      "logps/rejected": -0.2858577370643616,
      "loss": 1.4179147481918335,
      "loss/core": 1.4032930135726929,
      "loss/preference_sft": 0.28947877883911133,
      "loss/reference_anchor": 0.7021245956420898,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.5075790882110596,
      "rewards/margins": 2.2566137313842773,
      "rewards/rejected": 1.2509657144546509,
      "step": 115
    },
    {
      "drift/chosen_abs": 0.1810287982225418,
      "drift/rejected_abs": 0.14085547626018524,
      "epoch": 0.11465424579003941,
      "grad_norm": 181.0,
      "learning_rate": 2.492101441907714e-07,
      "logits/chosen": -2.2403197288513184,
      "logits/rejected": -2.1026992797851562,
      "logps/chosen": -0.260287880897522,
      "logps/rejected": -0.28016167879104614,
      "loss": 0.29911449551582336,
      "loss/core": 0.2956363260746002,
      "loss/preference_sft": 0.260287880897522,
      "loss/reference_anchor": 0.14787891507148743,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8089395761489868,
      "rewards/margins": 0.4003846049308777,
      "rewards/rejected": 1.4085547924041748,
      "step": 120
    },
    {
      "drift/chosen_abs": 0.4085167944431305,
      "drift/rejected_abs": 0.10499458014965057,
      "epoch": 0.11943150603129106,
      "grad_norm": 1.140625,
      "learning_rate": 2.4891472919490977e-07,
      "logits/chosen": -1.9146769046783447,
      "logits/rejected": -1.9590771198272705,
      "logps/chosen": -0.28118112683296204,
      "logps/rejected": -0.27787166833877563,
      "loss": 1.8055627346038818,
      "loss/core": 1.7871158123016357,
      "loss/preference_sft": 0.28118112683296204,
      "loss/reference_anchor": 0.8942378759384155,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.083503723144531,
      "rewards/margins": 3.059095859527588,
      "rewards/rejected": 1.024407982826233,
      "step": 125
    },
    {
      "drift/chosen_abs": 0.2690292298793793,
      "drift/rejected_abs": 0.1741829663515091,
      "epoch": 0.1242087662725427,
      "grad_norm": 165.0,
      "learning_rate": 2.4857271493713894e-07,
      "logits/chosen": -2.0733864307403564,
      "logits/rejected": -2.071460247039795,
      "logps/chosen": -0.29650020599365234,
      "logps/rejected": -0.29231899976730347,
      "loss": 1.0611131191253662,
      "loss/core": 1.050018072128296,
      "loss/preference_sft": 0.29650020599365234,
      "loss/reference_anchor": 0.5251095294952393,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.689784049987793,
      "rewards/margins": 0.9507070779800415,
      "rewards/rejected": 1.7390769720077515,
      "step": 130
    },
    {
      "drift/chosen_abs": 0.1724942922592163,
      "drift/rejected_abs": 0.08941344916820526,
      "epoch": 0.12898602651379434,
      "grad_norm": 7.40625,
      "learning_rate": 2.481842300350339e-07,
      "logits/chosen": -2.0276589393615723,
      "logits/rejected": -2.0920987129211426,
      "logps/chosen": -0.27670344710350037,
      "logps/rejected": -0.263897567987442,
      "loss": 0.19596371054649353,
      "loss/core": 0.19347220659255981,
      "loss/preference_sft": 0.27670344710350037,
      "loss/reference_anchor": 0.09690549224615097,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.7226674556732178,
      "rewards/margins": 0.83062744140625,
      "rewards/rejected": 0.8920401334762573,
      "step": 135
    },
    {
      "drift/chosen_abs": 0.22231808304786682,
      "drift/rejected_abs": 0.09158378839492798,
      "epoch": 0.13376328675504598,
      "grad_norm": 1.109375,
      "learning_rate": 2.4774942058187854e-07,
      "logits/chosen": -2.191605806350708,
      "logits/rejected": -2.250380039215088,
      "logps/chosen": -0.2993098199367523,
      "logps/rejected": -0.2922438979148865,
      "loss": 0.7870423793792725,
      "loss/core": 0.7786499261856079,
      "loss/preference_sft": 0.2993098199367523,
      "loss/reference_anchor": 0.38969308137893677,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2213363647460938,
      "rewards/margins": 1.306333303451538,
      "rewards/rejected": 0.9150028228759766,
      "step": 140
    },
    {
      "drift/chosen_abs": 0.1859746277332306,
      "drift/rejected_abs": 0.1087212786078453,
      "epoch": 0.13854054699629761,
      "grad_norm": 24.0,
      "learning_rate": 2.472684500917257e-07,
      "logits/chosen": -1.9669681787490845,
      "logits/rejected": -1.9938608407974243,
      "logps/chosen": -0.3318449854850769,
      "logps/rejected": -0.3153822422027588,
      "loss": 0.21972651779651642,
      "loss/core": 0.21688850224018097,
      "loss/preference_sft": 0.3318449854850769,
      "loss/reference_anchor": 0.10871516168117523,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.858870267868042,
      "rewards/margins": 0.7766832709312439,
      "rewards/rejected": 1.0821870565414429,
      "step": 145
    },
    {
      "drift/chosen_abs": 0.14986830949783325,
      "drift/rejected_abs": 0.05766589194536209,
      "epoch": 0.14331780723754928,
      "grad_norm": 3.046875,
      "learning_rate": 2.467414994379065e-07,
      "logits/chosen": -2.068981170654297,
      "logits/rejected": -2.1354150772094727,
      "logps/chosen": -0.31118273735046387,
      "logps/rejected": -0.316683828830719,
      "loss": 0.3042221665382385,
      "loss/core": 0.30058878660202026,
      "loss/preference_sft": 0.31118273735046387,
      "loss/reference_anchor": 0.15054963529109955,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.497855305671692,
      "rewards/margins": 0.9212591052055359,
      "rewards/rejected": 0.5765963792800903,
      "step": 150
    },
    {
      "drift/chosen_abs": 0.15884414315223694,
      "drift/rejected_abs": 0.08928201347589493,
      "epoch": 0.1480950674788009,
      "grad_norm": 17.25,
      "learning_rate": 2.4616876678501114e-07,
      "logits/chosen": -2.2533211708068848,
      "logits/rejected": -2.299926280975342,
      "logps/chosen": -0.3172222971916199,
      "logps/rejected": -0.3034704029560089,
      "loss": 0.1397572010755539,
      "loss/core": 0.13774201273918152,
      "loss/preference_sft": 0.3172222971916199,
      "loss/reference_anchor": 0.06903725862503052,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5872414112091064,
      "rewards/margins": 0.6962968111038208,
      "rewards/rejected": 0.89094477891922,
      "step": 155
    },
    {
      "drift/chosen_abs": 0.3720155358314514,
      "drift/rejected_abs": 0.11601382493972778,
      "epoch": 0.15287232772005255,
      "grad_norm": 948.0,
      "learning_rate": 2.4555046751436735e-07,
      "logits/chosen": -2.1271204948425293,
      "logits/rejected": -2.1136441230773926,
      "logps/chosen": -0.2963225841522217,
      "logps/rejected": -0.280845582485199,
      "loss": 1.3421189785003662,
      "loss/core": 1.3282365798950195,
      "loss/preference_sft": 0.2963225841522217,
      "loss/reference_anchor": 0.6644912958145142,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.718188762664795,
      "rewards/margins": 2.5581791400909424,
      "rewards/rejected": 1.160009741783142,
      "step": 160
    },
    {
      "drift/chosen_abs": 0.1847013384103775,
      "drift/rejected_abs": 0.09540971368551254,
      "epoch": 0.15764958796130418,
      "grad_norm": 14.9375,
      "learning_rate": 2.4488683414304425e-07,
      "logits/chosen": -2.129667282104492,
      "logits/rejected": -2.230297565460205,
      "logps/chosen": -0.3174905776977539,
      "logps/rejected": -0.31254857778549194,
      "loss": 0.2407360076904297,
      "loss/core": 0.23772159218788147,
      "loss/preference_sft": 0.3174905776977539,
      "loss/reference_anchor": 0.11897067725658417,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.846673607826233,
      "rewards/margins": 0.8936681747436523,
      "rewards/rejected": 0.9530051350593567,
      "step": 165
    },
    {
      "drift/chosen_abs": 0.15486204624176025,
      "drift/rejected_abs": 0.09620408713817596,
      "epoch": 0.16242684820255585,
      "grad_norm": 7.65625,
      "learning_rate": 2.4417811623641203e-07,
      "logits/chosen": -2.2738282680511475,
      "logits/rejected": -2.332890510559082,
      "logps/chosen": -0.27397435903549194,
      "logps/rejected": -0.2843550145626068,
      "loss": 0.23323550820350647,
      "loss/core": 0.230382040143013,
      "loss/preference_sft": 0.27397435903549194,
      "loss/reference_anchor": 0.11527644097805023,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.5484516620635986,
      "rewards/margins": 0.5872721076011658,
      "rewards/rejected": 0.9611795544624329,
      "step": 170
    },
    {
      "drift/chosen_abs": 0.5246831178665161,
      "drift/rejected_abs": 0.20105135440826416,
      "epoch": 0.16720410844380748,
      "grad_norm": 14.875,
      "learning_rate": 2.4342458031429113e-07,
      "logits/chosen": -1.8324203491210938,
      "logits/rejected": -1.8386204242706299,
      "logps/chosen": -0.2887090742588043,
      "logps/rejected": -0.2930302619934082,
      "loss": 2.519411563873291,
      "loss/core": 2.493882656097412,
      "loss/preference_sft": 0.2887090742588043,
      "loss/reference_anchor": 1.2475916147232056,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 5.246148109436035,
      "rewards/margins": 3.2455039024353027,
      "rewards/rejected": 2.0006444454193115,
      "step": 175
    },
    {
      "drift/chosen_abs": 0.3295663297176361,
      "drift/rejected_abs": 0.15013830363750458,
      "epoch": 0.17198136868505912,
      "grad_norm": 0.8984375,
      "learning_rate": 2.4262650975072444e-07,
      "logits/chosen": -2.1856048107147217,
      "logits/rejected": -2.3016016483306885,
      "logps/chosen": -0.28502634167671204,
      "logps/rejected": -0.29640719294548035,
      "loss": 1.1087981462478638,
      "loss/core": 1.0972480773925781,
      "loss/preference_sft": 0.28502634167671204,
      "loss/reference_anchor": 0.5489994883537292,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.2941269874572754,
      "rewards/margins": 1.8216378688812256,
      "rewards/rejected": 1.4724891185760498,
      "step": 180
    },
    {
      "drift/chosen_abs": 0.19066013395786285,
      "drift/rejected_abs": 0.07735215127468109,
      "epoch": 0.17675862892631075,
      "grad_norm": 2.484375,
      "learning_rate": 2.417842046674122e-07,
      "logits/chosen": -2.3281235694885254,
      "logits/rejected": -2.3521463871002197,
      "logps/chosen": -0.26029735803604126,
      "logps/rejected": -0.2679922580718994,
      "loss": 0.19912585616111755,
      "loss/core": 0.1966342329978943,
      "loss/preference_sft": 0.26029735803604126,
      "loss/reference_anchor": 0.09855085611343384,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9066013097763062,
      "rewards/margins": 1.1336138248443604,
      "rewards/rejected": 0.7729876041412354,
      "step": 185
    },
    {
      "drift/chosen_abs": 0.19174052774906158,
      "drift/rejected_abs": 0.09250342845916748,
      "epoch": 0.18153588916756241,
      "grad_norm": 2.0625,
      "learning_rate": 2.4089798182084843e-07,
      "logits/chosen": -2.0859601497650146,
      "logits/rejected": -2.158560276031494,
      "logps/chosen": -0.3072795271873474,
      "logps/rejected": -0.30823010206222534,
      "loss": 0.2847827076911926,
      "loss/core": 0.2813512980937958,
      "loss/preference_sft": 0.3072795271873474,
      "loss/reference_anchor": 0.14084270596504211,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.9145091772079468,
      "rewards/margins": 1.001261591911316,
      "rewards/rejected": 0.9132474660873413,
      "step": 190
    },
    {
      "drift/chosen_abs": 0.21405212581157684,
      "drift/rejected_abs": 0.13653413951396942,
      "epoch": 0.18631314940881405,
      "grad_norm": 2.34375,
      "learning_rate": 2.3996817448320195e-07,
      "logits/chosen": -2.154674530029297,
      "logits/rejected": -2.109269380569458,
      "logps/chosen": -0.28905612230300903,
      "logps/rejected": -0.2959071695804596,
      "loss": 0.47725993394851685,
      "loss/core": 0.47195786237716675,
      "loss/preference_sft": 0.28905612230300903,
      "loss/reference_anchor": 0.2361954003572464,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1385719776153564,
      "rewards/margins": 0.8496444821357727,
      "rewards/rejected": 1.288927435874939,
      "step": 195
    },
    {
      "drift/chosen_abs": 0.2820349335670471,
      "drift/rejected_abs": 0.15623252093791962,
      "epoch": 0.19109040965006568,
      "grad_norm": 118.5,
      "learning_rate": 2.3899513231698607e-07,
      "logits/chosen": -2.1225414276123047,
      "logits/rejected": -2.0437445640563965,
      "logps/chosen": -0.29230794310569763,
      "logps/rejected": -0.2899274230003357,
      "loss": 0.5702517628669739,
      "loss/core": 0.5640192031860352,
      "loss/preference_sft": 0.29230794310569763,
      "loss/reference_anchor": 0.28239911794662476,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.8195438385009766,
      "rewards/margins": 1.2614609003067017,
      "rewards/rejected": 1.5580826997756958,
      "step": 200
    },
    {
      "drift/chosen_abs": 0.2620348334312439,
      "drift/rejected_abs": 0.1418972909450531,
      "epoch": 0.19586766989131732,
      "grad_norm": 29.875,
      "learning_rate": 2.3797922124356506e-07,
      "logits/chosen": -2.029536008834839,
      "logits/rejected": -2.0266480445861816,
      "logps/chosen": -0.3911769688129425,
      "logps/rejected": -0.3083851635456085,
      "loss": 0.8301785588264465,
      "loss/core": 0.8211765289306641,
      "loss/preference_sft": 0.3911769688129425,
      "loss/reference_anchor": 0.41098326444625854,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.620229959487915,
      "rewards/margins": 1.3009237051010132,
      "rewards/rejected": 1.3193061351776123,
      "step": 205
    },
    {
      "drift/chosen_abs": 0.17181067168712616,
      "drift/rejected_abs": 0.07057416439056396,
      "epoch": 0.20064493013256898,
      "grad_norm": 1.625,
      "learning_rate": 2.3692082330554585e-07,
      "logits/chosen": -2.251944065093994,
      "logits/rejected": -2.347381114959717,
      "logps/chosen": -0.25848090648651123,
      "logps/rejected": -0.25597459077835083,
      "loss": 0.11508001387119293,
      "loss/core": 0.11342470347881317,
      "loss/preference_sft": 0.25848090648651123,
      "loss/reference_anchor": 0.056916702538728714,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7172927856445312,
      "rewards/margins": 1.0118815898895264,
      "rewards/rejected": 0.7054111957550049,
      "step": 210
    },
    {
      "drift/chosen_abs": 0.2751947343349457,
      "drift/rejected_abs": 0.17245322465896606,
      "epoch": 0.20542219037382062,
      "grad_norm": 454.0,
      "learning_rate": 2.3582033652310765e-07,
      "logits/chosen": -1.9821306467056274,
      "logits/rejected": -1.956804633140564,
      "logps/chosen": -0.2867414355278015,
      "logps/rejected": -0.2776336669921875,
      "loss": 0.5166824460029602,
      "loss/core": 0.5109937787055969,
      "loss/preference_sft": 0.2867414355278015,
      "loss/reference_anchor": 0.25575685501098633,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.7519474029541016,
      "rewards/margins": 1.0281121730804443,
      "rewards/rejected": 1.7238352298736572,
      "step": 215
    },
    {
      "drift/chosen_abs": 0.347667396068573,
      "drift/rejected_abs": 0.07685881108045578,
      "epoch": 0.21019945061507225,
      "grad_norm": 2.8125,
      "learning_rate": 2.346781747443227e-07,
      "logits/chosen": -2.0388574600219727,
      "logits/rejected": -2.1317574977874756,
      "logps/chosen": -0.2959466874599457,
      "logps/rejected": -0.2806416153907776,
      "loss": 1.4209736585617065,
      "loss/core": 1.4063067436218262,
      "loss/preference_sft": 0.2959466874599457,
      "loss/reference_anchor": 0.7037451267242432,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.4755759239196777,
      "rewards/margins": 2.707566022872925,
      "rewards/rejected": 0.7680096626281738,
      "step": 220
    },
    {
      "drift/chosen_abs": 0.3234902322292328,
      "drift/rejected_abs": 0.10148955881595612,
      "epoch": 0.2149767108563239,
      "grad_norm": 188.0,
      "learning_rate": 2.3349476748952508e-07,
      "logits/chosen": -2.1488888263702393,
      "logits/rejected": -2.237083911895752,
      "logps/chosen": -0.2739250063896179,
      "logps/rejected": -0.27243196964263916,
      "loss": 0.5956701040267944,
      "loss/core": 0.5892229676246643,
      "loss/preference_sft": 0.2739250063896179,
      "loss/reference_anchor": 0.2949652075767517,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.233933210372925,
      "rewards/margins": 2.221308469772339,
      "rewards/rejected": 1.0126246213912964,
      "step": 225
    },
    {
      "drift/chosen_abs": 0.36533501744270325,
      "drift/rejected_abs": 0.1348879039287567,
      "epoch": 0.21975397109757555,
      "grad_norm": 844.0,
      "learning_rate": 2.3227055978978545e-07,
      "logits/chosen": -2.0082271099090576,
      "logits/rejected": -2.0484988689422607,
      "logps/chosen": -0.25067609548568726,
      "logps/rejected": -0.2521665692329407,
      "loss": 1.3585067987442017,
      "loss/core": 1.3445472717285156,
      "loss/preference_sft": 0.25067609548568726,
      "loss/reference_anchor": 0.6729066967964172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.6500682830810547,
      "rewards/margins": 2.3029842376708984,
      "rewards/rejected": 1.3470841646194458,
      "step": 230
    },
    {
      "drift/chosen_abs": 0.2073029726743698,
      "drift/rejected_abs": 0.12087440490722656,
      "epoch": 0.2245312313388272,
      "grad_norm": 1.5703125,
      "learning_rate": 2.3100601201955321e-07,
      "logits/chosen": -2.326741933822632,
      "logits/rejected": -2.3083624839782715,
      "logps/chosen": -0.25950273871421814,
      "logps/rejected": -0.2679215967655182,
      "loss": 0.7174883484840393,
      "loss/core": 0.709865391254425,
      "loss/preference_sft": 0.25950273871421814,
      "loss/reference_anchor": 0.35520219802856445,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.0721540451049805,
      "rewards/margins": 0.8642433285713196,
      "rewards/rejected": 1.2079106569290161,
      "step": 235
    },
    {
      "drift/chosen_abs": 0.339974582195282,
      "drift/rejected_abs": 0.19081738591194153,
      "epoch": 0.22930849158007882,
      "grad_norm": 14.9375,
      "learning_rate": 2.2970159972352864e-07,
      "logits/chosen": -2.121175527572632,
      "logits/rejected": -2.1394689083099365,
      "logps/chosen": -0.27825844287872314,
      "logps/rejected": -0.29647335410118103,
      "loss": 1.2494022846221924,
      "loss/core": 1.236472249031067,
      "loss/preference_sft": 0.27825844287872314,
      "loss/reference_anchor": 0.6186779737472534,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.3997459411621094,
      "rewards/margins": 1.4949043989181519,
      "rewards/rejected": 1.9048411846160889,
      "step": 240
    },
    {
      "drift/chosen_abs": 0.3295367658138275,
      "drift/rejected_abs": 0.16237182915210724,
      "epoch": 0.23408575182133046,
      "grad_norm": 552.0,
      "learning_rate": 2.2835781343782966e-07,
      "logits/chosen": -1.8566938638687134,
      "logits/rejected": -1.875113844871521,
      "logps/chosen": -0.2936326861381531,
      "logps/rejected": -0.29784101247787476,
      "loss": 1.379624843597412,
      "loss/core": 1.3653779029846191,
      "loss/preference_sft": 0.2936326861381531,
      "loss/reference_anchor": 0.6829947829246521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.2930209636688232,
      "rewards/margins": 1.6766458749771118,
      "rewards/rejected": 1.6163749694824219,
      "step": 245
    },
    {
      "drift/chosen_abs": 0.15949814021587372,
      "drift/rejected_abs": 0.10824527591466904,
      "epoch": 0.23886301206258212,
      "grad_norm": 0.53125,
      "learning_rate": 2.2697515850552152e-07,
      "logits/chosen": -2.3425869941711426,
      "logits/rejected": -2.325498104095459,
      "logps/chosen": -0.2886843681335449,
      "logps/rejected": -0.2921077609062195,
      "loss": 0.29327505826950073,
      "loss/core": 0.28979676961898804,
      "loss/preference_sft": 0.2886843681335449,
      "loss/reference_anchor": 0.14504632353782654,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.5941321849822998,
      "rewards/margins": 0.5128880739212036,
      "rewards/rejected": 1.0812442302703857,
      "step": 250
    },
    {
      "drift/chosen_abs": 0.14964722096920013,
      "drift/rejected_abs": 0.05588113144040108,
      "epoch": 0.24364027230383375,
      "grad_norm": 0.63671875,
      "learning_rate": 2.2555415488657775e-07,
      "logits/chosen": -2.117321729660034,
      "logits/rejected": -2.180022716522217,
      "logps/chosen": -0.3025610148906708,
      "logps/rejected": -0.30650925636291504,
      "loss": 0.14917774498462677,
      "loss/core": 0.14709773659706116,
      "loss/preference_sft": 0.3025610148906708,
      "loss/reference_anchor": 0.07374367862939835,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.4962538480758667,
      "rewards/margins": 0.9412781000137329,
      "rewards/rejected": 0.5549757480621338,
      "step": 255
    },
    {
      "drift/chosen_abs": 0.23837116360664368,
      "drift/rejected_abs": 0.1084081158041954,
      "epoch": 0.2484175325450854,
      "grad_norm": 4.59375,
      "learning_rate": 2.240953369623447e-07,
      "logits/chosen": -2.271860122680664,
      "logits/rejected": -2.326420307159424,
      "logps/chosen": -0.2747231125831604,
      "logps/rejected": -0.276181697845459,
      "loss": 0.3998209536075592,
      "loss/core": 0.3953119218349457,
      "loss/preference_sft": 0.2747231125831604,
      "loss/reference_anchor": 0.19797541201114655,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3820481300354004,
      "rewards/margins": 1.2980557680130005,
      "rewards/rejected": 1.0839922428131104,
      "step": 260
    },
    {
      "drift/chosen_abs": 0.14227482676506042,
      "drift/rejected_abs": 0.07095784693956375,
      "epoch": 0.25319479278633705,
      "grad_norm": 1.3828125,
      "learning_rate": 2.225992533345824e-07,
      "logits/chosen": -2.364943265914917,
      "logits/rejected": -2.4003348350524902,
      "logps/chosen": -0.2837630808353424,
      "logps/rejected": -0.28678828477859497,
      "loss": 0.1420137584209442,
      "loss/core": 0.14004194736480713,
      "loss/preference_sft": 0.2837630808353424,
      "loss/reference_anchor": 0.07021477073431015,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.422141671180725,
      "rewards/margins": 0.7143291234970093,
      "rewards/rejected": 0.7078123688697815,
      "step": 265
    },
    {
      "drift/chosen_abs": 0.18406164646148682,
      "drift/rejected_abs": 0.0678694099187851,
      "epoch": 0.2579720530275887,
      "grad_norm": 12.5,
      "learning_rate": 2.210664666191579e-07,
      "logits/chosen": -2.1580262184143066,
      "logits/rejected": -2.2362945079803467,
      "logps/chosen": -0.3155515789985657,
      "logps/rejected": -0.29309743642807007,
      "loss": 0.20326463878154755,
      "loss/core": 0.20061977207660675,
      "loss/preference_sft": 0.3155515789985657,
      "loss/reference_anchor": 0.10068801790475845,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.839700698852539,
      "rewards/margins": 1.163073182106018,
      "rewards/rejected": 0.6766273379325867,
      "step": 270
    },
    {
      "drift/chosen_abs": 0.24695925414562225,
      "drift/rejected_abs": 0.11996563524007797,
      "epoch": 0.2627493132688403,
      "grad_norm": 17.25,
      "learning_rate": 2.1949755323446848e-07,
      "logits/chosen": -2.250136375427246,
      "logits/rejected": -2.232349157333374,
      "logps/chosen": -0.2562440037727356,
      "logps/rejected": -0.2609252333641052,
      "loss": 0.5029779672622681,
      "loss/core": 0.49748414754867554,
      "loss/preference_sft": 0.2562440037727356,
      "loss/reference_anchor": 0.2490653693675995,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4680981636047363,
      "rewards/margins": 1.2709671258926392,
      "rewards/rejected": 1.1971310377120972,
      "step": 275
    },
    {
      "drift/chosen_abs": 0.2423405647277832,
      "drift/rejected_abs": 0.13960115611553192,
      "epoch": 0.26752657351009196,
      "grad_norm": 27.375,
      "learning_rate": 2.1789310318467426e-07,
      "logits/chosen": -1.8027242422103882,
      "logits/rejected": -1.8857799768447876,
      "logps/chosen": -0.3176387846469879,
      "logps/rejected": -0.3207781910896301,
      "loss": 0.3058789372444153,
      "loss/core": 0.3022173047065735,
      "loss/preference_sft": 0.3176387846469879,
      "loss/reference_anchor": 0.1513180285692215,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.4204940795898438,
      "rewards/margins": 1.0279791355133057,
      "rewards/rejected": 1.3925148248672485,
      "step": 280
    },
    {
      "drift/chosen_abs": 0.2865580916404724,
      "drift/rejected_abs": 0.1650657057762146,
      "epoch": 0.2723038337513436,
      "grad_norm": 84.0,
      "learning_rate": 2.1625371983782182e-07,
      "logits/chosen": -2.08091402053833,
      "logits/rejected": -2.043867826461792,
      "logps/chosen": -0.2672872543334961,
      "logps/rejected": -0.2768842577934265,
      "loss": 0.6620380282402039,
      "loss/core": 0.6549504995346069,
      "loss/preference_sft": 0.2672872543334961,
      "loss/reference_anchor": 0.32764729857444763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.865417242050171,
      "rewards/margins": 1.2915067672729492,
      "rewards/rejected": 1.5739107131958008,
      "step": 285
    },
    {
      "drift/chosen_abs": 0.20399828255176544,
      "drift/rejected_abs": 0.08803847432136536,
      "epoch": 0.27708109399259523,
      "grad_norm": 20.25,
      "learning_rate": 2.14580019698942e-07,
      "logits/chosen": -2.1715924739837646,
      "logits/rejected": -2.2407965660095215,
      "logps/chosen": -0.2938459813594818,
      "logps/rejected": -0.2942553758621216,
      "loss": 0.3727453947067261,
      "loss/core": 0.3684656322002411,
      "loss/preference_sft": 0.2938459813594818,
      "loss/reference_anchor": 0.18460234999656677,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0391552448272705,
      "rewards/margins": 1.1640946865081787,
      "rewards/rejected": 0.8750606775283813,
      "step": 290
    },
    {
      "drift/chosen_abs": 0.22580035030841827,
      "drift/rejected_abs": 0.16777771711349487,
      "epoch": 0.28185835423384686,
      "grad_norm": 652.0,
      "learning_rate": 2.1287263217820773e-07,
      "logits/chosen": -2.1450181007385254,
      "logits/rejected": -2.1244781017303467,
      "logps/chosen": -0.2750265300273895,
      "logps/rejected": -0.2840802073478699,
      "loss": 0.6499078273773193,
      "loss/core": 0.6429253816604614,
      "loss/preference_sft": 0.2750265300273895,
      "loss/reference_anchor": 0.3216193616390228,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.2560317516326904,
      "rewards/margins": 0.5812076330184937,
      "rewards/rejected": 1.6748244762420654,
      "step": 295
    },
    {
      "drift/chosen_abs": 0.27815431356430054,
      "drift/rejected_abs": 0.13376235961914062,
      "epoch": 0.28663561447509855,
      "grad_norm": 11.5,
      "learning_rate": 2.111321993542385e-07,
      "logits/chosen": -1.9923107624053955,
      "logits/rejected": -1.9791291952133179,
      "logps/chosen": -0.2681702673435211,
      "logps/rejected": -0.28962358832359314,
      "loss": 0.390531450510025,
      "loss/core": 0.38612592220306396,
      "loss/preference_sft": 0.2681702673435211,
      "loss/reference_anchor": 0.19345811009407043,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.778730869293213,
      "rewards/margins": 1.4537692070007324,
      "rewards/rejected": 1.3249620199203491,
      "step": 300
    },
    {
      "drift/chosen_abs": 0.18262216448783875,
      "drift/rejected_abs": 0.1317087560892105,
      "epoch": 0.2914128747163502,
      "grad_norm": 55.0,
      "learning_rate": 2.0935937573264096e-07,
      "logits/chosen": -2.0926907062530518,
      "logits/rejected": -2.105894088745117,
      "logps/chosen": -0.28904959559440613,
      "logps/rejected": -0.30133944749832153,
      "loss": 0.2560952305793762,
      "loss/core": 0.2529827654361725,
      "loss/preference_sft": 0.28904959559440613,
      "loss/reference_anchor": 0.1267165243625641,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8257789611816406,
      "rewards/margins": 0.510686993598938,
      "rewards/rejected": 1.315091848373413,
      "step": 305
    },
    {
      "drift/chosen_abs": 0.20964379608631134,
      "drift/rejected_abs": 0.13515421748161316,
      "epoch": 0.2961901349576018,
      "grad_norm": 7.53125,
      "learning_rate": 2.0755482799987596e-07,
      "logits/chosen": -2.020278215408325,
      "logits/rejected": -1.9860916137695312,
      "logps/chosen": -0.29890328645706177,
      "logps/rejected": -0.3041743338108063,
      "loss": 0.5346760749816895,
      "loss/core": 0.5287860035896301,
      "loss/preference_sft": 0.29890328645706177,
      "loss/reference_anchor": 0.2646165192127228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.094669818878174,
      "rewards/margins": 0.7476687431335449,
      "rewards/rejected": 1.347001075744629,
      "step": 310
    },
    {
      "drift/chosen_abs": 0.15830251574516296,
      "drift/rejected_abs": 0.09903346002101898,
      "epoch": 0.30096739519885346,
      "grad_norm": 3.578125,
      "learning_rate": 2.0571923477254526e-07,
      "logits/chosen": -2.271230697631836,
      "logits/rejected": -2.2453927993774414,
      "logps/chosen": -0.29073256254196167,
      "logps/rejected": -0.290340393781662,
      "loss": 0.21692290902137756,
      "loss/core": 0.21419718861579895,
      "loss/preference_sft": 0.29073256254196167,
      "loss/reference_anchor": 0.10721520334482193,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.582291603088379,
      "rewards/margins": 0.6015550494194031,
      "rewards/rejected": 0.9807365536689758,
      "step": 315
    },
    {
      "drift/chosen_abs": 0.28570857644081116,
      "drift/rejected_abs": 0.19199606776237488,
      "epoch": 0.3057446554401051,
      "grad_norm": 2.859375,
      "learning_rate": 2.038532863421914e-07,
      "logits/chosen": -2.1323790550231934,
      "logits/rejected": -2.0531458854675293,
      "logps/chosen": -0.29020506143569946,
      "logps/rejected": -0.2748698890209198,
      "loss": 0.6716278195381165,
      "loss/core": 0.6644001007080078,
      "loss/preference_sft": 0.29020506143569946,
      "loss/reference_anchor": 0.3323640823364258,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.857085704803467,
      "rewards/margins": 0.9378870129585266,
      "rewards/rejected": 1.919198751449585,
      "step": 320
    },
    {
      "drift/chosen_abs": 0.15099409222602844,
      "drift/rejected_abs": 0.15222987532615662,
      "epoch": 0.31052191568135673,
      "grad_norm": 6.4375,
      "learning_rate": 2.0195768441570726e-07,
      "logits/chosen": -1.9938151836395264,
      "logits/rejected": -2.049948215484619,
      "logps/chosen": -0.2915424704551697,
      "logps/rejected": -0.28644031286239624,
      "loss": 0.6587117910385132,
      "loss/core": 0.6516103148460388,
      "loss/preference_sft": 0.2915424704551697,
      "loss/reference_anchor": 0.3259239196777344,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.5096241235733032,
      "rewards/margins": -0.008662676438689232,
      "rewards/rejected": 1.5182867050170898,
      "step": 325
    },
    {
      "drift/chosen_abs": 0.2591181993484497,
      "drift/rejected_abs": 0.11437433958053589,
      "epoch": 0.31529917592260837,
      "grad_norm": 18.75,
      "learning_rate": 2.0003314185145307e-07,
      "logits/chosen": -2.145712375640869,
      "logits/rejected": -2.0909924507141113,
      "logps/chosen": -0.28042882680892944,
      "logps/rejected": -0.2842772603034973,
      "loss": 0.46702370047569275,
      "loss/core": 0.46183985471725464,
      "loss/preference_sft": 0.28042882680892944,
      "loss/reference_anchor": 0.23115257918834686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.5909152030944824,
      "rewards/margins": 1.4475888013839722,
      "rewards/rejected": 1.1433266401290894,
      "step": 330
    },
    {
      "drift/chosen_abs": 0.2697688341140747,
      "drift/rejected_abs": 0.1486051231622696,
      "epoch": 0.32007643616386,
      "grad_norm": 26.5,
      "learning_rate": 1.9808038239117913e-07,
      "logits/chosen": -1.9927841424942017,
      "logits/rejected": -1.9601898193359375,
      "logps/chosen": -0.2884038984775543,
      "logps/rejected": -0.2860269844532013,
      "loss": 0.4839557111263275,
      "loss/core": 0.4785878658294678,
      "loss/preference_sft": 0.2884038984775543,
      "loss/reference_anchor": 0.23955455422401428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.6973326206207275,
      "rewards/margins": 1.2938122749328613,
      "rewards/rejected": 1.4035199880599976,
      "step": 335
    },
    {
      "drift/chosen_abs": 0.16600827872753143,
      "drift/rejected_abs": 0.08406911045312881,
      "epoch": 0.3248536964051117,
      "grad_norm": 8.5,
      "learning_rate": 1.9610014038785646e-07,
      "logits/chosen": -2.222121477127075,
      "logits/rejected": -2.2075448036193848,
      "logps/chosen": -0.2964716851711273,
      "logps/rejected": -0.30573147535324097,
      "loss": 0.17239664494991302,
      "loss/core": 0.17009879648685455,
      "loss/preference_sft": 0.2964716851711273,
      "loss/reference_anchor": 0.08524533361196518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6584606170654297,
      "rewards/margins": 0.8196709752082825,
      "rewards/rejected": 0.8387895822525024,
      "step": 340
    },
    {
      "drift/chosen_abs": 0.2879965305328369,
      "drift/rejected_abs": 0.2307172268629074,
      "epoch": 0.3296309566463633,
      "grad_norm": 20.75,
      "learning_rate": 1.9409316052951657e-07,
      "logits/chosen": -1.9839036464691162,
      "logits/rejected": -2.0710253715515137,
      "logps/chosen": -0.2770405411720276,
      "logps/rejected": -0.2928730249404907,
      "loss": 1.3981668949127197,
      "loss/core": 1.3837751150131226,
      "loss/preference_sft": 0.2770405411720276,
      "loss/reference_anchor": 0.6918903589248657,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.87996506690979,
      "rewards/margins": 0.5738177299499512,
      "rewards/rejected": 2.3061470985412598,
      "step": 345
    },
    {
      "drift/chosen_abs": 0.3735573887825012,
      "drift/rejected_abs": 0.1227908581495285,
      "epoch": 0.33440821688761496,
      "grad_norm": 75.5,
      "learning_rate": 1.920601975592047e-07,
      "logits/chosen": -2.171405076980591,
      "logits/rejected": -2.1451284885406494,
      "logps/chosen": -0.2867693305015564,
      "logps/rejected": -0.29171350598335266,
      "loss": 1.53303062915802,
      "loss/core": 1.5172761678695679,
      "loss/preference_sft": 0.2867693305015564,
      "loss/reference_anchor": 0.7590527534484863,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.7349324226379395,
      "rewards/margins": 2.521451473236084,
      "rewards/rejected": 1.2134809494018555,
      "step": 350
    },
    {
      "drift/chosen_abs": 0.1963830143213272,
      "drift/rejected_abs": 0.07833238691091537,
      "epoch": 0.3391854771288666,
      "grad_norm": 1.8203125,
      "learning_rate": 1.900020159911519e-07,
      "logits/chosen": -2.0924370288848877,
      "logits/rejected": -2.159611701965332,
      "logps/chosen": -0.3114902377128601,
      "logps/rejected": -0.31218671798706055,
      "loss": 0.28884080052375793,
      "loss/core": 0.2853604257106781,
      "loss/preference_sft": 0.3114902377128601,
      "loss/reference_anchor": 0.14286813139915466,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9624801874160767,
      "rewards/margins": 1.1923706531524658,
      "rewards/rejected": 0.7701095342636108,
      "step": 355
    },
    {
      "drift/chosen_abs": 0.1754159778356552,
      "drift/rejected_abs": 0.18023176491260529,
      "epoch": 0.34396273737011823,
      "grad_norm": 21.5,
      "learning_rate": 1.879193898232725e-07,
      "logits/chosen": -2.1387431621551514,
      "logits/rejected": -2.1139893531799316,
      "logps/chosen": -0.3140478730201721,
      "logps/rejected": -0.3199792206287384,
      "loss": 0.39963623881340027,
      "loss/core": 0.39505690336227417,
      "loss/preference_sft": 0.3140478730201721,
      "loss/reference_anchor": 0.19756412506103516,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.754159927368164,
      "rewards/margins": -0.04186423867940903,
      "rewards/rejected": 1.796023964881897,
      "step": 360
    },
    {
      "drift/chosen_abs": 0.37004557251930237,
      "drift/rejected_abs": 0.08763270825147629,
      "epoch": 0.34873999761136987,
      "grad_norm": 59.5,
      "learning_rate": 1.8581310224609496e-07,
      "logits/chosen": -2.1298484802246094,
      "logits/rejected": -2.202394962310791,
      "logps/chosen": -0.2909151315689087,
      "logps/rejected": -0.2845023572444916,
      "loss": 1.6612498760223389,
      "loss/core": 1.6442168951034546,
      "loss/preference_sft": 0.2909151315689087,
      "loss/reference_anchor": 0.8225665092468262,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.698554277420044,
      "rewards/margins": 2.8230626583099365,
      "rewards/rejected": 0.8754919767379761,
      "step": 365
    },
    {
      "drift/chosen_abs": 0.29484832286834717,
      "drift/rejected_abs": 0.12339384853839874,
      "epoch": 0.3535172578526215,
      "grad_norm": 30.125,
      "learning_rate": 1.8368394534823635e-07,
      "logits/chosen": -2.0275933742523193,
      "logits/rejected": -2.0372328758239746,
      "logps/chosen": -0.2683495581150055,
      "logps/rejected": -0.2806262671947479,
      "loss": 0.5110516548156738,
      "loss/core": 0.5054527521133423,
      "loss/preference_sft": 0.2683495581150055,
      "loss/reference_anchor": 0.2531072795391083,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.9470877647399902,
      "rewards/margins": 1.7142242193222046,
      "rewards/rejected": 1.2328637838363647,
      "step": 370
    },
    {
      "drift/chosen_abs": 0.4893800616264343,
      "drift/rejected_abs": 0.23103968799114227,
      "epoch": 0.35829451809387314,
      "grad_norm": 10.0625,
      "learning_rate": 1.8153271981852968e-07,
      "logits/chosen": -2.004993200302124,
      "logits/rejected": -1.9773603677749634,
      "logps/chosen": -0.3077290654182434,
      "logps/rejected": -0.2921420633792877,
      "loss": 3.063831329345703,
      "loss/core": 3.032869338989258,
      "loss/preference_sft": 0.3077290654182434,
      "loss/reference_anchor": 1.5173249244689941,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.893084526062012,
      "rewards/margins": 2.5864005088806152,
      "rewards/rejected": 2.3066840171813965,
      "step": 375
    },
    {
      "drift/chosen_abs": 0.22893986105918884,
      "drift/rejected_abs": 0.13307783007621765,
      "epoch": 0.36307177833512483,
      "grad_norm": 156.0,
      "learning_rate": 1.7936023464491812e-07,
      "logits/chosen": -2.0798957347869873,
      "logits/rejected": -2.190671920776367,
      "logps/chosen": -0.2794080078601837,
      "logps/rejected": -0.27889859676361084,
      "loss": 0.7344735860824585,
      "loss/core": 0.7266415357589722,
      "loss/preference_sft": 0.2794080078601837,
      "loss/reference_anchor": 0.3636623024940491,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2871289253234863,
      "rewards/margins": 0.9574325680732727,
      "rewards/rejected": 1.3296962976455688,
      "step": 380
    },
    {
      "drift/chosen_abs": 0.166697159409523,
      "drift/rejected_abs": 0.10592110455036163,
      "epoch": 0.36784903857637646,
      "grad_norm": 21.0,
      "learning_rate": 1.7716730681022723e-07,
      "logits/chosen": -2.1135611534118652,
      "logits/rejected": -2.0744059085845947,
      "logps/chosen": -0.31136012077331543,
      "logps/rejected": -0.2861802577972412,
      "loss": 0.23299483954906464,
      "loss/core": 0.23006677627563477,
      "loss/preference_sft": 0.31136012077331543,
      "loss/reference_anchor": 0.11526670306921005,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.6641048192977905,
      "rewards/margins": 0.6075918078422546,
      "rewards/rejected": 1.0565129518508911,
      "step": 385
    },
    {
      "drift/chosen_abs": 0.22516271471977234,
      "drift/rejected_abs": 0.1247841864824295,
      "epoch": 0.3726262988176281,
      "grad_norm": 1.9453125,
      "learning_rate": 1.7495476098493152e-07,
      "logits/chosen": -2.1953511238098145,
      "logits/rejected": -2.2125370502471924,
      "logps/chosen": -0.27903926372528076,
      "logps/rejected": -0.28518766164779663,
      "loss": 0.3761298954486847,
      "loss/core": 0.37184834480285645,
      "loss/preference_sft": 0.27903926372528076,
      "loss/reference_anchor": 0.18617263436317444,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.250232219696045,
      "rewards/margins": 1.0177348852157593,
      "rewards/rejected": 1.232497215270996,
      "step": 390
    },
    {
      "drift/chosen_abs": 0.3017219603061676,
      "drift/rejected_abs": 0.16852721571922302,
      "epoch": 0.37740355905887973,
      "grad_norm": 3.484375,
      "learning_rate": 1.7272342921702937e-07,
      "logits/chosen": -2.087266445159912,
      "logits/rejected": -2.1111271381378174,
      "logps/chosen": -0.3051111102104187,
      "logps/rejected": -0.30405479669570923,
      "loss": 1.1819294691085815,
      "loss/core": 1.1696186065673828,
      "loss/preference_sft": 0.3051111102104187,
      "loss/reference_anchor": 0.5850434303283691,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.0164356231689453,
      "rewards/margins": 1.3323934078216553,
      "rewards/rejected": 1.6840423345565796,
      "step": 395
    },
    {
      "drift/chosen_abs": 0.27575579285621643,
      "drift/rejected_abs": 0.20994627475738525,
      "epoch": 0.38218081930013137,
      "grad_norm": 2.453125,
      "learning_rate": 1.7047415061914393e-07,
      "logits/chosen": -2.057664632797241,
      "logits/rejected": -2.212043046951294,
      "logps/chosen": -0.2742888331413269,
      "logps/rejected": -0.2956988215446472,
      "loss": 1.1588386297225952,
      "loss/core": 1.1468188762664795,
      "loss/preference_sft": 0.2742888331413269,
      "loss/reference_anchor": 0.5735639333724976,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.7565925121307373,
      "rewards/margins": 0.6574581861495972,
      "rewards/rejected": 2.0991344451904297,
      "step": 400
    },
    {
      "drift/chosen_abs": 0.25579676032066345,
      "drift/rejected_abs": 0.1272418051958084,
      "epoch": 0.386958079541383,
      "grad_norm": 202.0,
      "learning_rate": 1.6820777105296707e-07,
      "logits/chosen": -2.1076152324676514,
      "logits/rejected": -2.20607590675354,
      "logps/chosen": -0.3273892402648926,
      "logps/rejected": -0.28913551568984985,
      "loss": 0.8974112272262573,
      "loss/core": 0.8878701329231262,
      "loss/preference_sft": 0.3273892402648926,
      "loss/reference_anchor": 0.4443127512931824,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.552988052368164,
      "rewards/margins": 1.2856799364089966,
      "rewards/rejected": 1.2673081159591675,
      "step": 405
    },
    {
      "drift/chosen_abs": 0.2772047519683838,
      "drift/rejected_abs": 0.1283237189054489,
      "epoch": 0.39173533978263464,
      "grad_norm": 1.5703125,
      "learning_rate": 1.6592514281116553e-07,
      "logits/chosen": -2.0812337398529053,
      "logits/rejected": -2.0970664024353027,
      "logps/chosen": -0.26817846298217773,
      "logps/rejected": -0.2727079689502716,
      "loss": 0.5803694725036621,
      "loss/core": 0.574087381362915,
      "loss/preference_sft": 0.26817846298217773,
      "loss/reference_anchor": 0.2872823476791382,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.771658420562744,
      "rewards/margins": 1.4890962839126587,
      "rewards/rejected": 1.2825623750686646,
      "step": 410
    },
    {
      "drift/chosen_abs": 0.23107464611530304,
      "drift/rejected_abs": 0.0726277232170105,
      "epoch": 0.3965126000238863,
      "grad_norm": 10.5625,
      "learning_rate": 1.636271242968684e-07,
      "logits/chosen": -2.0305263996124268,
      "logits/rejected": -2.0474953651428223,
      "logps/chosen": -0.30018436908721924,
      "logps/rejected": -0.2916948199272156,
      "loss": 0.43483519554138184,
      "loss/core": 0.42992615699768066,
      "loss/preference_sft": 0.30018436908721924,
      "loss/reference_anchor": 0.21543259918689728,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.310746431350708,
      "rewards/margins": 1.701195478439331,
      "rewards/rejected": 0.6095508933067322,
      "step": 415
    },
    {
      "drift/chosen_abs": 0.32632407546043396,
      "drift/rejected_abs": 0.18786516785621643,
      "epoch": 0.40128986026513797,
      "grad_norm": 7.3125,
      "learning_rate": 1.6131457970085684e-07,
      "logits/chosen": -1.924475908279419,
      "logits/rejected": -1.9443466663360596,
      "logps/chosen": -0.2839311957359314,
      "logps/rejected": -0.2862430214881897,
      "loss": 0.9467588663101196,
      "loss/core": 0.9368128776550293,
      "loss/preference_sft": 0.2839311957359314,
      "loss/reference_anchor": 0.4689028859138489,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.2614026069641113,
      "rewards/margins": 1.3844013214111328,
      "rewards/rejected": 1.8770010471343994,
      "step": 420
    },
    {
      "drift/chosen_abs": 0.29126811027526855,
      "drift/rejected_abs": 0.19718101620674133,
      "epoch": 0.4060671205063896,
      "grad_norm": 98.0,
      "learning_rate": 1.5898837867657727e-07,
      "logits/chosen": -1.9464651346206665,
      "logits/rejected": -1.9939228296279907,
      "logps/chosen": -0.2907729744911194,
      "logps/rejected": -0.27748432755470276,
      "loss": 0.9655182957649231,
      "loss/core": 0.9553742408752441,
      "loss/preference_sft": 0.2907729744911194,
      "loss/reference_anchor": 0.4781121611595154,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9124670028686523,
      "rewards/margins": 0.9434226751327515,
      "rewards/rejected": 1.9690440893173218,
      "step": 425
    },
    {
      "drift/chosen_abs": 0.2520788609981537,
      "drift/rejected_abs": 0.08256536722183228,
      "epoch": 0.41084438074764124,
      "grad_norm": 9.3125,
      "learning_rate": 1.5664939601310023e-07,
      "logits/chosen": -2.176443099975586,
      "logits/rejected": -2.245333194732666,
      "logps/chosen": -0.30696016550064087,
      "logps/rejected": -0.3062490224838257,
      "loss": 0.846347451210022,
      "loss/core": 0.8373504877090454,
      "loss/preference_sft": 0.30696016550064087,
      "loss/reference_anchor": 0.41915664076805115,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5197196006774902,
      "rewards/margins": 1.6960992813110352,
      "rewards/rejected": 0.8236203193664551,
      "step": 430
    },
    {
      "drift/chosen_abs": 0.24701747298240662,
      "drift/rejected_abs": 0.12345242500305176,
      "epoch": 0.41562164098889287,
      "grad_norm": 3.65625,
      "learning_rate": 1.5429851130614807e-07,
      "logits/chosen": -2.1048972606658936,
      "logits/rejected": -2.033947467803955,
      "logps/chosen": -0.2840856909751892,
      "logps/rejected": -0.2637600898742676,
      "loss": 0.45461979508399963,
      "loss/core": 0.4495505392551422,
      "loss/preference_sft": 0.2840856909751892,
      "loss/reference_anchor": 0.2250523865222931,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.468121290206909,
      "rewards/margins": 1.2350847721099854,
      "rewards/rejected": 1.233036756515503,
      "step": 435
    },
    {
      "drift/chosen_abs": 0.21977201104164124,
      "drift/rejected_abs": 0.0888163223862648,
      "epoch": 0.4203989012301445,
      "grad_norm": 88.5,
      "learning_rate": 1.51936608627315e-07,
      "logits/chosen": -1.996376395225525,
      "logits/rejected": -2.0227012634277344,
      "logps/chosen": -0.31263792514801025,
      "logps/rejected": -0.3157431185245514,
      "loss": 0.3650742173194885,
      "loss/core": 0.3608359098434448,
      "loss/preference_sft": 0.31263792514801025,
      "loss/reference_anchor": 0.1806512176990509,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1961891651153564,
      "rewards/margins": 1.3100742101669312,
      "rewards/rejected": 0.8861148953437805,
      "step": 440
    },
    {
      "drift/chosen_abs": 0.3700529634952545,
      "drift/rejected_abs": 0.17902246117591858,
      "epoch": 0.42517616147139614,
      "grad_norm": 180.0,
      "learning_rate": 1.4956457619160375e-07,
      "logits/chosen": -2.0445446968078613,
      "logits/rejected": -2.02923321723938,
      "logps/chosen": -0.2807472348213196,
      "logps/rejected": -0.26079219579696655,
      "loss": 1.3479831218719482,
      "loss/core": 1.3340699672698975,
      "loss/preference_sft": 0.2807472348213196,
      "loss/reference_anchor": 0.6675775647163391,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.700012683868408,
      "rewards/margins": 1.9215444326400757,
      "rewards/rejected": 1.778468132019043,
      "step": 445
    },
    {
      "drift/chosen_abs": 0.19167892634868622,
      "drift/rejected_abs": 0.08485953509807587,
      "epoch": 0.4299534217126478,
      "grad_norm": 5.6875,
      "learning_rate": 1.471833060234044e-07,
      "logits/chosen": -2.1029083728790283,
      "logits/rejected": -2.1657981872558594,
      "logps/chosen": -0.30128949880599976,
      "logps/rejected": -0.3000229001045227,
      "loss": 0.2482064664363861,
      "loss/core": 0.24514798820018768,
      "loss/preference_sft": 0.30128949880599976,
      "loss/reference_anchor": 0.12279714643955231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9167217016220093,
      "rewards/margins": 1.0688178539276123,
      "rewards/rejected": 0.847903847694397,
      "step": 450
    },
    {
      "drift/chosen_abs": 0.2169889211654663,
      "drift/rejected_abs": 0.13156619668006897,
      "epoch": 0.4347306819538994,
      "grad_norm": 46.5,
      "learning_rate": 1.4479369362104037e-07,
      "logits/chosen": -2.13124418258667,
      "logits/rejected": -2.2444398403167725,
      "logps/chosen": -0.2661250829696655,
      "logps/rejected": -0.28118157386779785,
      "loss": 0.36354073882102966,
      "loss/core": 0.35940924286842346,
      "loss/preference_sft": 0.2661250829696655,
      "loss/reference_anchor": 0.17996393144130707,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.1693220138549805,
      "rewards/margins": 0.8696813583374023,
      "rewards/rejected": 1.2996408939361572,
      "step": 455
    },
    {
      "drift/chosen_abs": 0.20733904838562012,
      "drift/rejected_abs": 0.09261975437402725,
      "epoch": 0.4395079421951511,
      "grad_norm": 3.65625,
      "learning_rate": 1.4239663762000817e-07,
      "logits/chosen": -2.1311872005462646,
      "logits/rejected": -2.0629684925079346,
      "logps/chosen": -0.29252859950065613,
      "logps/rejected": -0.2915221154689789,
      "loss": 0.34754329919815063,
      "loss/core": 0.3435174524784088,
      "loss/preference_sft": 0.29252859950065613,
      "loss/reference_anchor": 0.17203818261623383,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.070737361907959,
      "rewards/margins": 1.146690845489502,
      "rewards/rejected": 0.924046516418457,
      "step": 460
    },
    {
      "drift/chosen_abs": 0.25771060585975647,
      "drift/rejected_abs": 0.14189331233501434,
      "epoch": 0.44428520243640274,
      "grad_norm": 96.5,
      "learning_rate": 1.3999303945503747e-07,
      "logits/chosen": -2.1602511405944824,
      "logits/rejected": -2.2427048683166504,
      "logps/chosen": -0.3139217793941498,
      "logps/rejected": -0.3135758340358734,
      "loss": 0.8517921566963196,
      "loss/core": 0.842727780342102,
      "loss/preference_sft": 0.3139217793941498,
      "loss/reference_anchor": 0.4218299984931946,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.5764236450195312,
      "rewards/margins": 1.158347487449646,
      "rewards/rejected": 1.4180762767791748,
      "step": 465
    },
    {
      "drift/chosen_abs": 0.15380379557609558,
      "drift/rejected_abs": 0.10157140344381332,
      "epoch": 0.4490624626776544,
      "grad_norm": 4.71875,
      "learning_rate": 1.3758380302109808e-07,
      "logits/chosen": -2.1404902935028076,
      "logits/rejected": -2.1485772132873535,
      "logps/chosen": -0.2958316206932068,
      "logps/rejected": -0.26554960012435913,
      "loss": 0.1802210509777069,
      "loss/core": 0.17784848809242249,
      "loss/preference_sft": 0.2958316206932068,
      "loss/reference_anchor": 0.08904588967561722,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.537377953529358,
      "rewards/margins": 0.5247774124145508,
      "rewards/rejected": 1.0126006603240967,
      "step": 470
    },
    {
      "drift/chosen_abs": 0.1924518495798111,
      "drift/rejected_abs": 0.0540982261300087,
      "epoch": 0.453839722918906,
      "grad_norm": 1.3125,
      "learning_rate": 1.351698343334823e-07,
      "logits/chosen": -2.097944498062134,
      "logits/rejected": -2.1556663513183594,
      "logps/chosen": -0.3016926348209381,
      "logps/rejected": -0.29718315601348877,
      "loss": 0.4531112611293793,
      "loss/core": 0.44802337884902954,
      "loss/preference_sft": 0.3016926348209381,
      "loss/reference_anchor": 0.22422640025615692,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.9209903478622437,
      "rewards/margins": 1.3928687572479248,
      "rewards/rejected": 0.5281215906143188,
      "step": 475
    },
    {
      "drift/chosen_abs": 0.24358682334423065,
      "drift/rejected_abs": 0.13433735072612762,
      "epoch": 0.45861698316015764,
      "grad_norm": 219.0,
      "learning_rate": 1.3275204118708942e-07,
      "logits/chosen": -2.0871355533599854,
      "logits/rejected": -1.9663331508636475,
      "logps/chosen": -0.3032604455947876,
      "logps/rejected": -0.3135993778705597,
      "loss": 1.2207492589950562,
      "loss/core": 1.2080614566802979,
      "loss/preference_sft": 0.3032604455947876,
      "loss/reference_anchor": 0.6040719747543335,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4357502460479736,
      "rewards/margins": 1.0935547351837158,
      "rewards/rejected": 1.3421953916549683,
      "step": 480
    },
    {
      "drift/chosen_abs": 0.2088780403137207,
      "drift/rejected_abs": 0.10938602685928345,
      "epoch": 0.4633942434014093,
      "grad_norm": 105.5,
      "learning_rate": 1.3033133281504132e-07,
      "logits/chosen": -1.9471505880355835,
      "logits/rejected": -1.9641081094741821,
      "logps/chosen": -0.2977781295776367,
      "logps/rejected": -0.2958124577999115,
      "loss": 0.46540889143943787,
      "loss/core": 0.4602069854736328,
      "loss/preference_sft": 0.2977781295776367,
      "loss/reference_anchor": 0.23032017052173615,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0887653827667236,
      "rewards/margins": 0.9960877299308777,
      "rewards/rejected": 1.0926777124404907,
      "step": 485
    },
    {
      "drift/chosen_abs": 0.23758061230182648,
      "drift/rejected_abs": 0.11018860340118408,
      "epoch": 0.4681715036426609,
      "grad_norm": 10.0625,
      "learning_rate": 1.2790861954675702e-07,
      "logits/chosen": -2.0518007278442383,
      "logits/rejected": -2.0877254009246826,
      "logps/chosen": -0.31017646193504333,
      "logps/rejected": -0.3069363832473755,
      "loss": 0.29436618089675903,
      "loss/core": 0.2908337116241455,
      "loss/preference_sft": 0.31017646193504333,
      "loss/reference_anchor": 0.1456061452627182,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.374509334564209,
      "rewards/margins": 1.2785499095916748,
      "rewards/rejected": 1.0959594249725342,
      "step": 490
    },
    {
      "drift/chosen_abs": 0.1931905448436737,
      "drift/rejected_abs": 0.09922536462545395,
      "epoch": 0.47294876388391255,
      "grad_norm": 47.5,
      "learning_rate": 1.2548481246561504e-07,
      "logits/chosen": -2.055600643157959,
      "logits/rejected": -2.0807411670684814,
      "logps/chosen": -0.28315284848213196,
      "logps/rejected": -0.28561973571777344,
      "loss": 0.28335708379745483,
      "loss/core": 0.2799864411354065,
      "loss/preference_sft": 0.28315284848213196,
      "loss/reference_anchor": 0.14021827280521393,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9313852787017822,
      "rewards/margins": 0.9417422413825989,
      "rewards/rejected": 0.9896429181098938,
      "step": 495
    },
    {
      "drift/chosen_abs": 0.3655059039592743,
      "drift/rejected_abs": 0.16599538922309875,
      "epoch": 0.47772602412516424,
      "grad_norm": 400.0,
      "learning_rate": 1.230608230663321e-07,
      "logits/chosen": -2.061601161956787,
      "logits/rejected": -2.0343308448791504,
      "logps/chosen": -0.28122663497924805,
      "logps/rejected": -0.2838439345359802,
      "loss": 1.412589430809021,
      "loss/core": 1.3980388641357422,
      "loss/preference_sft": 0.28122663497924805,
      "loss/reference_anchor": 0.6994101405143738,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.652092456817627,
      "rewards/margins": 2.0014448165893555,
      "rewards/rejected": 1.650647759437561,
      "step": 500
    },
    {
      "drift/chosen_abs": 0.34218084812164307,
      "drift/rejected_abs": 0.13698555529117584,
      "epoch": 0.4825032843664159,
      "grad_norm": 3.296875,
      "learning_rate": 1.206375629121874e-07,
      "logits/chosen": -2.0119738578796387,
      "logits/rejected": -2.002357006072998,
      "logps/chosen": -0.31075453758239746,
      "logps/rejected": -0.31518369913101196,
      "loss": 0.8807415962219238,
      "loss/core": 0.8713995814323425,
      "loss/preference_sft": 0.31075453758239746,
      "loss/reference_anchor": 0.436026394367218,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.421598434448242,
      "rewards/margins": 2.0699105262756348,
      "rewards/rejected": 1.3516877889633179,
      "step": 505
    },
    {
      "drift/chosen_abs": 0.23318608105182648,
      "drift/rejected_abs": 0.12134194374084473,
      "epoch": 0.4872805446076675,
      "grad_norm": 9.3125,
      "learning_rate": 1.1821594329222079e-07,
      "logits/chosen": -2.206329584121704,
      "logits/rejected": -2.1498043537139893,
      "logps/chosen": -0.29152804613113403,
      "logps/rejected": -0.28346484899520874,
      "loss": 1.038659691810608,
      "loss/core": 1.027789831161499,
      "loss/preference_sft": 0.29152804613113403,
      "loss/reference_anchor": 0.5143306851387024,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.331209897994995,
      "rewards/margins": 1.118821382522583,
      "rewards/rejected": 1.2123886346817017,
      "step": 510
    },
    {
      "drift/chosen_abs": 0.18352241814136505,
      "drift/rejected_abs": 0.1801319718360901,
      "epoch": 0.49205780484891914,
      "grad_norm": 62.75,
      "learning_rate": 1.157968748785344e-07,
      "logits/chosen": -1.9077939987182617,
      "logits/rejected": -1.9757105112075806,
      "logps/chosen": -0.2810531258583069,
      "logps/rejected": -0.28047794103622437,
      "loss": 0.3822907507419586,
      "loss/core": 0.37794721126556396,
      "loss/preference_sft": 0.2810531258583069,
      "loss/reference_anchor": 0.18907247483730316,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.832468032836914,
      "rewards/margins": 0.033732883632183075,
      "rewards/rejected": 1.7987350225448608,
      "step": 515
    },
    {
      "drift/chosen_abs": 0.32048696279525757,
      "drift/rejected_abs": 0.1339716911315918,
      "epoch": 0.4968350650901708,
      "grad_norm": 11.625,
      "learning_rate": 1.1338126738382597e-07,
      "logits/chosen": -1.9192636013031006,
      "logits/rejected": -1.922795295715332,
      "logps/chosen": -0.28838467597961426,
      "logps/rejected": -0.29027271270751953,
      "loss": 1.2226439714431763,
      "loss/core": 1.209959864616394,
      "loss/preference_sft": 0.28838467597961426,
      "loss/reference_anchor": 0.6053675413131714,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.204869508743286,
      "rewards/margins": 1.8670326471328735,
      "rewards/rejected": 1.3378373384475708,
      "step": 520
    },
    {
      "drift/chosen_abs": 0.21345123648643494,
      "drift/rejected_abs": 0.07307306677103043,
      "epoch": 0.5016123253314224,
      "grad_norm": 2.796875,
      "learning_rate": 1.1097002921928316e-07,
      "logits/chosen": -2.0818798542022705,
      "logits/rejected": -2.164365291595459,
      "logps/chosen": -0.2766956388950348,
      "logps/rejected": -0.274677574634552,
      "loss": 0.38875702023506165,
      "loss/core": 0.3843541443347931,
      "loss/preference_sft": 0.2766956388950348,
      "loss/reference_anchor": 0.1924724131822586,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.1327221393585205,
      "rewards/margins": 1.4029709100723267,
      "rewards/rejected": 0.729751467704773,
      "step": 525
    },
    {
      "drift/chosen_abs": 0.16321828961372375,
      "drift/rejected_abs": 0.08229781687259674,
      "epoch": 0.5063895855726741,
      "grad_norm": 1.96875,
      "learning_rate": 1.0856406715296717e-07,
      "logits/chosen": -2.1804051399230957,
      "logits/rejected": -2.278212070465088,
      "logps/chosen": -0.2839670479297638,
      "logps/rejected": -0.2922574579715729,
      "loss": 0.15642455220222473,
      "loss/core": 0.1543111354112625,
      "loss/preference_sft": 0.2839670479297638,
      "loss/reference_anchor": 0.07727432250976562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6320085525512695,
      "rewards/margins": 0.8094345331192017,
      "rewards/rejected": 0.8225740194320679,
      "step": 530
    },
    {
      "drift/chosen_abs": 0.334153413772583,
      "drift/rejected_abs": 0.12186181545257568,
      "epoch": 0.5111668458139257,
      "grad_norm": 2.28125,
      "learning_rate": 1.061642859688146e-07,
      "logits/chosen": -2.1094107627868652,
      "logits/rejected": -2.038062334060669,
      "logps/chosen": -0.266519695520401,
      "logps/rejected": -0.27537840604782104,
      "loss": 0.5663573741912842,
      "loss/core": 0.560210108757019,
      "loss/preference_sft": 0.266519695520401,
      "loss/reference_anchor": 0.28070589900016785,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.3413596153259277,
      "rewards/margins": 2.1438353061676025,
      "rewards/rejected": 1.1975243091583252,
      "step": 535
    },
    {
      "drift/chosen_abs": 0.2915368974208832,
      "drift/rejected_abs": 0.08553571999073029,
      "epoch": 0.5159441060551774,
      "grad_norm": 402.0,
      "learning_rate": 1.0377158812638491e-07,
      "logits/chosen": -2.1358208656311035,
      "logits/rejected": -2.2264904975891113,
      "logps/chosen": -0.2915400564670563,
      "logps/rejected": -0.28075969219207764,
      "loss": 0.7008722424507141,
      "loss/core": 0.6933463215827942,
      "loss/preference_sft": 0.2915400564670563,
      "loss/reference_anchor": 0.34714406728744507,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.914684534072876,
      "rewards/margins": 2.095693588256836,
      "rewards/rejected": 0.8189910054206848,
      "step": 540
    },
    {
      "drift/chosen_abs": 0.2902545928955078,
      "drift/rejected_abs": 0.14741960167884827,
      "epoch": 0.520721366296429,
      "grad_norm": 203.0,
      "learning_rate": 1.0138687342148249e-07,
      "logits/chosen": -2.057218551635742,
      "logits/rejected": -2.1033124923706055,
      "logps/chosen": -0.30191048979759216,
      "logps/rejected": -0.32187768816947937,
      "loss": 1.2491960525512695,
      "loss/core": 1.2362278699874878,
      "loss/preference_sft": 0.30191048979759216,
      "loss/reference_anchor": 0.6182119846343994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.9013302326202393,
      "rewards/margins": 1.42949640750885,
      "rewards/rejected": 1.4718337059020996,
      "step": 545
    },
    {
      "drift/chosen_abs": 0.24866187572479248,
      "drift/rejected_abs": 0.20654296875,
      "epoch": 0.5254986265376806,
      "grad_norm": 4.46875,
      "learning_rate": 9.90110386477801e-08,
      "logits/chosen": -2.1251871585845947,
      "logits/rejected": -2.2004730701446533,
      "logps/chosen": -0.2589234709739685,
      "logps/rejected": -0.2543538212776184,
      "loss": 0.9448872804641724,
      "loss/core": 0.935012936592102,
      "loss/preference_sft": 0.2589234709739685,
      "loss/reference_anchor": 0.46782487630844116,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4865527153015137,
      "rewards/margins": 0.4227861762046814,
      "rewards/rejected": 2.0637667179107666,
      "step": 550
    },
    {
      "drift/chosen_abs": 0.2124369591474533,
      "drift/rejected_abs": 0.15426431596279144,
      "epoch": 0.5302758867789323,
      "grad_norm": 16.625,
      "learning_rate": 9.664497725957164e-08,
      "logits/chosen": -2.1896917819976807,
      "logits/rejected": -2.211423873901367,
      "logps/chosen": -0.29176199436187744,
      "logps/rejected": -0.27853789925575256,
      "loss": 0.46647167205810547,
      "loss/core": 0.46127018332481384,
      "loss/preference_sft": 0.29176199436187744,
      "loss/reference_anchor": 0.2309001386165619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1223769187927246,
      "rewards/margins": 0.5811587572097778,
      "rewards/rejected": 1.5412182807922363,
      "step": 555
    },
    {
      "drift/chosen_abs": 0.267036497592926,
      "drift/rejected_abs": 0.19418852031230927,
      "epoch": 0.5350531470201839,
      "grad_norm": 241.0,
      "learning_rate": 9.428957903578045e-08,
      "logits/chosen": -1.961870789527893,
      "logits/rejected": -1.9748855829238892,
      "logps/chosen": -0.28386375308036804,
      "logps/rejected": -0.28560274839401245,
      "loss": 1.1362648010253906,
      "loss/core": 1.1244474649429321,
      "loss/preference_sft": 0.28386375308036804,
      "loss/reference_anchor": 0.5624862909317017,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6667327880859375,
      "rewards/margins": 0.7276301980018616,
      "rewards/rejected": 1.9391027688980103,
      "step": 560
    },
    {
      "drift/chosen_abs": 0.348408579826355,
      "drift/rejected_abs": 0.19584575295448303,
      "epoch": 0.5398304072614356,
      "grad_norm": 3.71875,
      "learning_rate": 9.194572974534976e-08,
      "logits/chosen": -2.02203369140625,
      "logits/rejected": -2.0890095233917236,
      "logps/chosen": -0.2719302773475647,
      "logps/rejected": -0.2636726200580597,
      "loss": 1.6560252904891968,
      "loss/core": 1.6390788555145264,
      "loss/preference_sft": 0.2719302773475647,
      "loss/reference_anchor": 0.8201166391372681,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4829654693603516,
      "rewards/margins": 1.5329525470733643,
      "rewards/rejected": 1.9500129222869873,
      "step": 565
    },
    {
      "drift/chosen_abs": 0.22708716988563538,
      "drift/rejected_abs": 0.166388139128685,
      "epoch": 0.5446076675026872,
      "grad_norm": 79.5,
      "learning_rate": 8.96143108141412e-08,
      "logits/chosen": -1.9567362070083618,
      "logits/rejected": -1.9499162435531616,
      "logps/chosen": -0.27698129415512085,
      "logps/rejected": -0.27503517270088196,
      "loss": 0.4375968873500824,
      "loss/core": 0.4327124059200287,
      "loss/preference_sft": 0.27698129415512085,
      "loss/reference_anchor": 0.21652822196483612,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.270296812057495,
      "rewards/margins": 0.6087193489074707,
      "rewards/rejected": 1.6615772247314453,
      "step": 570
    },
    {
      "drift/chosen_abs": 0.2241770476102829,
      "drift/rejected_abs": 0.08824467658996582,
      "epoch": 0.5493849277439389,
      "grad_norm": 1.484375,
      "learning_rate": 8.72961989934668e-08,
      "logits/chosen": -2.2255711555480957,
      "logits/rejected": -2.238135814666748,
      "logps/chosen": -0.28202304244041443,
      "logps/rejected": -0.2808363735675812,
      "loss": 0.4496285319328308,
      "loss/core": 0.4446125626564026,
      "loss/preference_sft": 0.28202304244041443,
      "loss/reference_anchor": 0.22259919345378876,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.241278886795044,
      "rewards/margins": 1.3592052459716797,
      "rewards/rejected": 0.8820737600326538,
      "step": 575
    },
    {
      "drift/chosen_abs": 0.1400548666715622,
      "drift/rejected_abs": 0.10045614093542099,
      "epoch": 0.5541621879851905,
      "grad_norm": 9.0625,
      "learning_rate": 8.499226603037854e-08,
      "logits/chosen": -2.2342448234558105,
      "logits/rejected": -2.270289659500122,
      "logps/chosen": -0.29988688230514526,
      "logps/rejected": -0.301006555557251,
      "loss": 0.20430853962898254,
      "loss/core": 0.20168928802013397,
      "loss/preference_sft": 0.29988688230514526,
      "loss/reference_anchor": 0.1009727343916893,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.4003478288650513,
      "rewards/margins": 0.40323466062545776,
      "rewards/rejected": 0.9971132278442383,
      "step": 580
    },
    {
      "drift/chosen_abs": 0.22656722366809845,
      "drift/rejected_abs": 0.10197553783655167,
      "epoch": 0.5589394482264421,
      "grad_norm": 1.9921875,
      "learning_rate": 8.270337833983987e-08,
      "logits/chosen": -2.027678966522217,
      "logits/rejected": -2.1094918251037598,
      "logps/chosen": -0.30382272601127625,
      "logps/rejected": -0.29690489172935486,
      "loss": 0.5977300405502319,
      "loss/core": 0.5912034511566162,
      "loss/preference_sft": 0.30382272601127625,
      "loss/reference_anchor": 0.2959441840648651,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2652218341827393,
      "rewards/margins": 1.2462141513824463,
      "rewards/rejected": 1.019007682800293,
      "step": 585
    },
    {
      "drift/chosen_abs": 0.21109525859355927,
      "drift/rejected_abs": 0.1344117820262909,
      "epoch": 0.5637167084676937,
      "grad_norm": 42.75,
      "learning_rate": 8.04303966789025e-08,
      "logits/chosen": -2.124232769012451,
      "logits/rejected": -2.143303632736206,
      "logps/chosen": -0.2748453617095947,
      "logps/rejected": -0.27825242280960083,
      "loss": 0.5149407982826233,
      "loss/core": 0.5092951655387878,
      "loss/preference_sft": 0.2748453617095947,
      "loss/reference_anchor": 0.25479501485824585,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.109945297241211,
      "rewards/margins": 0.7668979167938232,
      "rewards/rejected": 1.3430473804473877,
      "step": 590
    },
    {
      "drift/chosen_abs": 0.21537485718727112,
      "drift/rejected_abs": 0.11827908456325531,
      "epoch": 0.5684939687089454,
      "grad_norm": 7.4375,
      "learning_rate": 7.817417582301098e-08,
      "logits/chosen": -1.8684651851654053,
      "logits/rejected": -1.888282060623169,
      "logps/chosen": -0.28441476821899414,
      "logps/rejected": -0.2889414429664612,
      "loss": 0.2612078785896301,
      "loss/core": 0.2580533027648926,
      "loss/preference_sft": 0.28441476821899414,
      "loss/reference_anchor": 0.1292877197265625,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.153564929962158,
      "rewards/margins": 0.9714028239250183,
      "rewards/rejected": 1.1821620464324951,
      "step": 595
    },
    {
      "drift/chosen_abs": 0.1943017989397049,
      "drift/rejected_abs": 0.12395434081554413,
      "epoch": 0.5732712289501971,
      "grad_norm": 37.25,
      "learning_rate": 7.59355642445566e-08,
      "logits/chosen": -2.2329204082489014,
      "logits/rejected": -2.143657922744751,
      "logps/chosen": -0.2619364559650421,
      "logps/rejected": -0.271796315908432,
      "loss": 0.325387179851532,
      "loss/core": 0.3216429650783539,
      "loss/preference_sft": 0.2619364559650421,
      "loss/reference_anchor": 0.16101622581481934,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9421056509017944,
      "rewards/margins": 0.7342923283576965,
      "rewards/rejected": 1.2078135013580322,
      "step": 600
    },
    {
      "drift/chosen_abs": 0.21480529010295868,
      "drift/rejected_abs": 0.09283845126628876,
      "epoch": 0.5780484891914487,
      "grad_norm": 226.0,
      "learning_rate": 7.37154037938015e-08,
      "logits/chosen": -2.0678582191467285,
      "logits/rejected": -2.1681721210479736,
      "logps/chosen": -0.32561880350112915,
      "logps/rejected": -0.33226823806762695,
      "loss": 0.3545982241630554,
      "loss/core": 0.3504359722137451,
      "loss/preference_sft": 0.32561880350112915,
      "loss/reference_anchor": 0.17555202543735504,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.146212339401245,
      "rewards/margins": 1.2209621667861938,
      "rewards/rejected": 0.925250232219696,
      "step": 605
    },
    {
      "drift/chosen_abs": 0.21430733799934387,
      "drift/rejected_abs": 0.11561570316553116,
      "epoch": 0.5828257494327004,
      "grad_norm": 6.875,
      "learning_rate": 7.151452938229325e-08,
      "logits/chosen": -2.048251152038574,
      "logits/rejected": -2.0495190620422363,
      "logps/chosen": -0.2833295464515686,
      "logps/rejected": -0.2934442460536957,
      "loss": 0.5116078853607178,
      "loss/core": 0.5059792399406433,
      "loss/preference_sft": 0.2833295464515686,
      "loss/reference_anchor": 0.2530977129936218,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.142573356628418,
      "rewards/margins": 0.9865756034851074,
      "rewards/rejected": 1.1559979915618896,
      "step": 610
    },
    {
      "drift/chosen_abs": 0.30491968989372253,
      "drift/rejected_abs": 0.16428294777870178,
      "epoch": 0.587603009673952,
      "grad_norm": 860.0,
      "learning_rate": 6.933376866888883e-08,
      "logits/chosen": -2.034879446029663,
      "logits/rejected": -2.1349658966064453,
      "logps/chosen": -0.2771604657173157,
      "logps/rejected": -0.3132755160331726,
      "loss": 1.1284635066986084,
      "loss/core": 1.1167380809783936,
      "loss/preference_sft": 0.2771604657173157,
      "loss/reference_anchor": 0.558564305305481,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.047480344772339,
      "rewards/margins": 1.4195116758346558,
      "rewards/rejected": 1.6279685497283936,
      "step": 615
    },
    {
      "drift/chosen_abs": 0.2592160105705261,
      "drift/rejected_abs": 0.14850470423698425,
      "epoch": 0.5923802699152036,
      "grad_norm": 788.0,
      "learning_rate": 6.717394174850605e-08,
      "logits/chosen": -2.144191265106201,
      "logits/rejected": -2.159964084625244,
      "logps/chosen": -0.28378695249557495,
      "logps/rejected": -0.2941509485244751,
      "loss": 0.699615478515625,
      "loss/core": 0.6921217441558838,
      "loss/preference_sft": 0.28378695249557495,
      "loss/reference_anchor": 0.3463062047958374,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.591275930404663,
      "rewards/margins": 1.1079087257385254,
      "rewards/rejected": 1.4833673238754272,
      "step": 620
    },
    {
      "drift/chosen_abs": 0.19440719485282898,
      "drift/rejected_abs": 0.1438429057598114,
      "epoch": 0.5971575301564552,
      "grad_norm": 7.3125,
      "learning_rate": 6.503586084371926e-08,
      "logits/chosen": -2.1661837100982666,
      "logits/rejected": -2.161391019821167,
      "logps/chosen": -0.2829047739505768,
      "logps/rejected": -0.30115213990211487,
      "loss": 0.34582698345184326,
      "loss/core": 0.3418407738208771,
      "loss/preference_sft": 0.2829047739505768,
      "loss/reference_anchor": 0.17102031409740448,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9429868459701538,
      "rewards/margins": 0.5073825120925903,
      "rewards/rejected": 1.4356043338775635,
      "step": 625
    },
    {
      "drift/chosen_abs": 0.29011595249176025,
      "drift/rejected_abs": 0.13060837984085083,
      "epoch": 0.6019347903977069,
      "grad_norm": 35.5,
      "learning_rate": 6.29203299993155e-08,
      "logits/chosen": -2.108480215072632,
      "logits/rejected": -2.092172622680664,
      "logps/chosen": -0.30079469084739685,
      "logps/rejected": -0.3097718358039856,
      "loss": 0.8803378939628601,
      "loss/core": 0.8710153698921204,
      "loss/preference_sft": 0.30079469084739685,
      "loss/reference_anchor": 0.4360447824001312,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.9011597633361816,
      "rewards/margins": 1.9104152917861938,
      "rewards/rejected": 0.9907445907592773,
      "step": 630
    },
    {
      "drift/chosen_abs": 0.17126503586769104,
      "drift/rejected_abs": 0.13067471981048584,
      "epoch": 0.6067120506389586,
      "grad_norm": 50.75,
      "learning_rate": 6.082814477992656e-08,
      "logits/chosen": -2.184509754180908,
      "logits/rejected": -2.1129708290100098,
      "logps/chosen": -0.304170161485672,
      "logps/rejected": -0.3077109456062317,
      "loss": 0.464498370885849,
      "loss/core": 0.45929422974586487,
      "loss/preference_sft": 0.304170161485672,
      "loss/reference_anchor": 0.22978965938091278,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7111921310424805,
      "rewards/margins": 0.4084002375602722,
      "rewards/rejected": 1.302791953086853,
      "step": 635
    },
    {
      "drift/chosen_abs": 0.1660601645708084,
      "drift/rejected_abs": 0.06357360631227493,
      "epoch": 0.6114893108802102,
      "grad_norm": 10.0,
      "learning_rate": 5.87600919708494e-08,
      "logits/chosen": -1.9939651489257812,
      "logits/rejected": -2.038660764694214,
      "logps/chosen": -0.28754812479019165,
      "logps/rejected": -0.2897412180900574,
      "loss": 0.26881492137908936,
      "loss/core": 0.2655784487724304,
      "loss/preference_sft": 0.28754812479019165,
      "loss/reference_anchor": 0.13306793570518494,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6595392227172852,
      "rewards/margins": 1.0255340337753296,
      "rewards/rejected": 0.634005069732666,
      "step": 640
    },
    {
      "drift/chosen_abs": 0.20235376060009003,
      "drift/rejected_abs": 0.1837695837020874,
      "epoch": 0.6162665711214619,
      "grad_norm": 200.0,
      "learning_rate": 5.671694928216829e-08,
      "logits/chosen": -2.2157418727874756,
      "logits/rejected": -2.2581088542938232,
      "logps/chosen": -0.2847815155982971,
      "logps/rejected": -0.2754765748977661,
      "loss": 0.5128556489944458,
      "loss/core": 0.5072119832038879,
      "loss/preference_sft": 0.2847815155982971,
      "loss/reference_anchor": 0.25370898842811584,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.0232532024383545,
      "rewards/margins": 0.18848243355751038,
      "rewards/rejected": 1.8347707986831665,
      "step": 645
    },
    {
      "drift/chosen_abs": 0.31445202231407166,
      "drift/rejected_abs": 0.2099848985671997,
      "epoch": 0.6210438313627135,
      "grad_norm": 152.0,
      "learning_rate": 5.469948505629e-08,
      "logits/chosen": -2.0116679668426514,
      "logits/rejected": -2.073791980743408,
      "logps/chosen": -0.2909887731075287,
      "logps/rejected": -0.28746503591537476,
      "loss": 1.5114305019378662,
      "loss/core": 1.4958856105804443,
      "loss/preference_sft": 0.2909887731075287,
      "loss/reference_anchor": 0.748149573802948,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.143667697906494,
      "rewards/margins": 1.044376254081726,
      "rewards/rejected": 2.0992913246154785,
      "step": 650
    },
    {
      "drift/chosen_abs": 0.17652888596057892,
      "drift/rejected_abs": 0.12569592893123627,
      "epoch": 0.6258210916039652,
      "grad_norm": 157.0,
      "learning_rate": 5.270845797900168e-08,
      "logits/chosen": -2.215951919555664,
      "logits/rejected": -2.147303342819214,
      "logps/chosen": -0.2824100852012634,
      "logps/rejected": -0.2910049557685852,
      "loss": 0.5359405279159546,
      "loss/core": 0.5300716757774353,
      "loss/preference_sft": 0.2824100852012634,
      "loss/reference_anchor": 0.2652043402194977,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.7618181705474854,
      "rewards/margins": 0.5078210234642029,
      "rewards/rejected": 1.2539972066879272,
      "step": 655
    },
    {
      "drift/chosen_abs": 0.18071475625038147,
      "drift/rejected_abs": 0.14076223969459534,
      "epoch": 0.6305983518452167,
      "grad_norm": 1.3203125,
      "learning_rate": 5.0744616794160104e-08,
      "logits/chosen": -2.0582504272460938,
      "logits/rejected": -2.1387999057769775,
      "logps/chosen": -0.2772466242313385,
      "logps/rejected": -0.28088733553886414,
      "loss": 0.2379394769668579,
      "loss/core": 0.2350325882434845,
      "loss/preference_sft": 0.2772466242313385,
      "loss/reference_anchor": 0.11762037128210068,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8062305450439453,
      "rewards/margins": 0.40019816160202026,
      "rewards/rejected": 1.4060323238372803,
      "step": 660
    },
    {
      "drift/chosen_abs": 0.22593644261360168,
      "drift/rejected_abs": 0.1269337683916092,
      "epoch": 0.6353756120864684,
      "grad_norm": 8.125,
      "learning_rate": 4.880870002211963e-08,
      "logits/chosen": -2.240903854370117,
      "logits/rejected": -2.2479407787323,
      "logps/chosen": -0.27797046303749084,
      "logps/rejected": -0.25145772099494934,
      "loss": 0.524042546749115,
      "loss/core": 0.5182952284812927,
      "loss/preference_sft": 0.27797046303749084,
      "loss/reference_anchor": 0.25957146286964417,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.2591378688812256,
      "rewards/margins": 0.9982420802116394,
      "rewards/rejected": 1.26089608669281,
      "step": 665
    },
    {
      "drift/chosen_abs": 0.13284215331077576,
      "drift/rejected_abs": 0.09541095048189163,
      "epoch": 0.64015287232772,
      "grad_norm": 1.09375,
      "learning_rate": 4.6901435682004996e-08,
      "logits/chosen": -2.236567974090576,
      "logits/rejected": -2.271984100341797,
      "logps/chosen": -0.29236721992492676,
      "logps/rejected": -0.2945176064968109,
      "loss": 0.12218450009822845,
      "loss/core": 0.12039539963006973,
      "loss/preference_sft": 0.29236721992492676,
      "loss/reference_anchor": 0.06021867319941521,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.3271570205688477,
      "rewards/margins": 0.3746449947357178,
      "rewards/rejected": 0.9525120854377747,
      "step": 670
    },
    {
      "drift/chosen_abs": 0.16276486217975616,
      "drift/rejected_abs": 0.1297510266304016,
      "epoch": 0.6449301325689717,
      "grad_norm": 27.875,
      "learning_rate": 4.502354101793314e-08,
      "logits/chosen": -2.358452320098877,
      "logits/rejected": -2.268037796020508,
      "logps/chosen": -0.279884934425354,
      "logps/rejected": -0.2925143837928772,
      "loss": 0.35353174805641174,
      "loss/core": 0.3494763970375061,
      "loss/preference_sft": 0.279884934425354,
      "loss/reference_anchor": 0.17477791011333466,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6270040273666382,
      "rewards/margins": 0.33251360058784485,
      "rewards/rejected": 1.2944905757904053,
      "step": 675
    },
    {
      "drift/chosen_abs": 0.31575721502304077,
      "drift/rejected_abs": 0.1288444995880127,
      "epoch": 0.6497073928102234,
      "grad_norm": 36.25,
      "learning_rate": 4.3175722229287034e-08,
      "logits/chosen": -2.114642381668091,
      "logits/rejected": -2.1079599857330322,
      "logps/chosen": -0.26992639899253845,
      "logps/rejected": -0.27495431900024414,
      "loss": 0.6401761174201965,
      "loss/core": 0.6332933306694031,
      "loss/preference_sft": 0.26992639899253845,
      "loss/reference_anchor": 0.3171465992927551,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.1569862365722656,
      "rewards/margins": 1.9471004009246826,
      "rewards/rejected": 1.209885835647583,
      "step": 680
    },
    {
      "drift/chosen_abs": 0.23132996261119843,
      "drift/rejected_abs": 0.23488974571228027,
      "epoch": 0.654484653051475,
      "grad_norm": 126.0,
      "learning_rate": 4.135867420514276e-08,
      "logits/chosen": -2.2158150672912598,
      "logits/rejected": -2.220139741897583,
      "logps/chosen": -0.29107457399368286,
      "logps/rejected": -0.26147180795669556,
      "loss": 0.934106707572937,
      "loss/core": 0.9242784380912781,
      "loss/preference_sft": 0.29107457399368286,
      "loss/reference_anchor": 0.4623136520385742,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.3132710456848145,
      "rewards/margins": -0.03536492586135864,
      "rewards/rejected": 2.3486359119415283,
      "step": 685
    },
    {
      "drift/chosen_abs": 0.2907050848007202,
      "drift/rejected_abs": 0.21539370715618134,
      "epoch": 0.6592619132927267,
      "grad_norm": 704.0,
      "learning_rate": 3.957308026295035e-08,
      "logits/chosen": -2.0876173973083496,
      "logits/rejected": -2.159592390060425,
      "logps/chosen": -0.3214958608150482,
      "logps/rejected": -0.3213821053504944,
      "loss": 1.7343063354492188,
      "loss/core": 1.716496467590332,
      "loss/preference_sft": 0.3214958608150482,
      "loss/reference_anchor": 0.858340859413147,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9058361053466797,
      "rewards/margins": 0.7544411420822144,
      "rewards/rejected": 2.151395082473755,
      "step": 690
    },
    {
      "drift/chosen_abs": 0.23619894683361053,
      "drift/rejected_abs": 0.11816170066595078,
      "epoch": 0.6640391735339782,
      "grad_norm": 40.25,
      "learning_rate": 3.7819611891566084e-08,
      "logits/chosen": -1.9863860607147217,
      "logits/rejected": -1.9883887767791748,
      "logps/chosen": -0.26810505986213684,
      "logps/rejected": -0.2822146713733673,
      "loss": 0.27285757660865784,
      "loss/core": 0.269618421792984,
      "loss/preference_sft": 0.26810505986213684,
      "loss/reference_anchor": 0.13514725863933563,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.361358880996704,
      "rewards/margins": 1.190822720527649,
      "rewards/rejected": 1.1705362796783447,
      "step": 695
    },
    {
      "drift/chosen_abs": 0.32739442586898804,
      "drift/rejected_abs": 0.16697341203689575,
      "epoch": 0.6688164337752299,
      "grad_norm": 45.25,
      "learning_rate": 3.609892849873286e-08,
      "logits/chosen": -2.238140106201172,
      "logits/rejected": -2.170215606689453,
      "logps/chosen": -0.28792768716812134,
      "logps/rejected": -0.26793545484542847,
      "loss": 1.2909228801727295,
      "loss/core": 1.2775663137435913,
      "loss/preference_sft": 0.28792768716812134,
      "loss/reference_anchor": 0.6390419006347656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.2727417945861816,
      "rewards/margins": 1.6051795482635498,
      "rewards/rejected": 1.6675622463226318,
      "step": 700
    },
    {
      "drift/chosen_abs": 0.1326175481081009,
      "drift/rejected_abs": 0.05240592360496521,
      "epoch": 0.6735936940164815,
      "grad_norm": 0.875,
      "learning_rate": 3.4411677163103894e-08,
      "logits/chosen": -2.1258556842803955,
      "logits/rejected": -2.1329469680786133,
      "logps/chosen": -0.3264055550098419,
      "logps/rejected": -0.31279516220092773,
      "loss": 0.1313130259513855,
      "loss/core": 0.1293625682592392,
      "loss/preference_sft": 0.3264055550098419,
      "loss/reference_anchor": 0.06488193571567535,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.3231408596038818,
      "rewards/margins": 0.7995508313179016,
      "rewards/rejected": 0.5235900282859802,
      "step": 705
    },
    {
      "drift/chosen_abs": 0.24739420413970947,
      "drift/rejected_abs": 0.08645138889551163,
      "epoch": 0.6783709542577332,
      "grad_norm": 39.0,
      "learning_rate": 3.2758492390902945e-08,
      "logits/chosen": -1.9009885787963867,
      "logits/rejected": -1.9612804651260376,
      "logps/chosen": -0.2778227627277374,
      "logps/rejected": -0.28797098994255066,
      "loss": 0.3639562726020813,
      "loss/core": 0.359793096780777,
      "loss/preference_sft": 0.2778227627277374,
      "loss/reference_anchor": 0.18037569522857666,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.4713692665100098,
      "rewards/margins": 1.64273202419281,
      "rewards/rejected": 0.828637421131134,
      "step": 710
    },
    {
      "drift/chosen_abs": 0.23490512371063232,
      "drift/rejected_abs": 0.0935138463973999,
      "epoch": 0.6831482144989849,
      "grad_norm": 10.5,
      "learning_rate": 3.11399958773126e-08,
      "logits/chosen": -2.0637762546539307,
      "logits/rejected": -2.1122887134552,
      "logps/chosen": -0.2840803563594818,
      "logps/rejected": -0.2814682424068451,
      "loss": 0.23643600940704346,
      "loss/core": 0.23352782428264618,
      "loss/preference_sft": 0.2840803563594818,
      "loss/reference_anchor": 0.11700389534235,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.348367214202881,
      "rewards/margins": 1.4155561923980713,
      "rewards/rejected": 0.9328109622001648,
      "step": 715
    },
    {
      "drift/chosen_abs": 0.25520801544189453,
      "drift/rejected_abs": 0.11677595227956772,
      "epoch": 0.6879254747402365,
      "grad_norm": 5.0625,
      "learning_rate": 2.9556796272679972e-08,
      "logits/chosen": -2.1001739501953125,
      "logits/rejected": -2.1356372833251953,
      "logps/chosen": -0.2884894907474518,
      "logps/rejected": -0.31645768880844116,
      "loss": 0.6001358032226562,
      "loss/core": 0.593619167804718,
      "loss/preference_sft": 0.2884894907474518,
      "loss/reference_anchor": 0.29698267579078674,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.5518627166748047,
      "rewards/margins": 1.5054526329040527,
      "rewards/rejected": 1.0464102029800415,
      "step": 720
    },
    {
      "drift/chosen_abs": 0.11836670339107513,
      "drift/rejected_abs": 0.11379599571228027,
      "epoch": 0.6927027349814882,
      "grad_norm": 2.453125,
      "learning_rate": 2.8009488953628215e-08,
      "logits/chosen": -2.0836520195007324,
      "logits/rejected": -2.0259056091308594,
      "logps/chosen": -0.28411412239074707,
      "logps/rejected": -0.2758258581161499,
      "loss": 0.23348887264728546,
      "loss/core": 0.2306121587753296,
      "loss/preference_sft": 0.28411412239074707,
      "loss/reference_anchor": 0.11542298644781113,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.1802574396133423,
      "rewards/margins": 0.044753752648830414,
      "rewards/rejected": 1.1355037689208984,
      "step": 725
    },
    {
      "drift/chosen_abs": 0.280209481716156,
      "drift/rejected_abs": 0.10185222327709198,
      "epoch": 0.6974799952227397,
      "grad_norm": 37.25,
      "learning_rate": 2.649865579915976e-08,
      "logits/chosen": -2.075974941253662,
      "logits/rejected": -2.050661563873291,
      "logps/chosen": -0.3032965660095215,
      "logps/rejected": -0.31960076093673706,
      "loss": 0.8421915769577026,
      "loss/core": 0.8332451581954956,
      "loss/preference_sft": 0.3032965660095215,
      "loss/reference_anchor": 0.41699153184890747,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.8020949363708496,
      "rewards/margins": 1.8411537408828735,
      "rewards/rejected": 0.9609411954879761,
      "step": 730
    },
    {
      "drift/chosen_abs": 0.2798532247543335,
      "drift/rejected_abs": 0.11052944511175156,
      "epoch": 0.7022572554639914,
      "grad_norm": 49.75,
      "learning_rate": 2.5024864971835507e-08,
      "logits/chosen": -2.1936328411102295,
      "logits/rejected": -2.1866257190704346,
      "logps/chosen": -0.266052782535553,
      "logps/rejected": -0.27340036630630493,
      "loss": 0.5987396240234375,
      "loss/core": 0.5922771692276001,
      "loss/preference_sft": 0.266052782535553,
      "loss/reference_anchor": 0.29651200771331787,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.7965667247772217,
      "rewards/margins": 1.6948391199111938,
      "rewards/rejected": 1.1017276048660278,
      "step": 735
    },
    {
      "drift/chosen_abs": 0.21841701865196228,
      "drift/rejected_abs": 0.16449300944805145,
      "epoch": 0.707034515705243,
      "grad_norm": 41.0,
      "learning_rate": 2.3588670704111997e-08,
      "logits/chosen": -1.9267356395721436,
      "logits/rejected": -1.848046898841858,
      "logps/chosen": -0.2917543947696686,
      "logps/rejected": -0.3047102689743042,
      "loss": 0.6575824022293091,
      "loss/core": 0.6504920125007629,
      "loss/preference_sft": 0.2917543947696686,
      "loss/reference_anchor": 0.3253471255302429,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.181687831878662,
      "rewards/margins": 0.5402703881263733,
      "rewards/rejected": 1.6414177417755127,
      "step": 740
    },
    {
      "drift/chosen_abs": 0.2311057597398758,
      "drift/rejected_abs": 0.12377341091632843,
      "epoch": 0.7118117759464947,
      "grad_norm": 2.3125,
      "learning_rate": 2.219061308991721e-08,
      "logits/chosen": -2.2026872634887695,
      "logits/rejected": -2.2262043952941895,
      "logps/chosen": -0.2942965626716614,
      "logps/rejected": -0.2818772792816162,
      "loss": 0.640469491481781,
      "loss/core": 0.6335400342941284,
      "loss/preference_sft": 0.2942965626716614,
      "loss/reference_anchor": 0.3170434832572937,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.3100028038024902,
      "rewards/margins": 1.0736254453659058,
      "rewards/rejected": 1.2363771200180054,
      "step": 745
    },
    {
      "drift/chosen_abs": 0.2225804626941681,
      "drift/rejected_abs": 0.14012092351913452,
      "epoch": 0.7165890361877463,
      "grad_norm": 147.0,
      "learning_rate": 2.0831217881543557e-08,
      "logits/chosen": -2.250375270843506,
      "logits/rejected": -2.1884655952453613,
      "logps/chosen": -0.27962955832481384,
      "logps/rejected": -0.30086082220077515,
      "loss": 0.2941736876964569,
      "loss/core": 0.29070359468460083,
      "loss/preference_sft": 0.27962955832481384,
      "loss/reference_anchor": 0.14554303884506226,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.221367120742798,
      "rewards/margins": 0.8238778114318848,
      "rewards/rejected": 1.397489309310913,
      "step": 750
    },
    {
      "drift/chosen_abs": 0.35864901542663574,
      "drift/rejected_abs": 0.1930883824825287,
      "epoch": 0.721366296428998,
      "grad_norm": 2.34375,
      "learning_rate": 1.951099629193366e-08,
      "logits/chosen": -2.1636476516723633,
      "logits/rejected": -2.140977144241333,
      "logps/chosen": -0.2771832346916199,
      "logps/rejected": -0.28425145149230957,
      "loss": 1.235163927078247,
      "loss/core": 1.2223742008209229,
      "loss/preference_sft": 0.2771832346916199,
      "loss/reference_anchor": 0.6117671132087708,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.585937976837158,
      "rewards/margins": 1.6583614349365234,
      "rewards/rejected": 1.9275766611099243,
      "step": 755
    },
    {
      "drift/chosen_abs": 0.3538215160369873,
      "drift/rejected_abs": 0.19870851933956146,
      "epoch": 0.7261435566702497,
      "grad_norm": 213.0,
      "learning_rate": 1.823044480243431e-08,
      "logits/chosen": -2.004826784133911,
      "logits/rejected": -2.0009334087371826,
      "logps/chosen": -0.30821096897125244,
      "logps/rejected": -0.35763052105903625,
      "loss": 1.0943807363510132,
      "loss/core": 1.0829265117645264,
      "loss/preference_sft": 0.30821096897125244,
      "loss/reference_anchor": 0.5418857932090759,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.537405014038086,
      "rewards/margins": 1.5527830123901367,
      "rewards/rejected": 1.9846217632293701,
      "step": 760
    },
    {
      "drift/chosen_abs": 0.16350261867046356,
      "drift/rejected_abs": 0.08384877443313599,
      "epoch": 0.7309208169115012,
      "grad_norm": 3.046875,
      "learning_rate": 1.699004497608994e-08,
      "logits/chosen": -2.3181560039520264,
      "logits/rejected": -2.325047016143799,
      "logps/chosen": -0.2780446410179138,
      "logps/rejected": -0.26890766620635986,
      "loss": 0.15251176059246063,
      "loss/core": 0.15044714510440826,
      "loss/preference_sft": 0.2780446410179138,
      "loss/reference_anchor": 0.07542603462934494,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.6326892375946045,
      "rewards/margins": 0.7961629629135132,
      "rewards/rejected": 0.8365263938903809,
      "step": 765
    },
    {
      "drift/chosen_abs": 0.22837817668914795,
      "drift/rejected_abs": 0.07515065371990204,
      "epoch": 0.7356980771527529,
      "grad_norm": 364.0,
      "learning_rate": 1.5790263276546658e-08,
      "logits/chosen": -2.214590549468994,
      "logits/rejected": -2.287729263305664,
      "logps/chosen": -0.2873011529445648,
      "logps/rejected": -0.27970749139785767,
      "loss": 0.502562940120697,
      "loss/core": 0.4970094561576843,
      "loss/preference_sft": 0.2873011529445648,
      "loss/reference_anchor": 0.24894189834594727,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.281993865966797,
      "rewards/margins": 1.531829833984375,
      "rewards/rejected": 0.7501639127731323,
      "step": 770
    },
    {
      "drift/chosen_abs": 0.1840764731168747,
      "drift/rejected_abs": 0.14043626189231873,
      "epoch": 0.7404753373940045,
      "grad_norm": 1.5859375,
      "learning_rate": 1.4631550892634126e-08,
      "logits/chosen": -2.0965075492858887,
      "logits/rejected": -1.9921375513076782,
      "logps/chosen": -0.2745542526245117,
      "logps/rejected": -0.26717105507850647,
      "loss": 0.3951452076435089,
      "loss/core": 0.3906872868537903,
      "loss/preference_sft": 0.2745542526245117,
      "loss/reference_anchor": 0.19543880224227905,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8390769958496094,
      "rewards/margins": 0.4357059895992279,
      "rewards/rejected": 1.4033708572387695,
      "step": 775
    },
    {
      "drift/chosen_abs": 0.2997458875179291,
      "drift/rejected_abs": 0.19411607086658478,
      "epoch": 0.7452525976352562,
      "grad_norm": 464.0,
      "learning_rate": 1.3514343568692132e-08,
      "logits/chosen": -2.240328788757324,
      "logits/rejected": -2.2070069313049316,
      "logps/chosen": -0.2926557660102844,
      "logps/rejected": -0.2933287024497986,
      "loss": 1.8442484140396118,
      "loss/core": 1.8254058361053467,
      "loss/preference_sft": 0.2926557660102844,
      "loss/reference_anchor": 0.9128621816635132,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.995630979537964,
      "rewards/margins": 1.0562885999679565,
      "rewards/rejected": 1.9393424987792969,
      "step": 780
    },
    {
      "drift/chosen_abs": 0.1899985373020172,
      "drift/rejected_abs": 0.07741059362888336,
      "epoch": 0.7500298578765078,
      "grad_norm": 3.65625,
      "learning_rate": 1.2439061440704724e-08,
      "logits/chosen": -2.209710121154785,
      "logits/rejected": -2.2208352088928223,
      "logps/chosen": -0.28208765387535095,
      "logps/rejected": -0.28293755650520325,
      "loss": 0.6761240363121033,
      "loss/core": 0.6688669323921204,
      "loss/preference_sft": 0.28208765387535095,
      "loss/reference_anchor": 0.3346477448940277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8997882604599,
      "rewards/margins": 1.1262468099594116,
      "rewards/rejected": 0.773541271686554,
      "step": 785
    },
    {
      "drift/chosen_abs": 0.3311009407043457,
      "drift/rejected_abs": 0.1309339702129364,
      "epoch": 0.7548071181177595,
      "grad_norm": 173.0,
      "learning_rate": 1.1406108878304468e-08,
      "logits/chosen": -2.0325357913970947,
      "logits/rejected": -2.026935577392578,
      "logps/chosen": -0.27717894315719604,
      "logps/rejected": -0.27080798149108887,
      "loss": 0.6806409955024719,
      "loss/core": 0.6733458638191223,
      "loss/preference_sft": 0.27717894315719604,
      "loss/reference_anchor": 0.33703652024269104,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.310328960418701,
      "rewards/margins": 2.0351784229278564,
      "rewards/rejected": 1.2751507759094238,
      "step": 790
    },
    {
      "drift/chosen_abs": 0.3741660416126251,
      "drift/rejected_abs": 0.0975070595741272,
      "epoch": 0.7595843783590112,
      "grad_norm": 350.0,
      "learning_rate": 1.0415874332705381e-08,
      "logits/chosen": -2.150209903717041,
      "logits/rejected": -2.0983715057373047,
      "logps/chosen": -0.27431005239486694,
      "logps/rejected": -0.2787257730960846,
      "loss": 1.4984773397445679,
      "loss/core": 1.4830867052078247,
      "loss/preference_sft": 0.27431005239486694,
      "loss/reference_anchor": 0.7421107888221741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.738903760910034,
      "rewards/margins": 2.7708230018615723,
      "rewards/rejected": 0.9680808186531067,
      "step": 795
    },
    {
      "drift/chosen_abs": 0.2510735094547272,
      "drift/rejected_abs": 0.1462600976228714,
      "epoch": 0.7643616386002627,
      "grad_norm": 18.625,
      "learning_rate": 9.468730190622484e-09,
      "logits/chosen": -2.4673285484313965,
      "logits/rejected": -2.4519200325012207,
      "logps/chosen": -0.29721131920814514,
      "logps/rejected": -0.3075062036514282,
      "loss": 1.2491406202316284,
      "loss/core": 1.23618483543396,
      "loss/preference_sft": 0.29721131920814514,
      "loss/reference_anchor": 0.6180749535560608,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.509737730026245,
      "rewards/margins": 1.0481786727905273,
      "rewards/rejected": 1.4615590572357178,
      "step": 800
    },
    {
      "drift/chosen_abs": 0.09586870670318604,
      "drift/rejected_abs": 0.0631202682852745,
      "epoch": 0.7691388988415144,
      "grad_norm": 5.0,
      "learning_rate": 8.565032634232194e-09,
      "logits/chosen": -2.117633104324341,
      "logits/rejected": -2.0666775703430176,
      "logps/chosen": -0.30181869864463806,
      "logps/rejected": -0.3010576367378235,
      "loss": 0.04290010407567024,
      "loss/core": 0.04187611863017082,
      "loss/preference_sft": 0.30181869864463806,
      "loss/reference_anchor": 0.02101728692650795,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9572784304618835,
      "rewards/margins": 0.32730334997177124,
      "rewards/rejected": 0.6299751400947571,
      "step": 805
    },
    {
      "drift/chosen_abs": 0.18182532489299774,
      "drift/rejected_abs": 0.16493472456932068,
      "epoch": 0.773916159082766,
      "grad_norm": 90.5,
      "learning_rate": 7.70512150722702e-09,
      "logits/chosen": -2.0990633964538574,
      "logits/rejected": -2.0376296043395996,
      "logps/chosen": -0.26729169487953186,
      "logps/rejected": -0.26321226358413696,
      "loss": 0.49664896726608276,
      "loss/core": 0.491199254989624,
      "loss/preference_sft": 0.26729169487953186,
      "loss/reference_anchor": 0.24575798213481903,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8167064189910889,
      "rewards/margins": 0.16888391971588135,
      "rewards/rejected": 1.647822380065918,
      "step": 810
    },
    {
      "drift/chosen_abs": 0.24560102820396423,
      "drift/rejected_abs": 0.10801911354064941,
      "epoch": 0.7786934193240177,
      "grad_norm": 1.7109375,
      "learning_rate": 6.8893201870139915e-09,
      "logits/chosen": -2.091200113296509,
      "logits/rejected": -2.27951979637146,
      "logps/chosen": -0.31718164682388306,
      "logps/rejected": -0.3068375885486603,
      "loss": 0.5775214433670044,
      "loss/core": 0.5711715817451477,
      "loss/preference_sft": 0.31718164682388306,
      "loss/reference_anchor": 0.2857758402824402,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.4512245655059814,
      "rewards/margins": 1.3882179260253906,
      "rewards/rejected": 1.0630064010620117,
      "step": 815
    },
    {
      "drift/chosen_abs": 0.26311054825782776,
      "drift/rejected_abs": 0.20147815346717834,
      "epoch": 0.7834706795652693,
      "grad_norm": 30.375,
      "learning_rate": 6.117935463105808e-09,
      "logits/chosen": -1.8164457082748413,
      "logits/rejected": -1.7971694469451904,
      "logps/chosen": -0.279045045375824,
      "logps/rejected": -0.33144164085388184,
      "loss": 0.9571006894111633,
      "loss/core": 0.947068989276886,
      "loss/preference_sft": 0.279045045375824,
      "loss/reference_anchor": 0.4736693501472473,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.6302883625030518,
      "rewards/margins": 0.6288776397705078,
      "rewards/rejected": 2.001410961151123,
      "step": 820
    },
    {
      "drift/chosen_abs": 0.3537200093269348,
      "drift/rejected_abs": 0.16787850856781006,
      "epoch": 0.788247939806521,
      "grad_norm": 14.1875,
      "learning_rate": 5.391257421749826e-09,
      "logits/chosen": -2.1316933631896973,
      "logits/rejected": -2.1172828674316406,
      "logps/chosen": -0.29390478134155273,
      "logps/rejected": -0.2766962945461273,
      "loss": 1.5460574626922607,
      "loss/core": 1.530153512954712,
      "loss/preference_sft": 0.29390478134155273,
      "loss/reference_anchor": 0.7657959461212158,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.5372002124786377,
      "rewards/margins": 1.859610915184021,
      "rewards/rejected": 1.6775890588760376,
      "step": 825
    },
    {
      "drift/chosen_abs": 0.27988651394844055,
      "drift/rejected_abs": 0.12321482598781586,
      "epoch": 0.7930252000477725,
      "grad_norm": 612.0,
      "learning_rate": 4.709559336838462e-09,
      "logits/chosen": -1.8963066339492798,
      "logits/rejected": -1.9476807117462158,
      "logps/chosen": -0.30415278673171997,
      "logps/rejected": -0.3033929169178009,
      "loss": 0.9329975247383118,
      "loss/core": 0.9231513142585754,
      "loss/preference_sft": 0.30415278673171997,
      "loss/reference_anchor": 0.4618992209434509,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.798447370529175,
      "rewards/margins": 1.5701082944869995,
      "rewards/rejected": 1.2283390760421753,
      "step": 830
    },
    {
      "drift/chosen_abs": 0.16968384385108948,
      "drift/rejected_abs": 0.1050095334649086,
      "epoch": 0.7978024602890242,
      "grad_norm": 1.7578125,
      "learning_rate": 4.073097567142025e-09,
      "logits/chosen": -2.2337470054626465,
      "logits/rejected": -2.3164491653442383,
      "logps/chosen": -0.2991805672645569,
      "logps/rejected": -0.28817689418792725,
      "loss": 0.22842451930046082,
      "loss/core": 0.2255687713623047,
      "loss/preference_sft": 0.2991805672645569,
      "loss/reference_anchor": 0.11287200450897217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.696568250656128,
      "rewards/margins": 0.6477723121643066,
      "rewards/rejected": 1.0487959384918213,
      "step": 835
    },
    {
      "drift/chosen_abs": 0.2029610425233841,
      "drift/rejected_abs": 0.15892741084098816,
      "epoch": 0.8025797205302759,
      "grad_norm": 9.4375,
      "learning_rate": 3.482111459902695e-09,
      "logits/chosen": -2.029393196105957,
      "logits/rejected": -2.1248087882995605,
      "logps/chosen": -0.28962570428848267,
      "logps/rejected": -0.3038334250450134,
      "loss": 0.6708376407623291,
      "loss/core": 0.6636188626289368,
      "loss/preference_sft": 0.28962570428848267,
      "loss/reference_anchor": 0.3319765329360962,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.0296101570129395,
      "rewards/margins": 0.4415357708930969,
      "rewards/rejected": 1.5880745649337769,
      "step": 840
    },
    {
      "drift/chosen_abs": 0.4587179124355316,
      "drift/rejected_abs": 0.22257618606090546,
      "epoch": 0.8073569807715275,
      "grad_norm": 22.875,
      "learning_rate": 2.9368232608258797e-09,
      "logits/chosen": -2.164417028427124,
      "logits/rejected": -2.176283121109009,
      "logps/chosen": -0.27492496371269226,
      "logps/rejected": -0.27344006299972534,
      "loss": 2.1201024055480957,
      "loss/core": 2.098555564880371,
      "loss/preference_sft": 0.27492496371269226,
      "loss/reference_anchor": 1.0498454570770264,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.5870513916015625,
      "rewards/margins": 2.364668846130371,
      "rewards/rejected": 2.222382068634033,
      "step": 845
    },
    {
      "drift/chosen_abs": 0.17442527413368225,
      "drift/rejected_abs": 0.06823588907718658,
      "epoch": 0.8121342410127792,
      "grad_norm": 0.87109375,
      "learning_rate": 2.4374380305025866e-09,
      "logits/chosen": -2.2421109676361084,
      "logits/rejected": -2.2991902828216553,
      "logps/chosen": -0.25456729531288147,
      "logps/rejected": -0.25157222151756287,
      "loss": 0.12556889653205872,
      "loss/core": 0.12381688505411148,
      "loss/preference_sft": 0.25456729531288147,
      "loss/reference_anchor": 0.06214389204978943,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7427303791046143,
      "rewards/margins": 1.0707417726516724,
      "rewards/rejected": 0.6719885468482971,
      "step": 850
    },
    {
      "drift/chosen_abs": 0.2316814661026001,
      "drift/rejected_abs": 0.18873625993728638,
      "epoch": 0.8169115012540308,
      "grad_norm": 49.25,
      "learning_rate": 1.984143567294594e-09,
      "logits/chosen": -2.0805776119232178,
      "logits/rejected": -2.073974609375,
      "logps/chosen": -0.2674004137516022,
      "logps/rejected": -0.29779064655303955,
      "loss": 0.6567851305007935,
      "loss/core": 0.6497505307197571,
      "loss/preference_sft": 0.2674004137516022,
      "loss/reference_anchor": 0.3249875009059906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3135294914245605,
      "rewards/margins": 0.4317246377468109,
      "rewards/rejected": 1.8818048238754272,
      "step": 855
    },
    {
      "drift/chosen_abs": 0.19219297170639038,
      "drift/rejected_abs": 0.1073886901140213,
      "epoch": 0.8216887614952825,
      "grad_norm": 1.2421875,
      "learning_rate": 1.577110336711096e-09,
      "logits/chosen": -2.1786136627197266,
      "logits/rejected": -2.251077175140381,
      "logps/chosen": -0.29990866780281067,
      "logps/rejected": -0.30773764848709106,
      "loss": 0.4442000985145569,
      "loss/core": 0.4392046332359314,
      "loss/preference_sft": 0.29990866780281067,
      "loss/reference_anchor": 0.21978160738945007,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9197685718536377,
      "rewards/margins": 0.8469489812850952,
      "rewards/rejected": 1.072819471359253,
      "step": 860
    },
    {
      "drift/chosen_abs": 0.26136937737464905,
      "drift/rejected_abs": 0.1216052770614624,
      "epoch": 0.826466021736534,
      "grad_norm": 3.8125,
      "learning_rate": 1.2164914073037048e-09,
      "logits/chosen": -2.1374754905700684,
      "logits/rejected": -2.2176711559295654,
      "logps/chosen": -0.29837021231651306,
      "logps/rejected": -0.282470166683197,
      "loss": 1.20570707321167,
      "loss/core": 1.1931687593460083,
      "loss/preference_sft": 0.29837021231651306,
      "loss/reference_anchor": 0.5970777273178101,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.6126089096069336,
      "rewards/margins": 1.4006534814834595,
      "rewards/rejected": 1.2119553089141846,
      "step": 865
    },
    {
      "drift/chosen_abs": 0.4436754286289215,
      "drift/rejected_abs": 0.2094762623310089,
      "epoch": 0.8312432819777857,
      "grad_norm": 46.75,
      "learning_rate": 9.024223931035357e-10,
      "logits/chosen": -1.9862560033798218,
      "logits/rejected": -1.9510629177093506,
      "logps/chosen": -0.29855862259864807,
      "logps/rejected": -0.28159067034721375,
      "loss": 1.7128210067749023,
      "loss/core": 1.6952579021453857,
      "loss/preference_sft": 0.29855862259864807,
      "loss/reference_anchor": 0.8482988476753235,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.433295726776123,
      "rewards/margins": 2.3404412269592285,
      "rewards/rejected": 2.0928549766540527,
      "step": 870
    },
    {
      "drift/chosen_abs": 0.19628901779651642,
      "drift/rejected_abs": 0.1381283849477768,
      "epoch": 0.8360205422190374,
      "grad_norm": 0.75390625,
      "learning_rate": 6.350214026223516e-10,
      "logits/chosen": -2.086601972579956,
      "logits/rejected": -2.0887794494628906,
      "logps/chosen": -0.3021940588951111,
      "logps/rejected": -0.30490565299987793,
      "loss": 0.4931362569332123,
      "loss/core": 0.48765283823013306,
      "loss/preference_sft": 0.3021940588951111,
      "loss/reference_anchor": 0.24395246803760529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9610573053359985,
      "rewards/margins": 0.5812469720840454,
      "rewards/rejected": 1.379810094833374,
      "step": 875
    },
    {
      "drift/chosen_abs": 0.37091922760009766,
      "drift/rejected_abs": 0.14499905705451965,
      "epoch": 0.840797802460289,
      "grad_norm": 0.443359375,
      "learning_rate": 4.143889944367429e-10,
      "logits/chosen": -2.001990795135498,
      "logits/rejected": -1.9710228443145752,
      "logps/chosen": -0.28516069054603577,
      "logps/rejected": -0.30990806221961975,
      "loss": 1.4451124668121338,
      "loss/core": 1.4302270412445068,
      "loss/preference_sft": 0.28516069054603577,
      "loss/reference_anchor": 0.7157606482505798,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.7089836597442627,
      "rewards/margins": 2.2644705772399902,
      "rewards/rejected": 1.444512963294983,
      "step": 880
    },
    {
      "drift/chosen_abs": 0.15534690022468567,
      "drift/rejected_abs": 0.07884477078914642,
      "epoch": 0.8455750627015407,
      "grad_norm": 14.1875,
      "learning_rate": 2.406081393722531e-10,
      "logits/chosen": -2.0966742038726807,
      "logits/rejected": -2.1267499923706055,
      "logps/chosen": -0.2981413006782532,
      "logps/rejected": -0.30681678652763367,
      "loss": 0.11275926977396011,
      "loss/core": 0.11104929447174072,
      "loss/preference_sft": 0.2981413006782532,
      "loss/reference_anchor": 0.05568493530154228,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5524650812149048,
      "rewards/margins": 0.7652949094772339,
      "rewards/rejected": 0.7871701121330261,
      "step": 885
    },
    {
      "drift/chosen_abs": 0.18543264269828796,
      "drift/rejected_abs": 0.16110344231128693,
      "epoch": 0.8503523229427923,
      "grad_norm": 14.9375,
      "learning_rate": 1.1374418930135133e-10,
      "logits/chosen": -2.1319258213043213,
      "logits/rejected": -2.1853280067443848,
      "logps/chosen": -0.26988840103149414,
      "logps/rejected": -0.2897631525993347,
      "loss": 0.5226936340332031,
      "loss/core": 0.5169836282730103,
      "loss/preference_sft": 0.26988840103149414,
      "loss/reference_anchor": 0.25851255655288696,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8515974283218384,
      "rewards/margins": 0.24080824851989746,
      "rewards/rejected": 1.6107892990112305,
      "step": 890
    },
    {
      "drift/chosen_abs": 0.21447238326072693,
      "drift/rejected_abs": 0.09845264256000519,
      "epoch": 0.855129583184044,
      "grad_norm": 176.0,
      "learning_rate": 3.3844852567285756e-11,
      "logits/chosen": -2.092803478240967,
      "logits/rejected": -2.0520870685577393,
      "logps/chosen": -0.289115846157074,
      "logps/rejected": -0.2914557456970215,
      "loss": 0.28112271428108215,
      "loss/core": 0.2777610719203949,
      "loss/preference_sft": 0.289115846157074,
      "loss/reference_anchor": 0.1391705721616745,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.144585132598877,
      "rewards/margins": 1.1624013185501099,
      "rewards/rejected": 0.9821839332580566,
      "step": 895
    },
    {
      "drift/chosen_abs": 0.29047515988349915,
      "drift/rejected_abs": 0.14344081282615662,
      "epoch": 0.8599068434252956,
      "grad_norm": 89.0,
      "learning_rate": 9.401760429905703e-13,
      "logits/chosen": -2.0498125553131104,
      "logits/rejected": -2.014761447906494,
      "logps/chosen": -0.26901915669441223,
      "logps/rejected": -0.2722063958644867,
      "loss": 0.7836966514587402,
      "loss/core": 0.7753987312316895,
      "loss/preference_sft": 0.26901915669441223,
      "loss/reference_anchor": 0.3879950940608978,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9047515392303467,
      "rewards/margins": 1.4703437089920044,
      "rewards/rejected": 1.4344079494476318,
      "step": 900
    },
    {
      "epoch": 0.8599068434252956,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 0.688900715245141,
      "train_runtime": 7064.5608,
      "train_samples_per_second": 2.038,
      "train_steps_per_second": 0.127
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}