{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8599068434252956,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "drift/chosen_abs": 0.3419838547706604,
      "drift/rejected_abs": 0.14085355401039124,
      "epoch": 0.004777260241251642,
      "grad_norm": 7168.0,
      "learning_rate": 1.111111111111111e-08,
      "logits/chosen": -2.274840831756592,
      "logits/rejected": -2.238874912261963,
      "logps/chosen": -0.30753669142723083,
      "logps/rejected": -0.3066186010837555,
      "loss": 4418.6552734375,
      "loss/core": 4418.6337890625,
      "loss/preference_sft": 0.30753669142723083,
      "loss/reference_anchor": 1.068562388420105,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4167866706848145,
      "rewards/margins": 2.012580156326294,
      "rewards/rejected": 1.4042068719863892,
      "step": 5
    },
    {
      "drift/chosen_abs": 0.29194900393486023,
      "drift/rejected_abs": 0.13348758220672607,
      "epoch": 0.009554520482503284,
      "grad_norm": 1528.0,
      "learning_rate": 2.5e-08,
      "logits/chosen": -2.2050859928131104,
      "logits/rejected": -2.1969170570373535,
      "logps/chosen": -0.29071494936943054,
      "logps/rejected": -0.2801753282546997,
      "loss": 4423.58544921875,
      "loss/core": 4423.5791015625,
      "loss/preference_sft": 0.29071494936943054,
      "loss/reference_anchor": 0.28804466128349304,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.919328212738037,
      "rewards/margins": 1.5846813917160034,
      "rewards/rejected": 1.3346469402313232,
      "step": 10
    },
    {
      "drift/chosen_abs": 0.1922224462032318,
      "drift/rejected_abs": 0.08871154487133026,
      "epoch": 0.014331780723754926,
      "grad_norm": 700.0,
      "learning_rate": 3.888888888888889e-08,
      "logits/chosen": -1.8447561264038086,
      "logits/rejected": -1.9001595973968506,
      "logps/chosen": -0.29011744260787964,
      "logps/rejected": -0.28691497445106506,
      "loss": 4430.5283203125,
      "loss/core": 4430.525390625,
      "loss/preference_sft": 0.29011744260787964,
      "loss/reference_anchor": 0.11491169780492783,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.9212669134140015,
      "rewards/margins": 1.0505340099334717,
      "rewards/rejected": 0.8707327842712402,
      "step": 15
    },
    {
      "drift/chosen_abs": 0.16317619383335114,
      "drift/rejected_abs": 0.05642751604318619,
      "epoch": 0.01910904096500657,
      "grad_norm": 676.0,
      "learning_rate": 5.2777777777777776e-08,
      "logits/chosen": -2.1695523262023926,
      "logits/rejected": -2.097202777862549,
      "logps/chosen": -0.2871994376182556,
      "logps/rejected": -0.28375035524368286,
      "loss": 4430.1630859375,
      "loss/core": 4430.16064453125,
      "loss/preference_sft": 0.2871994376182556,
      "loss/reference_anchor": 0.07742591947317123,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6311792135238647,
      "rewards/margins": 1.0771574974060059,
      "rewards/rejected": 0.5540218353271484,
      "step": 20
    },
    {
      "drift/chosen_abs": 0.18135179579257965,
      "drift/rejected_abs": 0.11895482242107391,
      "epoch": 0.02388630120625821,
      "grad_norm": 1336.0,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -2.2033162117004395,
      "logits/rejected": -2.218423366546631,
      "logps/chosen": -0.27016180753707886,
      "logps/rejected": -0.2789779007434845,
      "loss": 4436.3212890625,
      "loss/core": 4436.31787109375,
      "loss/preference_sft": 0.27016180753707886,
      "loss/reference_anchor": 0.13152866065502167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8121395111083984,
      "rewards/margins": 0.6226418018341064,
      "rewards/rejected": 1.1894975900650024,
      "step": 25
    },
    {
      "drift/chosen_abs": 0.26979273557662964,
      "drift/rejected_abs": 0.18020370602607727,
      "epoch": 0.028663561447509853,
      "grad_norm": 3728.0,
      "learning_rate": 8.055555555555555e-08,
      "logits/chosen": -2.1017913818359375,
      "logits/rejected": -2.063615083694458,
      "logps/chosen": -0.3164452612400055,
      "logps/rejected": -0.2725190818309784,
      "loss": 4433.0703125,
      "loss/core": 4433.05859375,
      "loss/preference_sft": 0.3164452612400055,
      "loss/reference_anchor": 0.557770848274231,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6958069801330566,
      "rewards/margins": 0.8953614234924316,
      "rewards/rejected": 1.800445318222046,
      "step": 30
    },
    {
      "drift/chosen_abs": 0.18841537833213806,
      "drift/rejected_abs": 0.08009415119886398,
      "epoch": 0.033440821688761495,
      "grad_norm": 912.0,
      "learning_rate": 9.444444444444444e-08,
      "logits/chosen": -2.1271777153015137,
      "logits/rejected": -2.22860050201416,
      "logps/chosen": -0.2820509970188141,
      "logps/rejected": -0.2648252844810486,
      "loss": 4429.8935546875,
      "loss/core": 4429.8916015625,
      "loss/preference_sft": 0.2820509970188141,
      "loss/reference_anchor": 0.08949244022369385,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.883462905883789,
      "rewards/margins": 1.0993850231170654,
      "rewards/rejected": 0.7840781211853027,
      "step": 35
    },
    {
      "drift/chosen_abs": 0.2343977689743042,
      "drift/rejected_abs": 0.09806372225284576,
      "epoch": 0.03821808193001314,
      "grad_norm": 1048.0,
      "learning_rate": 1.0833333333333334e-07,
      "logits/chosen": -2.1154208183288574,
      "logits/rejected": -2.1933438777923584,
      "logps/chosen": -0.29859572649002075,
      "logps/rejected": -0.29969868063926697,
      "loss": 4426.380859375,
      "loss/core": 4426.3779296875,
      "loss/preference_sft": 0.29859572649002075,
      "loss/reference_anchor": 0.15946529805660248,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.3425917625427246,
      "rewards/margins": 1.366955280303955,
      "rewards/rejected": 0.9756364822387695,
      "step": 40
    },
    {
      "drift/chosen_abs": 0.28305283188819885,
      "drift/rejected_abs": 0.1835290640592575,
      "epoch": 0.04299534217126478,
      "grad_norm": 4864.0,
      "learning_rate": 1.2222222222222222e-07,
      "logits/chosen": -2.038555383682251,
      "logits/rejected": -2.0794358253479004,
      "logps/chosen": -0.28872132301330566,
      "logps/rejected": -0.29276594519615173,
      "loss": 4432.30322265625,
      "loss/core": 4432.2890625,
      "loss/preference_sft": 0.28872132301330566,
      "loss/reference_anchor": 0.6807764172554016,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.8277623653411865,
      "rewards/margins": 0.9989973306655884,
      "rewards/rejected": 1.8287651538848877,
      "step": 45
    },
    {
      "drift/chosen_abs": 0.18194088339805603,
      "drift/rejected_abs": 0.13804134726524353,
      "epoch": 0.04777260241251642,
      "grad_norm": 430.0,
      "learning_rate": 1.3611111111111108e-07,
      "logits/chosen": -2.107499122619629,
      "logits/rejected": -2.1948068141937256,
      "logps/chosen": -0.28583410382270813,
      "logps/rejected": -0.289579838514328,
      "loss": 4438.6875,
      "loss/core": 4438.68359375,
      "loss/preference_sft": 0.28583410382270813,
      "loss/reference_anchor": 0.17273597419261932,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.818098783493042,
      "rewards/margins": 0.4417288303375244,
      "rewards/rejected": 1.3763700723648071,
      "step": 50
    },
    {
      "drift/chosen_abs": 0.16760578751564026,
      "drift/rejected_abs": 0.09244702756404877,
      "epoch": 0.05254986265376806,
      "grad_norm": 660.0,
      "learning_rate": 1.5e-07,
      "logits/chosen": -2.1269633769989014,
      "logits/rejected": -2.2465131282806396,
      "logps/chosen": -0.2968710958957672,
      "logps/rejected": -0.29279255867004395,
      "loss": 4434.4453125,
      "loss/core": 4434.4443359375,
      "loss/preference_sft": 0.2968710958957672,
      "loss/reference_anchor": 0.05782536417245865,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6760451793670654,
      "rewards/margins": 0.7522833347320557,
      "rewards/rejected": 0.9237619638442993,
      "step": 55
    },
    {
      "drift/chosen_abs": 0.14607155323028564,
      "drift/rejected_abs": 0.10737241804599762,
      "epoch": 0.057327122895019705,
      "grad_norm": 2272.0,
      "learning_rate": 1.6388888888888888e-07,
      "logits/chosen": -2.11145281791687,
      "logits/rejected": -2.1462674140930176,
      "logps/chosen": -0.29008573293685913,
      "logps/rejected": -0.2873132824897766,
      "loss": 4439.5068359375,
      "loss/core": 4439.5029296875,
      "loss/preference_sft": 0.29008573293685913,
      "loss/reference_anchor": 0.1456277072429657,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.459886908531189,
      "rewards/margins": 0.387167751789093,
      "rewards/rejected": 1.0727192163467407,
      "step": 60
    },
    {
      "drift/chosen_abs": 0.24463272094726562,
      "drift/rejected_abs": 0.09055524319410324,
      "epoch": 0.06210438313627135,
      "grad_norm": 2912.0,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -2.143834114074707,
      "logits/rejected": -2.2185416221618652,
      "logps/chosen": -0.3164866268634796,
      "logps/rejected": -0.3099859356880188,
      "loss": 4424.44091796875,
      "loss/core": 4424.4326171875,
      "loss/preference_sft": 0.3164866268634796,
      "loss/reference_anchor": 0.37436532974243164,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4456965923309326,
      "rewards/margins": 1.5431140661239624,
      "rewards/rejected": 0.9025824666023254,
      "step": 65
    },
    {
      "drift/chosen_abs": 0.23293833434581757,
      "drift/rejected_abs": 0.10600552707910538,
      "epoch": 0.06688164337752299,
      "grad_norm": 8192.0,
      "learning_rate": 1.9166666666666668e-07,
      "logits/chosen": -2.1075563430786133,
      "logits/rejected": -2.165278434753418,
      "logps/chosen": -0.3008618950843811,
      "logps/rejected": -0.30584263801574707,
      "loss": 4426.07080078125,
      "loss/core": 4426.06494140625,
      "loss/preference_sft": 0.3008618950843811,
      "loss/reference_anchor": 0.24336305260658264,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.329383373260498,
      "rewards/margins": 1.4130064249038696,
      "rewards/rejected": 0.9163768887519836,
      "step": 70
    },
    {
      "drift/chosen_abs": 0.1538292020559311,
      "drift/rejected_abs": 0.10185450315475464,
      "epoch": 0.07165890361877464,
      "grad_norm": 804.0,
      "learning_rate": 2.0555555555555553e-07,
      "logits/chosen": -2.0664281845092773,
      "logits/rejected": -2.1418495178222656,
      "logps/chosen": -0.2909933924674988,
      "logps/rejected": -0.2969188988208771,
      "loss": 4437.65087890625,
      "loss/core": 4437.6474609375,
      "loss/preference_sft": 0.2909933924674988,
      "loss/reference_anchor": 0.10792021453380585,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.5364402532577515,
      "rewards/margins": 0.5191212296485901,
      "rewards/rejected": 1.0173190832138062,
      "step": 75
    },
    {
      "drift/chosen_abs": 0.12916477024555206,
      "drift/rejected_abs": 0.07108369469642639,
      "epoch": 0.07643616386002627,
      "grad_norm": 892.0,
      "learning_rate": 2.1944444444444442e-07,
      "logits/chosen": -2.2707951068878174,
      "logits/rejected": -2.3466694355010986,
      "logps/chosen": -0.30908656120300293,
      "logps/rejected": -0.30672696232795715,
      "loss": 4436.71728515625,
      "loss/core": 4436.7158203125,
      "loss/preference_sft": 0.30908656120300293,
      "loss/reference_anchor": 0.04678858071565628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.291474461555481,
      "rewards/margins": 0.5815840363502502,
      "rewards/rejected": 0.7098905444145203,
      "step": 80
    },
    {
      "drift/chosen_abs": 0.24453523755073547,
      "drift/rejected_abs": 0.1695018708705902,
      "epoch": 0.08121342410127792,
      "grad_norm": 556.0,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -2.1169114112854004,
      "logits/rejected": -2.1277921199798584,
      "logps/chosen": -0.2921332120895386,
      "logps/rejected": -0.28643548488616943,
      "loss": 4435.0263671875,
      "loss/core": 4435.017578125,
      "loss/preference_sft": 0.2921332120895386,
      "loss/reference_anchor": 0.4582666754722595,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.4450902938842773,
      "rewards/margins": 0.7517923712730408,
      "rewards/rejected": 1.6932977437973022,
      "step": 85
    },
    {
      "drift/chosen_abs": 0.28909215331077576,
      "drift/rejected_abs": 0.06680949032306671,
      "epoch": 0.08599068434252956,
      "grad_norm": 1080.0,
      "learning_rate": 2.4722222222222224e-07,
      "logits/chosen": -2.001685380935669,
      "logits/rejected": -2.1864590644836426,
      "logps/chosen": -0.3068524897098541,
      "logps/rejected": -0.30106669664382935,
      "loss": 4415.2734375,
      "loss/core": 4415.2666015625,
      "loss/preference_sft": 0.3068524897098541,
      "loss/reference_anchor": 0.31496793031692505,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.889475107192993,
      "rewards/margins": 2.222256898880005,
      "rewards/rejected": 0.6672185659408569,
      "step": 90
    },
    {
      "drift/chosen_abs": 0.2167942225933075,
      "drift/rejected_abs": 0.13755588233470917,
      "epoch": 0.09076794458378121,
      "grad_norm": 2912.0,
      "learning_rate": 2.4998495746616845e-07,
      "logits/chosen": -2.0368285179138184,
      "logits/rejected": -1.9507942199707031,
      "logps/chosen": -0.28156915307044983,
      "logps/rejected": -0.30304011702537537,
      "loss": 4434.26171875,
      "loss/core": 4434.25439453125,
      "loss/preference_sft": 0.28156915307044983,
      "loss/reference_anchor": 0.33130401372909546,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.167041063308716,
      "rewards/margins": 0.7925023436546326,
      "rewards/rejected": 1.3745384216308594,
      "step": 95
    },
    {
      "drift/chosen_abs": 0.29789382219314575,
      "drift/rejected_abs": 0.1836269050836563,
      "epoch": 0.09554520482503284,
      "grad_norm": 600.0,
      "learning_rate": 2.4992385337738696e-07,
      "logits/chosen": -2.0101962089538574,
      "logits/rejected": -2.0421595573425293,
      "logps/chosen": -0.2820836007595062,
      "logps/rejected": -0.31330883502960205,
      "loss": 4426.01904296875,
      "loss/core": 4426.0087890625,
      "loss/preference_sft": 0.2820836007595062,
      "loss/reference_anchor": 0.4864688813686371,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9787633419036865,
      "rewards/margins": 1.4464952945709229,
      "rewards/rejected": 1.5322680473327637,
      "step": 100
    },
    {
      "drift/chosen_abs": 0.3174459934234619,
      "drift/rejected_abs": 0.13966134190559387,
      "epoch": 0.10032246506628449,
      "grad_norm": 812.0,
      "learning_rate": 2.4981577053636144e-07,
      "logits/chosen": -1.9345197677612305,
      "logits/rejected": -1.9873930215835571,
      "logps/chosen": -0.2772332727909088,
      "logps/rejected": -0.2635360360145569,
      "loss": 4421.1513671875,
      "loss/core": 4421.1435546875,
      "loss/preference_sft": 0.2772332727909088,
      "loss/reference_anchor": 0.37949052453041077,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.1736035346984863,
      "rewards/margins": 1.7784216403961182,
      "rewards/rejected": 1.3951818943023682,
      "step": 105
    },
    {
      "drift/chosen_abs": 0.21933257579803467,
      "drift/rejected_abs": 0.10228520631790161,
      "epoch": 0.10509972530753613,
      "grad_norm": 556.0,
      "learning_rate": 2.496607495886281e-07,
      "logits/chosen": -2.123267412185669,
      "logits/rejected": -2.0964272022247314,
      "logps/chosen": -0.2958363890647888,
      "logps/rejected": -0.30722880363464355,
      "loss": 4429.02294921875,
      "loss/core": 4429.0166015625,
      "loss/preference_sft": 0.2958363890647888,
      "loss/reference_anchor": 0.31792789697647095,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.1888623237609863,
      "rewards/margins": 1.1705862283706665,
      "rewards/rejected": 1.0182762145996094,
      "step": 110
    },
    {
      "drift/chosen_abs": 0.3494819402694702,
      "drift/rejected_abs": 0.1267239898443222,
      "epoch": 0.10987698554878778,
      "grad_norm": 5792.0,
      "learning_rate": 2.4945884883122553e-07,
      "logits/chosen": -2.022082805633545,
      "logits/rejected": -2.070870876312256,
      "logps/chosen": -0.2908271253108978,
      "logps/rejected": -0.28468307852745056,
      "loss": 4415.51806640625,
      "loss/core": 4415.5029296875,
      "loss/preference_sft": 0.2908271253108978,
      "loss/reference_anchor": 0.6909618377685547,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.49409556388855,
      "rewards/margins": 2.2313833236694336,
      "rewards/rejected": 1.2627122402191162,
      "step": 115
    },
    {
      "drift/chosen_abs": 0.18104346096515656,
      "drift/rejected_abs": 0.140670508146286,
      "epoch": 0.11465424579003941,
      "grad_norm": 9856.0,
      "learning_rate": 2.492101441907714e-07,
      "logits/chosen": -2.241240978240967,
      "logits/rejected": -2.1032841205596924,
      "logps/chosen": -0.2601907551288605,
      "logps/rejected": -0.28034669160842896,
      "loss": 4439.17724609375,
      "loss/core": 4439.17431640625,
      "loss/preference_sft": 0.2601907551288605,
      "loss/reference_anchor": 0.14795775711536407,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8099110126495361,
      "rewards/margins": 0.4032059609889984,
      "rewards/rejected": 1.4067050218582153,
      "step": 120
    },
    {
      "drift/chosen_abs": 0.408181756734848,
      "drift/rejected_abs": 0.10417692363262177,
      "epoch": 0.11943150603129106,
      "grad_norm": 422.0,
      "learning_rate": 2.4891472919490977e-07,
      "logits/chosen": -1.9139236211776733,
      "logits/rejected": -1.9596039056777954,
      "logps/chosen": -0.2813839018344879,
      "logps/rejected": -0.2793610095977783,
      "loss": 4404.7919921875,
      "loss/core": 4404.77392578125,
      "loss/preference_sft": 0.2813839018344879,
      "loss/reference_anchor": 0.8949528932571411,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 4.081476211547852,
      "rewards/margins": 3.0719616413116455,
      "rewards/rejected": 1.0095146894454956,
      "step": 125
    },
    {
      "drift/chosen_abs": 0.2684691548347473,
      "drift/rejected_abs": 0.1741405725479126,
      "epoch": 0.1242087662725427,
      "grad_norm": 8576.0,
      "learning_rate": 2.4857271493713894e-07,
      "logits/chosen": -2.075075626373291,
      "logits/rejected": -2.073489189147949,
      "logps/chosen": -0.29706940054893494,
      "logps/rejected": -0.2922501266002655,
      "loss": 4431.94091796875,
      "loss/core": 4431.92919921875,
      "loss/preference_sft": 0.29706940054893494,
      "loss/reference_anchor": 0.525195300579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6840920448303223,
      "rewards/margins": 0.9443261027336121,
      "rewards/rejected": 1.7397658824920654,
      "step": 130
    },
    {
      "drift/chosen_abs": 0.17242693901062012,
      "drift/rejected_abs": 0.0899103656411171,
      "epoch": 0.12898602651379434,
      "grad_norm": 1224.0,
      "learning_rate": 2.481842300350339e-07,
      "logits/chosen": -2.03088641166687,
      "logits/rejected": -2.094935894012451,
      "logps/chosen": -0.27687835693359375,
      "logps/rejected": -0.26349860429763794,
      "loss": 4433.5263671875,
      "loss/core": 4433.5234375,
      "loss/preference_sft": 0.27687835693359375,
      "loss/reference_anchor": 0.09716491401195526,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.7209184169769287,
      "rewards/margins": 0.8248888850212097,
      "rewards/rejected": 0.8960295915603638,
      "step": 135
    },
    {
      "drift/chosen_abs": 0.22251293063163757,
      "drift/rejected_abs": 0.09145991504192352,
      "epoch": 0.13376328675504598,
      "grad_norm": 500.0,
      "learning_rate": 2.4774942058187854e-07,
      "logits/chosen": -2.1904947757720947,
      "logits/rejected": -2.24971079826355,
      "logps/chosen": -0.2990642189979553,
      "logps/rejected": -0.2923958897590637,
      "loss": 4427.3486328125,
      "loss/core": 4427.3408203125,
      "loss/preference_sft": 0.2990642189979553,
      "loss/reference_anchor": 0.39207085967063904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.223792314529419,
      "rewards/margins": 1.3103091716766357,
      "rewards/rejected": 0.9134831428527832,
      "step": 140
    },
    {
      "drift/chosen_abs": 0.18590529263019562,
      "drift/rejected_abs": 0.10894735902547836,
      "epoch": 0.13854054699629761,
      "grad_norm": 1584.0,
      "learning_rate": 2.472684500917257e-07,
      "logits/chosen": -1.9672470092773438,
      "logits/rejected": -1.9949979782104492,
      "logps/chosen": -0.33192163705825806,
      "logps/rejected": -0.3151020109653473,
      "loss": 4434.21435546875,
      "loss/core": 4434.2119140625,
      "loss/preference_sft": 0.33192163705825806,
      "loss/reference_anchor": 0.10941161960363388,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8581035137176514,
      "rewards/margins": 0.773113489151001,
      "rewards/rejected": 1.0849900245666504,
      "step": 145
    },
    {
      "drift/chosen_abs": 0.15106205642223358,
      "drift/rejected_abs": 0.05738229304552078,
      "epoch": 0.14331780723754928,
      "grad_norm": 624.0,
      "learning_rate": 2.467414994379065e-07,
      "logits/chosen": -2.0708765983581543,
      "logits/rejected": -2.1375513076782227,
      "logps/chosen": -0.30995962023735046,
      "logps/rejected": -0.31707248091697693,
      "loss": 4432.21533203125,
      "loss/core": 4432.2119140625,
      "loss/preference_sft": 0.30995962023735046,
      "loss/reference_anchor": 0.15558083355426788,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5100862979888916,
      "rewards/margins": 0.9373763799667358,
      "rewards/rejected": 0.5727099776268005,
      "step": 150
    },
    {
      "drift/chosen_abs": 0.15896198153495789,
      "drift/rejected_abs": 0.08986209332942963,
      "epoch": 0.1480950674788009,
      "grad_norm": 1496.0,
      "learning_rate": 2.4616876678501114e-07,
      "logits/chosen": -2.2561638355255127,
      "logits/rejected": -2.3034205436706543,
      "logps/chosen": -0.31711262464523315,
      "logps/rejected": -0.302999347448349,
      "loss": 4435.2392578125,
      "loss/core": 4435.23681640625,
      "loss/preference_sft": 0.31711262464523315,
      "loss/reference_anchor": 0.06949084997177124,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.588337779045105,
      "rewards/margins": 0.6926823258399963,
      "rewards/rejected": 0.8956555128097534,
      "step": 155
    },
    {
      "drift/chosen_abs": 0.3718644082546234,
      "drift/rejected_abs": 0.11488336324691772,
      "epoch": 0.15287232772005255,
      "grad_norm": 13056.0,
      "learning_rate": 2.4555046751436735e-07,
      "logits/chosen": -2.1315560340881348,
      "logits/rejected": -2.117034912109375,
      "logps/chosen": -0.29637980461120605,
      "logps/rejected": -0.282162606716156,
      "loss": 4410.93359375,
      "loss/core": 4410.9189453125,
      "loss/preference_sft": 0.29637980461120605,
      "loss/reference_anchor": 0.6615738868713379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.717616558074951,
      "rewards/margins": 2.570777416229248,
      "rewards/rejected": 1.1468391418457031,
      "step": 160
    },
    {
      "drift/chosen_abs": 0.18394318222999573,
      "drift/rejected_abs": 0.09417982399463654,
      "epoch": 0.15764958796130418,
      "grad_norm": 944.0,
      "learning_rate": 2.4488683414304425e-07,
      "logits/chosen": -2.132768392562866,
      "logits/rejected": -2.2345738410949707,
      "logps/chosen": -0.31837791204452515,
      "logps/rejected": -0.31375187635421753,
      "loss": 4432.53857421875,
      "loss/core": 4432.53564453125,
      "loss/preference_sft": 0.31837791204452515,
      "loss/reference_anchor": 0.1169905811548233,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8377997875213623,
      "rewards/margins": 0.896827220916748,
      "rewards/rejected": 0.940972626209259,
      "step": 165
    },
    {
      "drift/chosen_abs": 0.15466679632663727,
      "drift/rejected_abs": 0.09628058969974518,
      "epoch": 0.16242684820255585,
      "grad_norm": 1496.0,
      "learning_rate": 2.4417811623641203e-07,
      "logits/chosen": -2.274855136871338,
      "logits/rejected": -2.3352713584899902,
      "logps/chosen": -0.27421072125434875,
      "logps/rejected": -0.2842170298099518,
      "loss": 4436.6865234375,
      "loss/core": 4436.68359375,
      "loss/preference_sft": 0.27421072125434875,
      "loss/reference_anchor": 0.11561869084835052,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.5460879802703857,
      "rewards/margins": 0.5835286378860474,
      "rewards/rejected": 0.9625593423843384,
      "step": 170
    },
    {
      "drift/chosen_abs": 0.5257049798965454,
      "drift/rejected_abs": 0.19930550456047058,
      "epoch": 0.16720410844380748,
      "grad_norm": 1896.0,
      "learning_rate": 2.4342458031429113e-07,
      "logits/chosen": -1.8325287103652954,
      "logits/rejected": -1.8387473821640015,
      "logps/chosen": -0.2876504957675934,
      "logps/rejected": -0.29608818888664246,
      "loss": 4401.90185546875,
      "loss/core": 4401.876953125,
      "loss/preference_sft": 0.2876504957675934,
      "loss/reference_anchor": 1.251502275466919,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 5.2567338943481445,
      "rewards/margins": 3.286668300628662,
      "rewards/rejected": 1.9700653553009033,
      "step": 175
    },
    {
      "drift/chosen_abs": 0.3301013112068176,
      "drift/rejected_abs": 0.15288423001766205,
      "epoch": 0.17198136868505912,
      "grad_norm": 458.0,
      "learning_rate": 2.4262650975072444e-07,
      "logits/chosen": -2.189413070678711,
      "logits/rejected": -2.3054258823394775,
      "logps/chosen": -0.2844337224960327,
      "logps/rejected": -0.2936963140964508,
      "loss": 4420.90087890625,
      "loss/core": 4420.8896484375,
      "loss/preference_sft": 0.2844337224960327,
      "loss/reference_anchor": 0.5586980581283569,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 3.300053119659424,
      "rewards/margins": 1.8004553318023682,
      "rewards/rejected": 1.4995979070663452,
      "step": 180
    },
    {
      "drift/chosen_abs": 0.1918279230594635,
      "drift/rejected_abs": 0.07734932750463486,
      "epoch": 0.17675862892631075,
      "grad_norm": 704.0,
      "learning_rate": 2.417842046674122e-07,
      "logits/chosen": -2.3335790634155273,
      "logits/rejected": -2.3553173542022705,
      "logps/chosen": -0.2591295838356018,
      "logps/rejected": -0.26806166768074036,
      "loss": 4429.29296875,
      "loss/core": 4429.2900390625,
      "loss/preference_sft": 0.2591295838356018,
      "loss/reference_anchor": 0.09960640221834183,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9182792901992798,
      "rewards/margins": 1.1459853649139404,
      "rewards/rejected": 0.7722936868667603,
      "step": 185
    },
    {
      "drift/chosen_abs": 0.19113460183143616,
      "drift/rejected_abs": 0.09281430393457413,
      "epoch": 0.18153588916756241,
      "grad_norm": 564.0,
      "learning_rate": 2.4089798182084843e-07,
      "logits/chosen": -2.0862205028533936,
      "logits/rejected": -2.1600217819213867,
      "logps/chosen": -0.30776557326316833,
      "logps/rejected": -0.30735090374946594,
      "loss": 4431.3701171875,
      "loss/core": 4431.36669921875,
      "loss/preference_sft": 0.30776557326316833,
      "loss/reference_anchor": 0.1401655375957489,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9096482992172241,
      "rewards/margins": 0.9876093864440918,
      "rewards/rejected": 0.9220390319824219,
      "step": 190
    },
    {
      "drift/chosen_abs": 0.21678169071674347,
      "drift/rejected_abs": 0.13430018723011017,
      "epoch": 0.18631314940881405,
      "grad_norm": 704.0,
      "learning_rate": 2.3996817448320195e-07,
      "logits/chosen": -2.158468246459961,
      "logits/rejected": -2.112973213195801,
      "logps/chosen": -0.28618916869163513,
      "logps/rejected": -0.29950863122940063,
      "loss": 4432.5107421875,
      "loss/core": 4432.50537109375,
      "loss/preference_sft": 0.28618916869163513,
      "loss/reference_anchor": 0.24302752315998077,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.1672415733337402,
      "rewards/margins": 0.914328932762146,
      "rewards/rejected": 1.2529126405715942,
      "step": 195
    },
    {
      "drift/chosen_abs": 0.28183844685554504,
      "drift/rejected_abs": 0.15761585533618927,
      "epoch": 0.19109040965006568,
      "grad_norm": 1928.0,
      "learning_rate": 2.3899513231698607e-07,
      "logits/chosen": -2.1264212131500244,
      "logits/rejected": -2.0478241443634033,
      "logps/chosen": -0.292626291513443,
      "logps/rejected": -0.28839462995529175,
      "loss": 4428.09912109375,
      "loss/core": 4428.0927734375,
      "loss/preference_sft": 0.292626291513443,
      "loss/reference_anchor": 0.28291893005371094,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.8163602352142334,
      "rewards/margins": 1.2429499626159668,
      "rewards/rejected": 1.5734103918075562,
      "step": 200
    },
    {
      "drift/chosen_abs": 0.26241064071655273,
      "drift/rejected_abs": 0.14250826835632324,
      "epoch": 0.19586766989131732,
      "grad_norm": 2240.0,
      "learning_rate": 2.3797922124356506e-07,
      "logits/chosen": -2.029865264892578,
      "logits/rejected": -2.0273633003234863,
      "logps/chosen": -0.39078933000564575,
      "logps/rejected": -0.30789244174957275,
      "loss": 4427.798828125,
      "loss/core": 4427.78955078125,
      "loss/preference_sft": 0.39078933000564575,
      "loss/reference_anchor": 0.41654643416404724,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.6241068840026855,
      "rewards/margins": 1.2998729944229126,
      "rewards/rejected": 1.3242336511611938,
      "step": 205
    },
    {
      "drift/chosen_abs": 0.17145346105098724,
      "drift/rejected_abs": 0.07196025550365448,
      "epoch": 0.20064493013256898,
      "grad_norm": 532.0,
      "learning_rate": 2.3692082330554585e-07,
      "logits/chosen": -2.2569234371185303,
      "logits/rejected": -2.352635145187378,
      "logps/chosen": -0.25879400968551636,
      "logps/rejected": -0.2546376585960388,
      "loss": 4431.224609375,
      "loss/core": 4431.22265625,
      "loss/preference_sft": 0.25879400968551636,
      "loss/reference_anchor": 0.05695471167564392,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7141616344451904,
      "rewards/margins": 0.9953814744949341,
      "rewards/rejected": 0.7187802791595459,
      "step": 210
    },
    {
      "drift/chosen_abs": 0.2754186987876892,
      "drift/rejected_abs": 0.17294225096702576,
      "epoch": 0.20542219037382062,
      "grad_norm": 6752.0,
      "learning_rate": 2.3582033652310765e-07,
      "logits/chosen": -1.9878219366073608,
      "logits/rejected": -1.9628711938858032,
      "logps/chosen": -0.2865174412727356,
      "logps/rejected": -0.27714353799819946,
      "loss": 4431.0849609375,
      "loss/core": 4431.0791015625,
      "loss/preference_sft": 0.2865174412727356,
      "loss/reference_anchor": 0.26327061653137207,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.7541871070861816,
      "rewards/margins": 1.025450587272644,
      "rewards/rejected": 1.7287365198135376,
      "step": 215
    },
    {
      "drift/chosen_abs": 0.3495247960090637,
      "drift/rejected_abs": 0.07642688602209091,
      "epoch": 0.21019945061507225,
      "grad_norm": 544.0,
      "learning_rate": 2.346781747443227e-07,
      "logits/chosen": -2.042937755584717,
      "logits/rejected": -2.1352648735046387,
      "logps/chosen": -0.29404938220977783,
      "logps/rejected": -0.28113672137260437,
      "loss": 4409.341796875,
      "loss/core": 4409.32666015625,
      "loss/preference_sft": 0.29404938220977783,
      "loss/reference_anchor": 0.7147288918495178,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.494548797607422,
      "rewards/margins": 2.73149037361145,
      "rewards/rejected": 0.7630582451820374,
      "step": 220
    },
    {
      "drift/chosen_abs": 0.3245381712913513,
      "drift/rejected_abs": 0.10127030313014984,
      "epoch": 0.2149767108563239,
      "grad_norm": 3024.0,
      "learning_rate": 2.3349476748952508e-07,
      "logits/chosen": -2.1589510440826416,
      "logits/rejected": -2.2461776733398438,
      "logps/chosen": -0.27290159463882446,
      "logps/rejected": -0.27267274260520935,
      "loss": 4415.03759765625,
      "loss/core": 4415.03125,
      "loss/preference_sft": 0.27290159463882446,
      "loss/reference_anchor": 0.2985621392726898,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.2441680431365967,
      "rewards/margins": 2.2339510917663574,
      "rewards/rejected": 1.0102171897888184,
      "step": 225
    },
    {
      "drift/chosen_abs": 0.36413469910621643,
      "drift/rejected_abs": 0.13577529788017273,
      "epoch": 0.21975397109757555,
      "grad_norm": 9600.0,
      "learning_rate": 2.3227055978978545e-07,
      "logits/chosen": -2.0139591693878174,
      "logits/rejected": -2.053098678588867,
      "logps/chosen": -0.2517145872116089,
      "logps/rejected": -0.25124287605285645,
      "loss": 4415.037109375,
      "loss/core": 4415.02294921875,
      "loss/preference_sft": 0.2517145872116089,
      "loss/reference_anchor": 0.6684554815292358,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.639683246612549,
      "rewards/margins": 2.2833621501922607,
      "rewards/rejected": 1.3563209772109985,
      "step": 230
    },
    {
      "drift/chosen_abs": 0.2075643092393875,
      "drift/rejected_abs": 0.12083862721920013,
      "epoch": 0.2245312313388272,
      "grad_norm": 616.0,
      "learning_rate": 2.3100601201955321e-07,
      "logits/chosen": -2.329841136932373,
      "logits/rejected": -2.312140464782715,
      "logps/chosen": -0.25918227434158325,
      "logps/rejected": -0.2679867744445801,
      "loss": 4433.1220703125,
      "loss/core": 4433.1142578125,
      "loss/preference_sft": 0.25918227434158325,
      "loss/reference_anchor": 0.35554802417755127,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.0753586292266846,
      "rewards/margins": 0.868099570274353,
      "rewards/rejected": 1.207258939743042,
      "step": 235
    },
    {
      "drift/chosen_abs": 0.3381517231464386,
      "drift/rejected_abs": 0.19172975420951843,
      "epoch": 0.22930849158007882,
      "grad_norm": 1488.0,
      "learning_rate": 2.2970159972352864e-07,
      "logits/chosen": -2.123835325241089,
      "logits/rejected": -2.1424546241760254,
      "logps/chosen": -0.2800813317298889,
      "logps/rejected": -0.2956727147102356,
      "loss": 4425.13037109375,
      "loss/core": 4425.1181640625,
      "loss/preference_sft": 0.2800813317298889,
      "loss/reference_anchor": 0.6074963808059692,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.381516695022583,
      "rewards/margins": 1.4686692953109741,
      "rewards/rejected": 1.9128472805023193,
      "step": 240
    },
    {
      "drift/chosen_abs": 0.3294285535812378,
      "drift/rejected_abs": 0.16067305207252502,
      "epoch": 0.23408575182133046,
      "grad_norm": 5056.0,
      "learning_rate": 2.2835781343782966e-07,
      "logits/chosen": -1.8609027862548828,
      "logits/rejected": -1.8787086009979248,
      "logps/chosen": -0.2937183976173401,
      "logps/rejected": -0.2992231547832489,
      "loss": 4422.87890625,
      "loss/core": 4422.8642578125,
      "loss/preference_sft": 0.2937183976173401,
      "loss/reference_anchor": 0.6773186922073364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.292163848876953,
      "rewards/margins": 1.6896095275878906,
      "rewards/rejected": 1.6025537252426147,
      "step": 245
    },
    {
      "drift/chosen_abs": 0.15980662405490875,
      "drift/rejected_abs": 0.10799809545278549,
      "epoch": 0.23886301206258212,
      "grad_norm": 418.0,
      "learning_rate": 2.2697515850552152e-07,
      "logits/chosen": -2.347092866897583,
      "logits/rejected": -2.329366683959961,
      "logps/chosen": -0.288370281457901,
      "logps/rejected": -0.29249584674835205,
      "loss": 4437.6669921875,
      "loss/core": 4437.66357421875,
      "loss/preference_sft": 0.288370281457901,
      "loss/reference_anchor": 0.1452062875032425,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5972728729248047,
      "rewards/margins": 0.5199095606803894,
      "rewards/rejected": 1.07736337184906,
      "step": 250
    },
    {
      "drift/chosen_abs": 0.1499588042497635,
      "drift/rejected_abs": 0.0562698058784008,
      "epoch": 0.24364027230383375,
      "grad_norm": 376.0,
      "learning_rate": 2.2555415488657775e-07,
      "logits/chosen": -2.120940685272217,
      "logits/rejected": -2.184211254119873,
      "logps/chosen": -0.30225870013237,
      "logps/rejected": -0.3060735762119293,
      "loss": 4431.9951171875,
      "loss/core": 4431.9931640625,
      "loss/preference_sft": 0.30225870013237,
      "loss/reference_anchor": 0.07390942424535751,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.499276876449585,
      "rewards/margins": 0.9399444460868835,
      "rewards/rejected": 0.5593323707580566,
      "step": 255
    },
    {
      "drift/chosen_abs": 0.23949389159679413,
      "drift/rejected_abs": 0.10860034078359604,
      "epoch": 0.2484175325450854,
      "grad_norm": 482.0,
      "learning_rate": 2.240953369623447e-07,
      "logits/chosen": -2.2790729999542236,
      "logits/rejected": -2.3324427604675293,
      "logps/chosen": -0.27358928322792053,
      "logps/rejected": -0.2760041356086731,
      "loss": 4427.1962890625,
      "loss/core": 4427.19140625,
      "loss/preference_sft": 0.27358928322792053,
      "loss/reference_anchor": 0.19914691150188446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3933863639831543,
      "rewards/margins": 1.307618260383606,
      "rewards/rejected": 1.0857679843902588,
      "step": 260
    },
    {
      "drift/chosen_abs": 0.14223968982696533,
      "drift/rejected_abs": 0.07098479568958282,
      "epoch": 0.25319479278633705,
      "grad_norm": 624.0,
      "learning_rate": 2.225992533345824e-07,
      "logits/chosen": -2.369666337966919,
      "logits/rejected": -2.405613422393799,
      "logps/chosen": -0.28374093770980835,
      "logps/rejected": -0.2866232693195343,
      "loss": 4434.9873046875,
      "loss/core": 4434.9853515625,
      "loss/preference_sft": 0.28374093770980835,
      "loss/reference_anchor": 0.07014183700084686,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.4223629236221313,
      "rewards/margins": 0.7129005193710327,
      "rewards/rejected": 0.7094623446464539,
      "step": 265
    },
    {
      "drift/chosen_abs": 0.18329541385173798,
      "drift/rejected_abs": 0.06777513027191162,
      "epoch": 0.2579720530275887,
      "grad_norm": 1004.0,
      "learning_rate": 2.210664666191579e-07,
      "logits/chosen": -2.1638669967651367,
      "logits/rejected": -2.2433056831359863,
      "logps/chosen": -0.3162863850593567,
      "logps/rejected": -0.2930854558944702,
      "loss": 4429.1513671875,
      "loss/core": 4429.1494140625,
      "loss/preference_sft": 0.3162863850593567,
      "loss/reference_anchor": 0.09944357722997665,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8323523998260498,
      "rewards/margins": 1.1556055545806885,
      "rewards/rejected": 0.6767469644546509,
      "step": 270
    },
    {
      "drift/chosen_abs": 0.24756188690662384,
      "drift/rejected_abs": 0.1202814131975174,
      "epoch": 0.2627493132688403,
      "grad_norm": 1320.0,
      "learning_rate": 2.1949755323446848e-07,
      "logits/chosen": -2.2548816204071045,
      "logits/rejected": -2.2354800701141357,
      "logps/chosen": -0.2556923031806946,
      "logps/rejected": -0.2606434226036072,
      "loss": 4427.748046875,
      "loss/core": 4427.7431640625,
      "loss/preference_sft": 0.2556923031806946,
      "loss/reference_anchor": 0.2529018521308899,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.473615884780884,
      "rewards/margins": 1.273666501045227,
      "rewards/rejected": 1.1999495029449463,
      "step": 275
    },
    {
      "drift/chosen_abs": 0.2426796853542328,
      "drift/rejected_abs": 0.13973277807235718,
      "epoch": 0.26752657351009196,
      "grad_norm": 2256.0,
      "learning_rate": 2.1789310318467426e-07,
      "logits/chosen": -1.8090507984161377,
      "logits/rejected": -1.8918311595916748,
      "logps/chosen": -0.3173496425151825,
      "logps/rejected": -0.32060688734054565,
      "loss": 4430.7841796875,
      "loss/core": 4430.7802734375,
      "loss/preference_sft": 0.3173496425151825,
      "loss/reference_anchor": 0.15054082870483398,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.4233853816986084,
      "rewards/margins": 1.029157280921936,
      "rewards/rejected": 1.3942279815673828,
      "step": 280
    },
    {
      "drift/chosen_abs": 0.28747934103012085,
      "drift/rejected_abs": 0.16268548369407654,
      "epoch": 0.2723038337513436,
      "grad_norm": 2704.0,
      "learning_rate": 2.1625371983782182e-07,
      "logits/chosen": -2.0867972373962402,
      "logits/rejected": -2.0485246181488037,
      "logps/chosen": -0.2663497030735016,
      "logps/rejected": -0.27806463837623596,
      "loss": 4427.39501953125,
      "loss/core": 4427.3876953125,
      "loss/preference_sft": 0.2663497030735016,
      "loss/reference_anchor": 0.32496556639671326,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.874793291091919,
      "rewards/margins": 1.3126866817474365,
      "rewards/rejected": 1.562106728553772,
      "step": 285
    },
    {
      "drift/chosen_abs": 0.2037903517484665,
      "drift/rejected_abs": 0.09013260900974274,
      "epoch": 0.27708109399259523,
      "grad_norm": 1144.0,
      "learning_rate": 2.14580019698942e-07,
      "logits/chosen": -2.1753249168395996,
      "logits/rejected": -2.2431302070617676,
      "logps/chosen": -0.294014573097229,
      "logps/rejected": -0.292147696018219,
      "loss": 4429.5263671875,
      "loss/core": 4429.5224609375,
      "loss/preference_sft": 0.294014573097229,
      "loss/reference_anchor": 0.18461060523986816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.0374693870544434,
      "rewards/margins": 1.1413319110870361,
      "rewards/rejected": 0.8961375951766968,
      "step": 290
    },
    {
      "drift/chosen_abs": 0.2267070710659027,
      "drift/rejected_abs": 0.16824612021446228,
      "epoch": 0.28185835423384686,
      "grad_norm": 14912.0,
      "learning_rate": 2.1287263217820773e-07,
      "logits/chosen": -2.1485249996185303,
      "logits/rejected": -2.1281566619873047,
      "logps/chosen": -0.27404850721359253,
      "logps/rejected": -0.2835575342178345,
      "loss": 4437.10986328125,
      "loss/core": 4437.1025390625,
      "loss/preference_sft": 0.27404850721359253,
      "loss/reference_anchor": 0.32569584250450134,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.265812397003174,
      "rewards/margins": 0.5857616662979126,
      "rewards/rejected": 1.6800508499145508,
      "step": 295
    },
    {
      "drift/chosen_abs": 0.27692022919654846,
      "drift/rejected_abs": 0.1342446506023407,
      "epoch": 0.28663561447509855,
      "grad_norm": 1496.0,
      "learning_rate": 2.111321993542385e-07,
      "logits/chosen": -1.9965057373046875,
      "logits/rejected": -1.9842058420181274,
      "logps/chosen": -0.2693387567996979,
      "logps/rejected": -0.28916555643081665,
      "loss": 4425.4814453125,
      "loss/core": 4425.4775390625,
      "loss/preference_sft": 0.2693387567996979,
      "loss/reference_anchor": 0.1915903389453888,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.7670462131500244,
      "rewards/margins": 1.4375035762786865,
      "rewards/rejected": 1.3295425176620483,
      "step": 300
    },
    {
      "drift/chosen_abs": 0.18260391056537628,
      "drift/rejected_abs": 0.13512252271175385,
      "epoch": 0.2914128747163502,
      "grad_norm": 4416.0,
      "learning_rate": 2.0935937573264096e-07,
      "logits/chosen": -2.097672700881958,
      "logits/rejected": -2.110541820526123,
      "logps/chosen": -0.28904464840888977,
      "logps/rejected": -0.29786160588264465,
      "loss": 4438.23974609375,
      "loss/core": 4438.23681640625,
      "loss/preference_sft": 0.28904464840888977,
      "loss/reference_anchor": 0.13433417677879333,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8258289098739624,
      "rewards/margins": 0.4759580194950104,
      "rewards/rejected": 1.3498706817626953,
      "step": 305
    },
    {
      "drift/chosen_abs": 0.20982956886291504,
      "drift/rejected_abs": 0.137477308511734,
      "epoch": 0.2961901349576018,
      "grad_norm": 3392.0,
      "learning_rate": 2.0755482799987596e-07,
      "logits/chosen": -2.025709629058838,
      "logits/rejected": -1.9913642406463623,
      "logps/chosen": -0.2986164689064026,
      "logps/rejected": -0.3017011284828186,
      "loss": 4434.96875,
      "loss/core": 4434.96337890625,
      "loss/preference_sft": 0.2986164689064026,
      "loss/reference_anchor": 0.26511436700820923,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.0975375175476074,
      "rewards/margins": 0.7258044481277466,
      "rewards/rejected": 1.3717331886291504,
      "step": 310
    },
    {
      "drift/chosen_abs": 0.15806427597999573,
      "drift/rejected_abs": 0.09864726662635803,
      "epoch": 0.30096739519885346,
      "grad_norm": 640.0,
      "learning_rate": 2.0571923477254526e-07,
      "logits/chosen": -2.2739768028259277,
      "logits/rejected": -2.2487998008728027,
      "logps/chosen": -0.2909921109676361,
      "logps/rejected": -0.2905241847038269,
      "loss": 4436.4794921875,
      "loss/core": 4436.4765625,
      "loss/preference_sft": 0.2909921109676361,
      "loss/reference_anchor": 0.10613216459751129,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.5796961784362793,
      "rewards/margins": 0.600797176361084,
      "rewards/rejected": 0.9788990020751953,
      "step": 315
    },
    {
      "drift/chosen_abs": 0.28660792112350464,
      "drift/rejected_abs": 0.19245363771915436,
      "epoch": 0.3057446554401051,
      "grad_norm": 668.0,
      "learning_rate": 2.038532863421914e-07,
      "logits/chosen": -2.1396350860595703,
      "logits/rejected": -2.0602240562438965,
      "logps/chosen": -0.28930574655532837,
      "logps/rejected": -0.2744773030281067,
      "loss": 4432.1318359375,
      "loss/core": 4432.1240234375,
      "loss/preference_sft": 0.28930574655532837,
      "loss/reference_anchor": 0.3356773257255554,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.8660788536071777,
      "rewards/margins": 0.9429539442062378,
      "rewards/rejected": 1.9231246709823608,
      "step": 320
    },
    {
      "drift/chosen_abs": 0.15086838603019714,
      "drift/rejected_abs": 0.1520775854587555,
      "epoch": 0.31052191568135673,
      "grad_norm": 856.0,
      "learning_rate": 2.0195768441570726e-07,
      "logits/chosen": -1.9964319467544556,
      "logits/rejected": -2.05317759513855,
      "logps/chosen": -0.2916659116744995,
      "logps/rejected": -0.2865615487098694,
      "loss": 4445.12451171875,
      "loss/core": 4445.11767578125,
      "loss/preference_sft": 0.2916659116744995,
      "loss/reference_anchor": 0.3259647488594055,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5083897113800049,
      "rewards/margins": -0.008684945292770863,
      "rewards/rejected": 1.5170745849609375,
      "step": 325
    },
    {
      "drift/chosen_abs": 0.259127140045166,
      "drift/rejected_abs": 0.11504222452640533,
      "epoch": 0.31529917592260837,
      "grad_norm": 1272.0,
      "learning_rate": 2.0003314185145307e-07,
      "logits/chosen": -2.14955735206604,
      "logits/rejected": -2.0943520069122314,
      "logps/chosen": -0.28039321303367615,
      "logps/rejected": -0.283574640750885,
      "loss": 4425.5029296875,
      "loss/core": 4425.498046875,
      "loss/preference_sft": 0.28039321303367615,
      "loss/reference_anchor": 0.229940265417099,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.5912716388702393,
      "rewards/margins": 1.4409186840057373,
      "rewards/rejected": 1.150353193283081,
      "step": 330
    },
    {
      "drift/chosen_abs": 0.27030646800994873,
      "drift/rejected_abs": 0.14940232038497925,
      "epoch": 0.32007643616386,
      "grad_norm": 5376.0,
      "learning_rate": 1.9808038239117913e-07,
      "logits/chosen": -1.9976972341537476,
      "logits/rejected": -1.965847373008728,
      "logps/chosen": -0.28784728050231934,
      "logps/rejected": -0.2873719334602356,
      "loss": 4427.10205078125,
      "loss/core": 4427.0966796875,
      "loss/preference_sft": 0.28784728050231934,
      "loss/reference_anchor": 0.2397988736629486,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.7028987407684326,
      "rewards/margins": 1.3128286600112915,
      "rewards/rejected": 1.3900703191757202,
      "step": 335
    },
    {
      "drift/chosen_abs": 0.16661614179611206,
      "drift/rejected_abs": 0.08331886678934097,
      "epoch": 0.3248536964051117,
      "grad_norm": 1120.0,
      "learning_rate": 1.9610014038785646e-07,
      "logits/chosen": -2.2249255180358887,
      "logits/rejected": -2.211129665374756,
      "logps/chosen": -0.2958478629589081,
      "logps/rejected": -0.3064306080341339,
      "loss": 4433.41796875,
      "loss/core": 4433.41552734375,
      "loss/preference_sft": 0.2958478629589081,
      "loss/reference_anchor": 0.08543743938207626,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.664698600769043,
      "rewards/margins": 0.83290034532547,
      "rewards/rejected": 0.8317981958389282,
      "step": 340
    },
    {
      "drift/chosen_abs": 0.28869912028312683,
      "drift/rejected_abs": 0.23126927018165588,
      "epoch": 0.3296309566463633,
      "grad_norm": 1304.0,
      "learning_rate": 1.9409316052951657e-07,
      "logits/chosen": -1.9891431331634521,
      "logits/rejected": -2.0764071941375732,
      "logps/chosen": -0.2763379216194153,
      "logps/rejected": -0.2923387885093689,
      "loss": 4436.98974609375,
      "loss/core": 4436.9755859375,
      "loss/preference_sft": 0.2763379216194153,
      "loss/reference_anchor": 0.6921347975730896,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.886991500854492,
      "rewards/margins": 0.5755015015602112,
      "rewards/rejected": 2.311490058898926,
      "step": 345
    },
    {
      "drift/chosen_abs": 0.37554502487182617,
      "drift/rejected_abs": 0.12323877960443497,
      "epoch": 0.33440821688761496,
      "grad_norm": 2496.0,
      "learning_rate": 1.920601975592047e-07,
      "logits/chosen": -2.176884174346924,
      "logits/rejected": -2.1506075859069824,
      "logps/chosen": -0.2848382294178009,
      "logps/rejected": -0.29111620783805847,
      "loss": 4411.89306640625,
      "loss/core": 4411.876953125,
      "loss/preference_sft": 0.2848382294178009,
      "loss/reference_anchor": 0.7674547433853149,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.7542431354522705,
      "rewards/margins": 2.534789562225342,
      "rewards/rejected": 1.2194538116455078,
      "step": 350
    },
    {
      "drift/chosen_abs": 0.1966264843940735,
      "drift/rejected_abs": 0.07787960767745972,
      "epoch": 0.3391854771288666,
      "grad_norm": 470.0,
      "learning_rate": 1.900020159911519e-07,
      "logits/chosen": -2.094768524169922,
      "logits/rejected": -2.162729263305664,
      "logps/chosen": -0.3112557828426361,
      "logps/rejected": -0.31276172399520874,
      "loss": 4428.56884765625,
      "loss/core": 4428.56494140625,
      "loss/preference_sft": 0.3112557828426361,
      "loss/reference_anchor": 0.14194537699222565,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9648243188858032,
      "rewards/margins": 1.2004649639129639,
      "rewards/rejected": 0.7643591165542603,
      "step": 355
    },
    {
      "drift/chosen_abs": 0.17585548758506775,
      "drift/rejected_abs": 0.1803542524576187,
      "epoch": 0.34396273737011823,
      "grad_norm": 2096.0,
      "learning_rate": 1.879193898232725e-07,
      "logits/chosen": -2.1437807083129883,
      "logits/rejected": -2.1195027828216553,
      "logps/chosen": -0.31370845437049866,
      "logps/rejected": -0.3198932111263275,
      "loss": 4445.20849609375,
      "loss/core": 4445.2041015625,
      "loss/preference_sft": 0.31370845437049866,
      "loss/reference_anchor": 0.19741208851337433,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.757554292678833,
      "rewards/margins": -0.0393296480178833,
      "rewards/rejected": 1.7968838214874268,
      "step": 360
    },
    {
      "drift/chosen_abs": 0.3701009154319763,
      "drift/rejected_abs": 0.08817116916179657,
      "epoch": 0.34873999761136987,
      "grad_norm": 2512.0,
      "learning_rate": 1.8581310224609496e-07,
      "logits/chosen": -2.1306817531585693,
      "logits/rejected": -2.202624797821045,
      "logps/chosen": -0.2909499704837799,
      "logps/rejected": -0.2839373052120209,
      "loss": 4408.3310546875,
      "loss/core": 4408.3134765625,
      "loss/preference_sft": 0.2909499704837799,
      "loss/reference_anchor": 0.8197543025016785,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.6982064247131348,
      "rewards/margins": 2.817063808441162,
      "rewards/rejected": 0.8811424970626831,
      "step": 365
    },
    {
      "drift/chosen_abs": 0.2949160039424896,
      "drift/rejected_abs": 0.12354662269353867,
      "epoch": 0.3535172578526215,
      "grad_norm": 1032.0,
      "learning_rate": 1.8368394534823635e-07,
      "logits/chosen": -2.03263258934021,
      "logits/rejected": -2.0422098636627197,
      "logps/chosen": -0.268283873796463,
      "logps/rejected": -0.2804756760597229,
      "loss": 4421.8681640625,
      "loss/core": 4421.8623046875,
      "loss/preference_sft": 0.268283873796463,
      "loss/reference_anchor": 0.2536497116088867,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.947744846343994,
      "rewards/margins": 1.7133744955062866,
      "rewards/rejected": 1.234370231628418,
      "step": 370
    },
    {
      "drift/chosen_abs": 0.49125900864601135,
      "drift/rejected_abs": 0.23263196647167206,
      "epoch": 0.35829451809387314,
      "grad_norm": 1624.0,
      "learning_rate": 1.8153271981852968e-07,
      "logits/chosen": -2.010446310043335,
      "logits/rejected": -1.984309196472168,
      "logps/chosen": -0.30579593777656555,
      "logps/rejected": -0.29049938917160034,
      "loss": 4411.03564453125,
      "loss/core": 4411.00390625,
      "loss/preference_sft": 0.30579593777656555,
      "loss/reference_anchor": 1.5279619693756104,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.912415504455566,
      "rewards/margins": 2.5893051624298096,
      "rewards/rejected": 2.323111057281494,
      "step": 375
    },
    {
      "drift/chosen_abs": 0.2286815345287323,
      "drift/rejected_abs": 0.1312122792005539,
      "epoch": 0.36307177833512483,
      "grad_norm": 3968.0,
      "learning_rate": 1.7936023464491812e-07,
      "logits/chosen": -2.0856316089630127,
      "logits/rejected": -2.1967413425445557,
      "logps/chosen": -0.27951109409332275,
      "logps/rejected": -0.2807064950466156,
      "loss": 4431.7783203125,
      "loss/core": 4431.7705078125,
      "loss/preference_sft": 0.27951109409332275,
      "loss/reference_anchor": 0.35531511902809143,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.286098003387451,
      "rewards/margins": 0.9744806289672852,
      "rewards/rejected": 1.311617136001587,
      "step": 380
    },
    {
      "drift/chosen_abs": 0.1671498715877533,
      "drift/rejected_abs": 0.1056649461388588,
      "epoch": 0.36784903857637646,
      "grad_norm": 1776.0,
      "learning_rate": 1.7716730681022723e-07,
      "logits/chosen": -2.121617078781128,
      "logits/rejected": -2.081577777862549,
      "logps/chosen": -0.31088462471961975,
      "logps/rejected": -0.28644299507141113,
      "loss": 4436.3212890625,
      "loss/core": 4436.31787109375,
      "loss/preference_sft": 0.31088462471961975,
      "loss/reference_anchor": 0.11547583341598511,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6688594818115234,
      "rewards/margins": 0.6149738430976868,
      "rewards/rejected": 1.0538856983184814,
      "step": 385
    },
    {
      "drift/chosen_abs": 0.22545960545539856,
      "drift/rejected_abs": 0.12498001754283905,
      "epoch": 0.3726262988176281,
      "grad_norm": 580.0,
      "learning_rate": 1.7495476098493152e-07,
      "logits/chosen": -2.199582099914551,
      "logits/rejected": -2.216892957687378,
      "logps/chosen": -0.27879446744918823,
      "logps/rejected": -0.28513437509536743,
      "loss": 4430.99560546875,
      "loss/core": 4430.9912109375,
      "loss/preference_sft": 0.27879446744918823,
      "loss/reference_anchor": 0.1869107335805893,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.2526803016662598,
      "rewards/margins": 1.0196499824523926,
      "rewards/rejected": 1.2330303192138672,
      "step": 390
    },
    {
      "drift/chosen_abs": 0.3004913330078125,
      "drift/rejected_abs": 0.16865012049674988,
      "epoch": 0.37740355905887973,
      "grad_norm": 704.0,
      "learning_rate": 1.7272342921702937e-07,
      "logits/chosen": -2.0919270515441895,
      "logits/rejected": -2.11561918258667,
      "logps/chosen": -0.30641791224479675,
      "logps/rejected": -0.303880900144577,
      "loss": 4427.22021484375,
      "loss/core": 4427.20849609375,
      "loss/preference_sft": 0.30641791224479675,
      "loss/reference_anchor": 0.5793026089668274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.0033681392669678,
      "rewards/margins": 1.31758713722229,
      "rewards/rejected": 1.6857811212539673,
      "step": 395
    },
    {
      "drift/chosen_abs": 0.2755824029445648,
      "drift/rejected_abs": 0.2149803191423416,
      "epoch": 0.38218081930013137,
      "grad_norm": 432.0,
      "learning_rate": 1.7047415061914393e-07,
      "logits/chosen": -2.0630311965942383,
      "logits/rejected": -2.2178916931152344,
      "logps/chosen": -0.27443453669548035,
      "logps/rejected": -0.2907470166683197,
      "loss": 4437.09130859375,
      "loss/core": 4437.0791015625,
      "loss/preference_sft": 0.27443453669548035,
      "loss/reference_anchor": 0.5900745987892151,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.7551357746124268,
      "rewards/margins": 0.606482982635498,
      "rewards/rejected": 2.1486525535583496,
      "step": 400
    },
    {
      "drift/chosen_abs": 0.26228779554367065,
      "drift/rejected_abs": 0.12948082387447357,
      "epoch": 0.386958079541383,
      "grad_norm": 4704.0,
      "learning_rate": 1.6820777105296707e-07,
      "logits/chosen": -2.1135382652282715,
      "logits/rejected": -2.212303876876831,
      "logps/chosen": -0.3208262026309967,
      "logps/rejected": -0.2868503928184509,
      "loss": 4427.3076171875,
      "loss/core": 4427.296875,
      "loss/preference_sft": 0.3208262026309967,
      "loss/reference_anchor": 0.47984394431114197,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.6186184883117676,
      "rewards/margins": 1.3284586668014526,
      "rewards/rejected": 1.2901595830917358,
      "step": 405
    },
    {
      "drift/chosen_abs": 0.2780519723892212,
      "drift/rejected_abs": 0.13061827421188354,
      "epoch": 0.39173533978263464,
      "grad_norm": 540.0,
      "learning_rate": 1.6592514281116553e-07,
      "logits/chosen": -2.0848355293273926,
      "logits/rejected": -2.101752519607544,
      "logps/chosen": -0.26733091473579407,
      "logps/rejected": -0.2704174518585205,
      "loss": 4425.0966796875,
      "loss/core": 4425.09033203125,
      "loss/preference_sft": 0.26733091473579407,
      "loss/reference_anchor": 0.29540133476257324,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.780134439468384,
      "rewards/margins": 1.4746670722961426,
      "rewards/rejected": 1.3054672479629517,
      "step": 410
    },
    {
      "drift/chosen_abs": 0.23144730925559998,
      "drift/rejected_abs": 0.07132745534181595,
      "epoch": 0.3965126000238863,
      "grad_norm": 7296.0,
      "learning_rate": 1.636271242968684e-07,
      "logits/chosen": -2.039712429046631,
      "logits/rejected": -2.0561647415161133,
      "logps/chosen": -0.2998116612434387,
      "logps/rejected": -0.28820380568504333,
      "loss": 4422.5302734375,
      "loss/core": 4422.5244140625,
      "loss/preference_sft": 0.2998116612434387,
      "loss/reference_anchor": 0.21635302901268005,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.3144731521606445,
      "rewards/margins": 1.6700124740600586,
      "rewards/rejected": 0.644460916519165,
      "step": 415
    },
    {
      "drift/chosen_abs": 0.32540231943130493,
      "drift/rejected_abs": 0.18714219331741333,
      "epoch": 0.40128986026513797,
      "grad_norm": 900.0,
      "learning_rate": 1.6131457970085684e-07,
      "logits/chosen": -1.9318962097167969,
      "logits/rejected": -1.9521112442016602,
      "logps/chosen": -0.284771591424942,
      "logps/rejected": -0.2870418131351471,
      "loss": 4426.5439453125,
      "loss/core": 4426.5341796875,
      "loss/preference_sft": 0.284771591424942,
      "loss/reference_anchor": 0.46427035331726074,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2529983520507812,
      "rewards/margins": 1.3839852809906006,
      "rewards/rejected": 1.8690131902694702,
      "step": 420
    },
    {
      "drift/chosen_abs": 0.2929674983024597,
      "drift/rejected_abs": 0.1980661153793335,
      "epoch": 0.4060671205063896,
      "grad_norm": 1904.0,
      "learning_rate": 1.5898837867657727e-07,
      "logits/chosen": -1.9516912698745728,
      "logits/rejected": -1.9994312524795532,
      "logps/chosen": -0.28913792967796326,
      "logps/rejected": -0.27645057439804077,
      "loss": 4432.4873046875,
      "loss/core": 4432.4775390625,
      "loss/preference_sft": 0.28913792967796326,
      "loss/reference_anchor": 0.4840005040168762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.9288175106048584,
      "rewards/margins": 0.9494360685348511,
      "rewards/rejected": 1.9793813228607178,
      "step": 425
    },
    {
      "drift/chosen_abs": 0.25099247694015503,
      "drift/rejected_abs": 0.0831591859459877,
      "epoch": 0.41084438074764124,
      "grad_norm": 668.0,
      "learning_rate": 1.5664939601310023e-07,
      "logits/chosen": -2.182654857635498,
      "logits/rejected": -2.2524609565734863,
      "logps/chosen": -0.3082042336463928,
      "logps/rejected": -0.30552542209625244,
      "loss": 4422.7587890625,
      "loss/core": 4422.75,
      "loss/preference_sft": 0.3082042336463928,
      "loss/reference_anchor": 0.41477394104003906,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.5072789192199707,
      "rewards/margins": 1.676422357559204,
      "rewards/rejected": 0.830856442451477,
      "step": 430
    },
    {
      "drift/chosen_abs": 0.24665072560310364,
      "drift/rejected_abs": 0.12369408458471298,
      "epoch": 0.41562164098889287,
      "grad_norm": 1112.0,
      "learning_rate": 1.5429851130614807e-07,
      "logits/chosen": -2.1079397201538086,
      "logits/rejected": -2.0382118225097656,
      "logps/chosen": -0.28440576791763306,
      "logps/rejected": -0.26352208852767944,
      "loss": 4428.2412109375,
      "loss/core": 4428.23681640625,
      "loss/preference_sft": 0.28440576791763306,
      "loss/reference_anchor": 0.22342030704021454,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4649205207824707,
      "rewards/margins": 1.229503870010376,
      "rewards/rejected": 1.2354166507720947,
      "step": 435
    },
    {
      "drift/chosen_abs": 0.22085194289684296,
      "drift/rejected_abs": 0.08876828849315643,
      "epoch": 0.4203989012301445,
      "grad_norm": 4608.0,
      "learning_rate": 1.51936608627315e-07,
      "logits/chosen": -2.0002598762512207,
      "logits/rejected": -2.0259335041046143,
      "logps/chosen": -0.3116574287414551,
      "logps/rejected": -0.31580719351768494,
      "loss": 4426.97900390625,
      "loss/core": 4426.974609375,
      "loss/preference_sft": 0.3116574287414551,
      "loss/reference_anchor": 0.183195561170578,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.205994129180908,
      "rewards/margins": 1.3205196857452393,
      "rewards/rejected": 0.8854740858078003,
      "step": 440
    },
    {
      "drift/chosen_abs": 0.37103694677352905,
      "drift/rejected_abs": 0.18114325404167175,
      "epoch": 0.42517616147139614,
      "grad_norm": 7104.0,
      "learning_rate": 1.4956457619160375e-07,
      "logits/chosen": -2.050297975540161,
      "logits/rejected": -2.0345191955566406,
      "logps/chosen": -0.27982237935066223,
      "logps/rejected": -0.25793343782424927,
      "loss": 4420.06103515625,
      "loss/core": 4420.046875,
      "loss/preference_sft": 0.27982237935066223,
      "loss/reference_anchor": 0.6739336252212524,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.709260940551758,
      "rewards/margins": 1.9022051095962524,
      "rewards/rejected": 1.8070557117462158,
      "step": 445
    },
    {
      "drift/chosen_abs": 0.19236549735069275,
      "drift/rejected_abs": 0.08619394898414612,
      "epoch": 0.4299534217126478,
      "grad_norm": 768.0,
      "learning_rate": 1.471833060234044e-07,
      "logits/chosen": -2.10903263092041,
      "logits/rejected": -2.172616481781006,
      "logps/chosen": -0.30059614777565,
      "logps/rejected": -0.2986820936203003,
      "loss": 4430.38623046875,
      "loss/core": 4430.3837890625,
      "loss/preference_sft": 0.30059614777565,
      "loss/reference_anchor": 0.12340258061885834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9236551523208618,
      "rewards/margins": 1.0623431205749512,
      "rewards/rejected": 0.8613119125366211,
      "step": 450
    },
    {
      "drift/chosen_abs": 0.21780920028686523,
      "drift/rejected_abs": 0.13248507678508759,
      "epoch": 0.4347306819538994,
      "grad_norm": 1728.0,
      "learning_rate": 1.4479369362104037e-07,
      "logits/chosen": -2.1391212940216064,
      "logits/rejected": -2.252551555633545,
      "logps/chosen": -0.26534712314605713,
      "logps/rejected": -0.27983564138412476,
      "loss": 4433.15234375,
      "loss/core": 4433.1484375,
      "loss/preference_sft": 0.26534712314605713,
      "loss/reference_anchor": 0.1820131540298462,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.1771018505096436,
      "rewards/margins": 0.8640018701553345,
      "rewards/rejected": 1.313099980354309,
      "step": 455
    },
    {
      "drift/chosen_abs": 0.20761752128601074,
      "drift/rejected_abs": 0.09282220900058746,
      "epoch": 0.4395079421951511,
      "grad_norm": 652.0,
      "learning_rate": 1.4239663762000817e-07,
      "logits/chosen": -2.139101505279541,
      "logits/rejected": -2.070023536682129,
      "logps/chosen": -0.2922465205192566,
      "logps/rejected": -0.29120540618896484,
      "loss": 4429.29296875,
      "loss/core": 4429.28857421875,
      "loss/preference_sft": 0.2922465205192566,
      "loss/reference_anchor": 0.17264321446418762,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.0735580921173096,
      "rewards/margins": 1.1463443040847778,
      "rewards/rejected": 0.9272136688232422,
      "step": 460
    },
    {
      "drift/chosen_abs": 0.2610882520675659,
      "drift/rejected_abs": 0.143789604306221,
      "epoch": 0.44428520243640274,
      "grad_norm": 10240.0,
      "learning_rate": 1.3999303945503747e-07,
      "logits/chosen": -2.1653521060943604,
      "logits/rejected": -2.247696876525879,
      "logps/chosen": -0.31048905849456787,
      "logps/rejected": -0.31169334053993225,
      "loss": 4429.36865234375,
      "loss/core": 4429.359375,
      "loss/preference_sft": 0.31048905849456787,
      "loss/reference_anchor": 0.44403213262557983,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.610751152038574,
      "rewards/margins": 1.1738497018814087,
      "rewards/rejected": 1.4369014501571655,
      "step": 465
    },
    {
      "drift/chosen_abs": 0.15377414226531982,
      "drift/rejected_abs": 0.10233066231012344,
      "epoch": 0.4490624626776544,
      "grad_norm": 884.0,
      "learning_rate": 1.3758380302109808e-07,
      "logits/chosen": -2.145686626434326,
      "logits/rejected": -2.1528048515319824,
      "logps/chosen": -0.2958281636238098,
      "logps/rejected": -0.26482534408569336,
      "loss": 4437.5927734375,
      "loss/core": 4437.5908203125,
      "loss/preference_sft": 0.2958281636238098,
      "loss/reference_anchor": 0.08905575424432755,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.5374128818511963,
      "rewards/margins": 0.5175694227218628,
      "rewards/rejected": 1.019843578338623,
      "step": 470
    },
    {
      "drift/chosen_abs": 0.19338826835155487,
      "drift/rejected_abs": 0.05476611852645874,
      "epoch": 0.453839722918906,
      "grad_norm": 520.0,
      "learning_rate": 1.351698343334823e-07,
      "logits/chosen": -2.10408091545105,
      "logits/rejected": -2.1622872352600098,
      "logps/chosen": -0.3007555603981018,
      "logps/rejected": -0.29687556624412537,
      "loss": 4426.18212890625,
      "loss/core": 4426.17724609375,
      "loss/preference_sft": 0.3007555603981018,
      "loss/reference_anchor": 0.22863420844078064,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.9303607940673828,
      "rewards/margins": 1.399163842201233,
      "rewards/rejected": 0.5311970114707947,
      "step": 475
    },
    {
      "drift/chosen_abs": 0.24555771052837372,
      "drift/rejected_abs": 0.133039191365242,
      "epoch": 0.45861698316015764,
      "grad_norm": 5504.0,
      "learning_rate": 1.3275204118708942e-07,
      "logits/chosen": -2.0921359062194824,
      "logits/rejected": -1.9713423252105713,
      "logps/chosen": -0.301283597946167,
      "logps/rejected": -0.31479784846305847,
      "loss": 4430.07861328125,
      "loss/core": 4430.0654296875,
      "loss/preference_sft": 0.301283597946167,
      "loss/reference_anchor": 0.6136071681976318,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4555182456970215,
      "rewards/margins": 1.125307559967041,
      "rewards/rejected": 1.33021080493927,
      "step": 480
    },
    {
      "drift/chosen_abs": 0.20899991691112518,
      "drift/rejected_abs": 0.10918338596820831,
      "epoch": 0.4633942434014093,
      "grad_norm": 2944.0,
      "learning_rate": 1.3033133281504132e-07,
      "logits/chosen": -1.9529765844345093,
      "logits/rejected": -1.9704357385635376,
      "logps/chosen": -0.29765480756759644,
      "logps/rejected": -0.296121209859848,
      "loss": 4431.419921875,
      "loss/core": 4431.4150390625,
      "loss/preference_sft": 0.29765480756759644,
      "loss/reference_anchor": 0.23028752207756042,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.089999198913574,
      "rewards/margins": 1.000409483909607,
      "rewards/rejected": 1.0895897150039673,
      "step": 485
    },
    {
      "drift/chosen_abs": 0.23929134011268616,
      "drift/rejected_abs": 0.11025607585906982,
      "epoch": 0.4681715036426609,
      "grad_norm": 980.0,
      "learning_rate": 1.2790861954675702e-07,
      "logits/chosen": -2.0607128143310547,
      "logits/rejected": -2.0974857807159424,
      "logps/chosen": -0.3084251582622528,
      "logps/rejected": -0.30655646324157715,
      "loss": 4427.3017578125,
      "loss/core": 4427.2978515625,
      "loss/preference_sft": 0.3084251582622528,
      "loss/reference_anchor": 0.14670784771442413,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3920226097106934,
      "rewards/margins": 1.2922639846801758,
      "rewards/rejected": 1.0997586250305176,
      "step": 490
    },
    {
      "drift/chosen_abs": 0.19296832382678986,
      "drift/rejected_abs": 0.09996967762708664,
      "epoch": 0.47294876388391255,
      "grad_norm": 1216.0,
      "learning_rate": 1.2548481246561504e-07,
      "logits/chosen": -2.0633440017700195,
      "logits/rejected": -2.0872926712036133,
      "logps/chosen": -0.2833230495452881,
      "logps/rejected": -0.28482872247695923,
      "loss": 4432.07861328125,
      "loss/core": 4432.0751953125,
      "loss/preference_sft": 0.2833230495452881,
      "loss/reference_anchor": 0.14006902277469635,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.929682970046997,
      "rewards/margins": 0.9321301579475403,
      "rewards/rejected": 0.9975531697273254,
      "step": 495
    },
    {
      "drift/chosen_abs": 0.36558204889297485,
      "drift/rejected_abs": 0.1663735955953598,
      "epoch": 0.47772602412516424,
      "grad_norm": 5088.0,
      "learning_rate": 1.230608230663321e-07,
      "logits/chosen": -2.070899486541748,
      "logits/rejected": -2.0423977375030518,
      "logps/chosen": -0.2811706066131592,
      "logps/rejected": -0.28334346413612366,
      "loss": 4418.2373046875,
      "loss/core": 4418.22265625,
      "loss/preference_sft": 0.2811706066131592,
      "loss/reference_anchor": 0.7044563889503479,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.652653455734253,
      "rewards/margins": 1.9970003366470337,
      "rewards/rejected": 1.6556532382965088,
      "step": 500
    },
    {
      "drift/chosen_abs": 0.34318313002586365,
      "drift/rejected_abs": 0.13852709531784058,
      "epoch": 0.4825032843664159,
      "grad_norm": 816.0,
      "learning_rate": 1.206375629121874e-07,
      "logits/chosen": -2.0183844566345215,
      "logits/rejected": -2.008913278579712,
      "logps/chosen": -0.3097713589668274,
      "logps/rejected": -0.31553733348846436,
      "loss": 4417.1552734375,
      "loss/core": 4417.146484375,
      "loss/preference_sft": 0.3097713589668274,
      "loss/reference_anchor": 0.4379523694515228,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4314303398132324,
      "rewards/margins": 2.0832791328430176,
      "rewards/rejected": 1.348151683807373,
      "step": 505
    },
    {
      "drift/chosen_abs": 0.2345653474330902,
      "drift/rejected_abs": 0.12095403671264648,
      "epoch": 0.4872805446076675,
      "grad_norm": 1072.0,
      "learning_rate": 1.1821594329222079e-07,
      "logits/chosen": -2.2127575874328613,
      "logits/rejected": -2.1563878059387207,
      "logps/chosen": -0.2902802526950836,
      "logps/rejected": -0.28386133909225464,
      "loss": 4429.57666015625,
      "loss/core": 4429.5654296875,
      "loss/preference_sft": 0.2902802526950836,
      "loss/reference_anchor": 0.5237426161766052,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.3436882495880127,
      "rewards/margins": 1.1352647542953491,
      "rewards/rejected": 1.208423376083374,
      "step": 510
    },
    {
      "drift/chosen_abs": 0.18333016335964203,
      "drift/rejected_abs": 0.1809997260570526,
      "epoch": 0.49205780484891914,
      "grad_norm": 2576.0,
      "learning_rate": 1.157968748785344e-07,
      "logits/chosen": -1.914223074913025,
      "logits/rejected": -1.981766939163208,
      "logps/chosen": -0.2812362015247345,
      "logps/rejected": -0.27950748801231384,
      "loss": 4444.3720703125,
      "loss/core": 4444.36767578125,
      "loss/preference_sft": 0.2812362015247345,
      "loss/reference_anchor": 0.19075706601142883,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.8306372165679932,
      "rewards/margins": 0.022197848185896873,
      "rewards/rejected": 1.8084392547607422,
      "step": 515
    },
    {
      "drift/chosen_abs": 0.3202487826347351,
      "drift/rejected_abs": 0.13263753056526184,
      "epoch": 0.4968350650901708,
      "grad_norm": 2064.0,
      "learning_rate": 1.1338126738382597e-07,
      "logits/chosen": -1.9274823665618896,
      "logits/rejected": -1.931096076965332,
      "logps/chosen": -0.2886351943016052,
      "logps/rejected": -0.2917082607746124,
      "loss": 4419.962890625,
      "loss/core": 4419.9501953125,
      "loss/preference_sft": 0.2886351943016052,
      "loss/reference_anchor": 0.6030197143554688,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.202364444732666,
      "rewards/margins": 1.8788830041885376,
      "rewards/rejected": 1.3234814405441284,
      "step": 520
    },
    {
      "drift/chosen_abs": 0.21438315510749817,
      "drift/rejected_abs": 0.07398609071969986,
      "epoch": 0.5016123253314224,
      "grad_norm": 604.0,
      "learning_rate": 1.1097002921928316e-07,
      "logits/chosen": -2.086273670196533,
      "logits/rejected": -2.169018268585205,
      "logps/chosen": -0.2757102847099304,
      "logps/rejected": -0.27384692430496216,
      "loss": 4425.99462890625,
      "loss/core": 4425.9892578125,
      "loss/preference_sft": 0.2757102847099304,
      "loss/reference_anchor": 0.19271975755691528,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.142575740814209,
      "rewards/margins": 1.4045182466506958,
      "rewards/rejected": 0.7380577921867371,
      "step": 525
    },
    {
      "drift/chosen_abs": 0.16398575901985168,
      "drift/rejected_abs": 0.08183001726865768,
      "epoch": 0.5063895855726741,
      "grad_norm": 556.0,
      "learning_rate": 1.0856406715296717e-07,
      "logits/chosen": -2.1869404315948486,
      "logits/rejected": -2.284536361694336,
      "logps/chosen": -0.2831934988498688,
      "logps/rejected": -0.2927997410297394,
      "loss": 4433.50634765625,
      "loss/core": 4433.50341796875,
      "loss/preference_sft": 0.2831934988498688,
      "loss/reference_anchor": 0.07691037654876709,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6397440433502197,
      "rewards/margins": 0.822593092918396,
      "rewards/rejected": 0.817150890827179,
      "step": 530
    },
    {
      "drift/chosen_abs": 0.3339373767375946,
      "drift/rejected_abs": 0.12080500274896622,
      "epoch": 0.5111668458139257,
      "grad_norm": 564.0,
      "learning_rate": 1.061642859688146e-07,
      "logits/chosen": -2.11379075050354,
      "logits/rejected": -2.042389392852783,
      "logps/chosen": -0.2667238712310791,
      "logps/rejected": -0.27604439854621887,
      "loss": 4416.22265625,
      "loss/core": 4416.216796875,
      "loss/preference_sft": 0.2667238712310791,
      "loss/reference_anchor": 0.2798369526863098,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.339317798614502,
      "rewards/margins": 2.1484532356262207,
      "rewards/rejected": 1.1908643245697021,
      "step": 535
    },
    {
      "drift/chosen_abs": 0.2929610311985016,
      "drift/rejected_abs": 0.08645624667406082,
      "epoch": 0.5159441060551774,
      "grad_norm": 2688.0,
      "learning_rate": 1.0377158812638491e-07,
      "logits/chosen": -2.1425912380218506,
      "logits/rejected": -2.234184980392456,
      "logps/chosen": -0.29007601737976074,
      "logps/rejected": -0.280531108379364,
      "loss": 4416.90625,
      "loss/core": 4416.8984375,
      "loss/preference_sft": 0.29007601737976074,
      "loss/reference_anchor": 0.3554505705833435,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9293251037597656,
      "rewards/margins": 2.1080482006073,
      "rewards/rejected": 0.821276843547821,
      "step": 540
    },
    {
      "drift/chosen_abs": 0.2901972234249115,
      "drift/rejected_abs": 0.14684346318244934,
      "epoch": 0.520721366296429,
      "grad_norm": 3792.0,
      "learning_rate": 1.0138687342148249e-07,
      "logits/chosen": -2.062620162963867,
      "logits/rejected": -2.1082162857055664,
      "logps/chosen": -0.30190232396125793,
      "logps/rejected": -0.32236361503601074,
      "loss": 4426.2060546875,
      "loss/core": 4426.193359375,
      "loss/preference_sft": 0.30190232396125793,
      "loss/reference_anchor": 0.6151673197746277,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.90141224861145,
      "rewards/margins": 1.4344381093978882,
      "rewards/rejected": 1.4669743776321411,
      "step": 545
    },
    {
      "drift/chosen_abs": 0.24899080395698547,
      "drift/rejected_abs": 0.20708198845386505,
      "epoch": 0.5254986265376806,
      "grad_norm": 592.0,
      "learning_rate": 9.90110386477801e-08,
      "logits/chosen": -2.1318652629852295,
      "logits/rejected": -2.2062549591064453,
      "logps/chosen": -0.25858795642852783,
      "logps/rejected": -0.253892183303833,
      "loss": 4438.9169921875,
      "loss/core": 4438.9072265625,
      "loss/preference_sft": 0.25858795642852783,
      "loss/reference_anchor": 0.47126907110214233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.489907741546631,
      "rewards/margins": 0.4215244650840759,
      "rewards/rejected": 2.0683834552764893,
      "step": 550
    },
    {
      "drift/chosen_abs": 0.21308758854866028,
      "drift/rejected_abs": 0.15445470809936523,
      "epoch": 0.5302758867789323,
      "grad_norm": 1928.0,
      "learning_rate": 9.664497725957164e-08,
      "logits/chosen": -2.1934444904327393,
      "logits/rejected": -2.2147269248962402,
      "logps/chosen": -0.2912050783634186,
      "logps/rejected": -0.2784259617328644,
      "loss": 4436.732421875,
      "loss/core": 4436.72607421875,
      "loss/preference_sft": 0.2912050783634186,
      "loss/reference_anchor": 0.23212206363677979,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.127945899963379,
      "rewards/margins": 0.5856080651283264,
      "rewards/rejected": 1.5423381328582764,
      "step": 555
    },
    {
      "drift/chosen_abs": 0.2658155560493469,
      "drift/rejected_abs": 0.19533106684684753,
      "epoch": 0.5350531470201839,
      "grad_norm": 7616.0,
      "learning_rate": 9.428957903578045e-08,
      "logits/chosen": -1.9657647609710693,
      "logits/rejected": -1.980133295059204,
      "logps/chosen": -0.2851273715496063,
      "logps/rejected": -0.28451281785964966,
      "loss": 4435.39208984375,
      "loss/core": 4435.38037109375,
      "loss/preference_sft": 0.2851273715496063,
      "loss/reference_anchor": 0.557953417301178,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.6540966033935547,
      "rewards/margins": 0.7040950655937195,
      "rewards/rejected": 1.9500014781951904,
      "step": 560
    },
    {
      "drift/chosen_abs": 0.34854352474212646,
      "drift/rejected_abs": 0.19613364338874817,
      "epoch": 0.5398304072614356,
      "grad_norm": 696.0,
      "learning_rate": 9.194572974534976e-08,
      "logits/chosen": -2.024336338043213,
      "logits/rejected": -2.093052864074707,
      "logps/chosen": -0.27176398038864136,
      "logps/rejected": -0.26332786679267883,
      "loss": 4424.57861328125,
      "loss/core": 4424.56201171875,
      "loss/preference_sft": 0.27176398038864136,
      "loss/reference_anchor": 0.8150361776351929,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4846279621124268,
      "rewards/margins": 1.5311681032180786,
      "rewards/rejected": 1.9534600973129272,
      "step": 565
    },
    {
      "drift/chosen_abs": 0.227237731218338,
      "drift/rejected_abs": 0.16712255775928497,
      "epoch": 0.5446076675026872,
      "grad_norm": 4416.0,
      "learning_rate": 8.96143108141412e-08,
      "logits/chosen": -1.9618107080459595,
      "logits/rejected": -1.954664945602417,
      "logps/chosen": -0.2768324613571167,
      "logps/rejected": -0.27417004108428955,
      "loss": 4436.580078125,
      "loss/core": 4436.5751953125,
      "loss/preference_sft": 0.2768324613571167,
      "loss/reference_anchor": 0.21785183250904083,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.271784543991089,
      "rewards/margins": 0.6015557050704956,
      "rewards/rejected": 1.670229196548462,
      "step": 570
    },
    {
      "drift/chosen_abs": 0.22515693306922913,
      "drift/rejected_abs": 0.08775409311056137,
      "epoch": 0.5493849277439389,
      "grad_norm": 360.0,
      "learning_rate": 8.72961989934668e-08,
      "logits/chosen": -2.231755495071411,
      "logits/rejected": -2.245164632797241,
      "logps/chosen": -0.2810257077217102,
      "logps/rejected": -0.28133219480514526,
      "loss": 4426.43994140625,
      "loss/core": 4426.4345703125,
      "loss/preference_sft": 0.2810257077217102,
      "loss/reference_anchor": 0.2272057831287384,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.2512524127960205,
      "rewards/margins": 1.3741371631622314,
      "rewards/rejected": 0.8771151304244995,
      "step": 575
    },
    {
      "drift/chosen_abs": 0.14085161685943604,
      "drift/rejected_abs": 0.1031990498304367,
      "epoch": 0.5541621879851905,
      "grad_norm": 720.0,
      "learning_rate": 8.499226603037854e-08,
      "logits/chosen": -2.2430803775787354,
      "logits/rejected": -2.2786197662353516,
      "logps/chosen": -0.2992137372493744,
      "logps/rejected": -0.29944515228271484,
      "loss": 4439.31689453125,
      "loss/core": 4439.31396484375,
      "loss/preference_sft": 0.2992137372493744,
      "loss/reference_anchor": 0.10587255656719208,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.407078742980957,
      "rewards/margins": 0.39435142278671265,
      "rewards/rejected": 1.0127273797988892,
      "step": 580
    },
    {
      "drift/chosen_abs": 0.22692131996154785,
      "drift/rejected_abs": 0.10220160335302353,
      "epoch": 0.5589394482264421,
      "grad_norm": 608.0,
      "learning_rate": 8.270337833983987e-08,
      "logits/chosen": -2.0355045795440674,
      "logits/rejected": -2.1195898056030273,
      "logps/chosen": -0.30346351861953735,
      "logps/rejected": -0.2966625392436981,
      "loss": 4427.97900390625,
      "loss/core": 4427.97216796875,
      "loss/preference_sft": 0.30346351861953735,
      "loss/reference_anchor": 0.29685312509536743,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.2688138484954834,
      "rewards/margins": 1.247382402420044,
      "rewards/rejected": 1.0214314460754395,
      "step": 585
    },
    {
      "drift/chosen_abs": 0.21154217422008514,
      "drift/rejected_abs": 0.13458245992660522,
      "epoch": 0.5637167084676937,
      "grad_norm": 1672.0,
      "learning_rate": 8.04303966789025e-08,
      "logits/chosen": -2.1316933631896973,
      "logits/rejected": -2.150923252105713,
      "logps/chosen": -0.27440136671066284,
      "logps/rejected": -0.27802544832229614,
      "loss": 4434.2373046875,
      "loss/core": 4434.2314453125,
      "loss/preference_sft": 0.27440136671066284,
      "loss/reference_anchor": 0.25449132919311523,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.114384889602661,
      "rewards/margins": 0.7690680623054504,
      "rewards/rejected": 1.3453166484832764,
      "step": 590
    },
    {
      "drift/chosen_abs": 0.21570686995983124,
      "drift/rejected_abs": 0.11859677731990814,
      "epoch": 0.5684939687089454,
      "grad_norm": 1072.0,
      "learning_rate": 7.817417582301098e-08,
      "logits/chosen": -1.8743267059326172,
      "logits/rejected": -1.8942207098007202,
      "logps/chosen": -0.284064382314682,
      "logps/rejected": -0.28866657614707947,
      "loss": 4431.5322265625,
      "loss/core": 4431.529296875,
      "loss/preference_sft": 0.284064382314682,
      "loss/reference_anchor": 0.1300676316022873,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1570687294006348,
      "rewards/margins": 0.9721587896347046,
      "rewards/rejected": 1.1849100589752197,
      "step": 595
    },
    {
      "drift/chosen_abs": 0.19431225955486298,
      "drift/rejected_abs": 0.1243596076965332,
      "epoch": 0.5732712289501971,
      "grad_norm": 1800.0,
      "learning_rate": 7.59355642445566e-08,
      "logits/chosen": -2.2408175468444824,
      "logits/rejected": -2.15144419670105,
      "logps/chosen": -0.26184308528900146,
      "logps/rejected": -0.2716737985610962,
      "loss": 4434.80078125,
      "loss/core": 4434.79736328125,
      "loss/preference_sft": 0.26184308528900146,
      "loss/reference_anchor": 0.16086336970329285,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9430396556854248,
      "rewards/margins": 0.7340011596679688,
      "rewards/rejected": 1.2090386152267456,
      "step": 600
    },
    {
      "drift/chosen_abs": 0.21506313979625702,
      "drift/rejected_abs": 0.09374111890792847,
      "epoch": 0.5780484891914487,
      "grad_norm": 7328.0,
      "learning_rate": 7.37154037938015e-08,
      "logits/chosen": -2.073078155517578,
      "logits/rejected": -2.174865245819092,
      "logps/chosen": -0.3253878951072693,
      "logps/rejected": -0.33154183626174927,
      "loss": 4428.4287109375,
      "loss/core": 4428.4248046875,
      "loss/preference_sft": 0.3253878951072693,
      "loss/reference_anchor": 0.17495615780353546,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.148521900177002,
      "rewards/margins": 1.2160072326660156,
      "rewards/rejected": 0.9325143694877625,
      "step": 605
    },
    {
      "drift/chosen_abs": 0.21458843350410461,
      "drift/rejected_abs": 0.11676479876041412,
      "epoch": 0.5828257494327004,
      "grad_norm": 1976.0,
      "learning_rate": 7.151452938229325e-08,
      "logits/chosen": -2.0549139976501465,
      "logits/rejected": -2.0558762550354004,
      "logps/chosen": -0.2830585837364197,
      "logps/rejected": -0.29241445660591125,
      "loss": 4431.64208984375,
      "loss/core": 4431.63623046875,
      "loss/preference_sft": 0.2830585837364197,
      "loss/reference_anchor": 0.25421541929244995,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.145282745361328,
      "rewards/margins": 0.9789865612983704,
      "rewards/rejected": 1.166296362876892,
      "step": 610
    },
    {
      "drift/chosen_abs": 0.3049691319465637,
      "drift/rejected_abs": 0.16369685530662537,
      "epoch": 0.587603009673952,
      "grad_norm": 11968.0,
      "learning_rate": 6.933376866888883e-08,
      "logits/chosen": -2.038628339767456,
      "logits/rejected": -2.1392931938171387,
      "logps/chosen": -0.2770799994468689,
      "logps/rejected": -0.3140313923358917,
      "loss": 4426.2294921875,
      "loss/core": 4426.2177734375,
      "loss/preference_sft": 0.2770799994468689,
      "loss/reference_anchor": 0.5493339896202087,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.0482850074768066,
      "rewards/margins": 1.4278751611709595,
      "rewards/rejected": 1.6204099655151367,
      "step": 615
    },
    {
      "drift/chosen_abs": 0.26089203357696533,
      "drift/rejected_abs": 0.1435510367155075,
      "epoch": 0.5923802699152036,
      "grad_norm": 15232.0,
      "learning_rate": 6.717394174850605e-08,
      "logits/chosen": -2.150646924972534,
      "logits/rejected": -2.166919708251953,
      "logps/chosen": -0.2822549045085907,
      "logps/rejected": -0.29912179708480835,
      "loss": 4429.2724609375,
      "loss/core": 4429.2646484375,
      "loss/preference_sft": 0.2822549045085907,
      "loss/reference_anchor": 0.33313748240470886,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6065964698791504,
      "rewards/margins": 1.1729382276535034,
      "rewards/rejected": 1.433658480644226,
      "step": 620
    },
    {
      "drift/chosen_abs": 0.19429583847522736,
      "drift/rejected_abs": 0.1452254056930542,
      "epoch": 0.5971575301564552,
      "grad_norm": 1168.0,
      "learning_rate": 6.503586084371926e-08,
      "logits/chosen": -2.172908306121826,
      "logits/rejected": -2.1675751209259033,
      "logps/chosen": -0.2829391360282898,
      "logps/rejected": -0.2997308373451233,
      "loss": 4438.06396484375,
      "loss/core": 4438.0595703125,
      "loss/preference_sft": 0.2829391360282898,
      "loss/reference_anchor": 0.17451608180999756,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9426425695419312,
      "rewards/margins": 0.4928255081176758,
      "rewards/rejected": 1.4498172998428345,
      "step": 625
    },
    {
      "drift/chosen_abs": 0.2902236878871918,
      "drift/rejected_abs": 0.13140039145946503,
      "epoch": 0.6019347903977069,
      "grad_norm": 2480.0,
      "learning_rate": 6.29203299993155e-08,
      "logits/chosen": -2.1176817417144775,
      "logits/rejected": -2.0981597900390625,
      "logps/chosen": -0.30078643560409546,
      "logps/rejected": -0.3083663880825043,
      "loss": 4419.7548828125,
      "loss/core": 4419.74609375,
      "loss/preference_sft": 0.30078643560409546,
      "loss/reference_anchor": 0.43593829870224,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.901242256164551,
      "rewards/margins": 1.8964433670043945,
      "rewards/rejected": 1.0047987699508667,
      "step": 630
    },
    {
      "drift/chosen_abs": 0.17258086800575256,
      "drift/rejected_abs": 0.1321617066860199,
      "epoch": 0.6067120506389586,
      "grad_norm": 3680.0,
      "learning_rate": 6.082814477992656e-08,
      "logits/chosen": -2.1916544437408447,
      "logits/rejected": -2.11995530128479,
      "logps/chosen": -0.30291134119033813,
      "logps/rejected": -0.306274950504303,
      "loss": 4439.31982421875,
      "loss/core": 4439.31494140625,
      "loss/preference_sft": 0.30291134119033813,
      "loss/reference_anchor": 0.23412831127643585,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.723780632019043,
      "rewards/margins": 0.4066292643547058,
      "rewards/rejected": 1.317151427268982,
      "step": 635
    },
    {
      "drift/chosen_abs": 0.1660202294588089,
      "drift/rejected_abs": 0.06376107037067413,
      "epoch": 0.6114893108802102,
      "grad_norm": 1200.0,
      "learning_rate": 5.87600919708494e-08,
      "logits/chosen": -2.0012378692626953,
      "logits/rejected": -2.0457763671875,
      "logps/chosen": -0.28748735785484314,
      "logps/rejected": -0.2896059453487396,
      "loss": 4430.99169921875,
      "loss/core": 4430.98876953125,
      "loss/preference_sft": 0.28748735785484314,
      "loss/reference_anchor": 0.1316143423318863,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6601464748382568,
      "rewards/margins": 1.0247886180877686,
      "rewards/rejected": 0.6353579163551331,
      "step": 640
    },
    {
      "drift/chosen_abs": 0.203060582280159,
      "drift/rejected_abs": 0.18572744727134705,
      "epoch": 0.6162665711214619,
      "grad_norm": 5792.0,
      "learning_rate": 5.671694928216829e-08,
      "logits/chosen": -2.225250244140625,
      "logits/rejected": -2.2663300037384033,
      "logps/chosen": -0.28407204151153564,
      "logps/rejected": -0.27350106835365295,
      "loss": 4442.513671875,
      "loss/core": 4442.50830078125,
      "loss/preference_sft": 0.28407204151153564,
      "loss/reference_anchor": 0.25616979598999023,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0303478240966797,
      "rewards/margins": 0.17582125961780548,
      "rewards/rejected": 1.8545265197753906,
      "step": 645
    },
    {
      "drift/chosen_abs": 0.3154805302619934,
      "drift/rejected_abs": 0.2100723534822464,
      "epoch": 0.6210438313627135,
      "grad_norm": 6400.0,
      "learning_rate": 5.469948505629e-08,
      "logits/chosen": -2.017364263534546,
      "logits/rejected": -2.079219341278076,
      "logps/chosen": -0.28993043303489685,
      "logps/rejected": -0.28738412261009216,
      "loss": 4430.4951171875,
      "loss/core": 4430.47998046875,
      "loss/preference_sft": 0.28993043303489685,
      "loss/reference_anchor": 0.7488113641738892,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.1542510986328125,
      "rewards/margins": 1.0541503429412842,
      "rewards/rejected": 2.1001007556915283,
      "step": 650
    },
    {
      "drift/chosen_abs": 0.1768004447221756,
      "drift/rejected_abs": 0.12505300343036652,
      "epoch": 0.6258210916039652,
      "grad_norm": 8960.0,
      "learning_rate": 5.270845797900168e-08,
      "logits/chosen": -2.223820209503174,
      "logits/rejected": -2.153284788131714,
      "logps/chosen": -0.28208497166633606,
      "logps/rejected": -0.29163843393325806,
      "loss": 4437.68603515625,
      "loss/core": 4437.68017578125,
      "loss/preference_sft": 0.28208497166633606,
      "loss/reference_anchor": 0.26417121291160583,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.765069603919983,
      "rewards/margins": 0.5174070596694946,
      "rewards/rejected": 1.2476627826690674,
      "step": 655
    },
    {
      "drift/chosen_abs": 0.18167448043823242,
      "drift/rejected_abs": 0.14043021202087402,
      "epoch": 0.6305983518452167,
      "grad_norm": 524.0,
      "learning_rate": 5.0744616794160104e-08,
      "logits/chosen": -2.0674781799316406,
      "logits/rejected": -2.1492257118225098,
      "logps/chosen": -0.27626532316207886,
      "logps/rejected": -0.28123727440834045,
      "loss": 4439.0361328125,
      "loss/core": 4439.03369140625,
      "loss/preference_sft": 0.27626532316207886,
      "loss/reference_anchor": 0.11728350073099136,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8160436153411865,
      "rewards/margins": 0.41351088881492615,
      "rewards/rejected": 1.402532935142517,
      "step": 660
    },
    {
      "drift/chosen_abs": 0.22600972652435303,
      "drift/rejected_abs": 0.1264456957578659,
      "epoch": 0.6353756120864684,
      "grad_norm": 796.0,
      "learning_rate": 4.880870002211963e-08,
      "logits/chosen": -2.2491302490234375,
      "logits/rejected": -2.25697660446167,
      "logps/chosen": -0.27793681621551514,
      "logps/rejected": -0.25179094076156616,
      "loss": 4431.15283203125,
      "loss/core": 4431.146484375,
      "loss/preference_sft": 0.27793681621551514,
      "loss/reference_anchor": 0.2595617473125458,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.259474754333496,
      "rewards/margins": 1.0019110441207886,
      "rewards/rejected": 1.2575637102127075,
      "step": 665
    },
    {
      "drift/chosen_abs": 0.13364611566066742,
      "drift/rejected_abs": 0.09645704180002213,
      "epoch": 0.64015287232772,
      "grad_norm": 540.0,
      "learning_rate": 4.6901435682004996e-08,
      "logits/chosen": -2.2417209148406982,
      "logits/rejected": -2.2763659954071045,
      "logps/chosen": -0.29147717356681824,
      "logps/rejected": -0.2934878468513489,
      "loss": 4439.51708984375,
      "loss/core": 4439.515625,
      "loss/preference_sft": 0.29147717356681824,
      "loss/reference_anchor": 0.06112232059240341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3360573053359985,
      "rewards/margins": 0.3732479214668274,
      "rewards/rejected": 0.9628093838691711,
      "step": 670
    },
    {
      "drift/chosen_abs": 0.1628817468881607,
      "drift/rejected_abs": 0.13016179203987122,
      "epoch": 0.6449301325689717,
      "grad_norm": 2032.0,
      "learning_rate": 4.502354101793314e-08,
      "logits/chosen": -2.3644509315490723,
      "logits/rejected": -2.2733235359191895,
      "logps/chosen": -0.27977296710014343,
      "logps/rejected": -0.29269108176231384,
      "loss": 4440.22900390625,
      "loss/core": 4440.2255859375,
      "loss/preference_sft": 0.27977296710014343,
      "loss/reference_anchor": 0.17271290719509125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6281238794326782,
      "rewards/margins": 0.33540040254592896,
      "rewards/rejected": 1.2927234172821045,
      "step": 675
    },
    {
      "drift/chosen_abs": 0.3136964738368988,
      "drift/rejected_abs": 0.12950031459331512,
      "epoch": 0.6497073928102234,
      "grad_norm": 1472.0,
      "learning_rate": 4.3175722229287034e-08,
      "logits/chosen": -2.1214194297790527,
      "logits/rejected": -2.114104747772217,
      "logps/chosen": -0.27199044823646545,
      "logps/rejected": -0.27481597661972046,
      "loss": 4419.1142578125,
      "loss/core": 4419.107421875,
      "loss/preference_sft": 0.27199044823646545,
      "loss/reference_anchor": 0.31056779623031616,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.136345624923706,
      "rewards/margins": 1.9250768423080444,
      "rewards/rejected": 1.2112689018249512,
      "step": 680
    },
    {
      "drift/chosen_abs": 0.23225578665733337,
      "drift/rejected_abs": 0.23389002680778503,
      "epoch": 0.654484653051475,
      "grad_norm": 2960.0,
      "learning_rate": 4.135867420514276e-08,
      "logits/chosen": -2.2221508026123047,
      "logits/rejected": -2.2248616218566895,
      "logps/chosen": -0.2901459038257599,
      "logps/rejected": -0.26247456669807434,
      "loss": 4445.1943359375,
      "loss/core": 4445.1845703125,
      "loss/preference_sft": 0.2901459038257599,
      "loss/reference_anchor": 0.46210724115371704,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.3225579261779785,
      "rewards/margins": -0.016050493344664574,
      "rewards/rejected": 2.3386082649230957,
      "step": 685
    },
    {
      "drift/chosen_abs": 0.2951485216617584,
      "drift/rejected_abs": 0.2140798568725586,
      "epoch": 0.6592619132927267,
      "grad_norm": 9984.0,
      "learning_rate": 3.957308026295035e-08,
      "logits/chosen": -2.093303680419922,
      "logits/rejected": -2.1656064987182617,
      "logps/chosen": -0.31711024045944214,
      "logps/rejected": -0.32263869047164917,
      "loss": 4435.220703125,
      "loss/core": 4435.2021484375,
      "loss/preference_sft": 0.31711024045944214,
      "loss/reference_anchor": 0.8856789469718933,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9496922492980957,
      "rewards/margins": 0.8108631372451782,
      "rewards/rejected": 2.138829469680786,
      "step": 690
    },
    {
      "drift/chosen_abs": 0.23619715869426727,
      "drift/rejected_abs": 0.11902375519275665,
      "epoch": 0.6640391735339782,
      "grad_norm": 1752.0,
      "learning_rate": 3.7819611891566084e-08,
      "logits/chosen": -1.9933494329452515,
      "logits/rejected": -1.9954357147216797,
      "logps/chosen": -0.268182635307312,
      "logps/rejected": -0.28130608797073364,
      "loss": 4428.77734375,
      "loss/core": 4428.77392578125,
      "loss/preference_sft": 0.268182635307312,
      "loss/reference_anchor": 0.13460581004619598,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3605830669403076,
      "rewards/margins": 1.1809606552124023,
      "rewards/rejected": 1.1796222925186157,
      "step": 695
    },
    {
      "drift/chosen_abs": 0.3273255228996277,
      "drift/rejected_abs": 0.16635169088840485,
      "epoch": 0.6688164337752299,
      "grad_norm": 3360.0,
      "learning_rate": 3.609892849873286e-08,
      "logits/chosen": -2.2435264587402344,
      "logits/rejected": -2.1759636402130127,
      "logps/chosen": -0.2879250645637512,
      "logps/rejected": -0.2684784531593323,
      "loss": 4424.0283203125,
      "loss/core": 4424.0146484375,
      "loss/preference_sft": 0.2879250645637512,
      "loss/reference_anchor": 0.6359601616859436,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2727675437927246,
      "rewards/margins": 1.61063551902771,
      "rewards/rejected": 1.6621326208114624,
      "step": 700
    },
    {
      "drift/chosen_abs": 0.13324177265167236,
      "drift/rejected_abs": 0.052085865288972855,
      "epoch": 0.6735936940164815,
      "grad_norm": 450.0,
      "learning_rate": 3.4411677163103894e-08,
      "logits/chosen": -2.1316683292388916,
      "logits/rejected": -2.1392605304718018,
      "logps/chosen": -0.3257368206977844,
      "logps/rejected": -0.31321820616722107,
      "loss": 4433.72265625,
      "loss/core": 4433.72119140625,
      "loss/preference_sft": 0.3257368206977844,
      "loss/reference_anchor": 0.06571613252162933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3298277854919434,
      "rewards/margins": 0.8104680180549622,
      "rewards/rejected": 0.5193597078323364,
      "step": 705
    },
    {
      "drift/chosen_abs": 0.24692943692207336,
      "drift/rejected_abs": 0.08578932285308838,
      "epoch": 0.6783709542577332,
      "grad_norm": 1712.0,
      "learning_rate": 3.2758492390902945e-08,
      "logits/chosen": -1.9040149450302124,
      "logits/rejected": -1.964320421218872,
      "logps/chosen": -0.278387188911438,
      "logps/rejected": -0.28676044940948486,
      "loss": 4422.9931640625,
      "loss/core": 4422.98876953125,
      "loss/preference_sft": 0.278387188911438,
      "loss/reference_anchor": 0.17939075827598572,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.4657251834869385,
      "rewards/margins": 1.624982237815857,
      "rewards/rejected": 0.8407428860664368,
      "step": 710
    },
    {
      "drift/chosen_abs": 0.2357044667005539,
      "drift/rejected_abs": 0.09313996881246567,
      "epoch": 0.6831482144989849,
      "grad_norm": 884.0,
      "learning_rate": 3.11399958773126e-08,
      "logits/chosen": -2.0695366859436035,
      "logits/rejected": -2.1179842948913574,
      "logps/chosen": -0.28330761194229126,
      "logps/rejected": -0.2819029688835144,
      "loss": 4425.52294921875,
      "loss/core": 4425.52001953125,
      "loss/preference_sft": 0.28330761194229126,
      "loss/reference_anchor": 0.11744773387908936,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3560948371887207,
      "rewards/margins": 1.4276307821273804,
      "rewards/rejected": 0.9284639358520508,
      "step": 715
    },
    {
      "drift/chosen_abs": 0.2549819350242615,
      "drift/rejected_abs": 0.11347774416208267,
      "epoch": 0.6879254747402365,
      "grad_norm": 704.0,
      "learning_rate": 2.9556796272679972e-08,
      "logits/chosen": -2.1076555252075195,
      "logits/rejected": -2.143085479736328,
      "logps/chosen": -0.28872913122177124,
      "logps/rejected": -0.3133319020271301,
      "loss": 4425.2392578125,
      "loss/core": 4425.23291015625,
      "loss/preference_sft": 0.28872913122177124,
      "loss/reference_anchor": 0.2932053208351135,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.549466133117676,
      "rewards/margins": 1.4717979431152344,
      "rewards/rejected": 1.0776679515838623,
      "step": 720
    },
    {
      "drift/chosen_abs": 0.11858288198709488,
      "drift/rejected_abs": 0.1146119013428688,
      "epoch": 0.6927027349814882,
      "grad_norm": 396.0,
      "learning_rate": 2.8009488953628215e-08,
      "logits/chosen": -2.0915627479553223,
      "logits/rejected": -2.0317933559417725,
      "logps/chosen": -0.2838587760925293,
      "logps/rejected": -0.2749176621437073,
      "loss": 4444.0947265625,
      "loss/core": 4444.091796875,
      "loss/preference_sft": 0.2838587760925293,
      "loss/reference_anchor": 0.11706352233886719,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.1828110218048096,
      "rewards/margins": 0.038224995136260986,
      "rewards/rejected": 1.1445858478546143,
      "step": 725
    },
    {
      "drift/chosen_abs": 0.28038161993026733,
      "drift/rejected_abs": 0.10303550958633423,
      "epoch": 0.6974799952227397,
      "grad_norm": 2032.0,
      "learning_rate": 2.649865579915976e-08,
      "logits/chosen": -2.084571123123169,
      "logits/rejected": -2.0591697692871094,
      "logps/chosen": -0.3031243681907654,
      "logps/rejected": -0.3205743432044983,
      "loss": 4420.45458984375,
      "loss/core": 4420.4453125,
      "loss/preference_sft": 0.3031243681907654,
      "loss/reference_anchor": 0.4167904257774353,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.803816556930542,
      "rewards/margins": 1.8526108264923096,
      "rewards/rejected": 0.9512056112289429,
      "step": 730
    },
    {
      "drift/chosen_abs": 0.28037887811660767,
      "drift/rejected_abs": 0.11086313426494598,
      "epoch": 0.7022572554639914,
      "grad_norm": 7264.0,
      "learning_rate": 2.5024864971835507e-08,
      "logits/chosen": -2.197540760040283,
      "logits/rejected": -2.1911709308624268,
      "logps/chosen": -0.2654857039451599,
      "logps/rejected": -0.27312350273132324,
      "loss": 4422.1875,
      "loss/core": 4422.1806640625,
      "loss/preference_sft": 0.2654857039451599,
      "loss/reference_anchor": 0.29703524708747864,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.802237033843994,
      "rewards/margins": 1.697741150856018,
      "rewards/rejected": 1.1044957637786865,
      "step": 735
    },
    {
      "drift/chosen_abs": 0.21863920986652374,
      "drift/rejected_abs": 0.165724977850914,
      "epoch": 0.707034515705243,
      "grad_norm": 3120.0,
      "learning_rate": 2.3588670704111997e-08,
      "logits/chosen": -1.931133508682251,
      "logits/rejected": -1.8537521362304688,
      "logps/chosen": -0.2914375364780426,
      "logps/rejected": -0.3034089207649231,
      "loss": 4437.83544921875,
      "loss/core": 4437.828125,
      "loss/preference_sft": 0.2914375364780426,
      "loss/reference_anchor": 0.32963576912879944,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.18485689163208,
      "rewards/margins": 0.5304257869720459,
      "rewards/rejected": 1.6544309854507446,
      "step": 740
    },
    {
      "drift/chosen_abs": 0.22953541576862335,
      "drift/rejected_abs": 0.12510935962200165,
      "epoch": 0.7118117759464947,
      "grad_norm": 636.0,
      "learning_rate": 2.219061308991721e-08,
      "logits/chosen": -2.2116262912750244,
      "logits/rejected": -2.234771251678467,
      "logps/chosen": -0.2957614064216614,
      "logps/rejected": -0.28059297800064087,
      "loss": 4430.69677734375,
      "loss/core": 4430.68994140625,
      "loss/preference_sft": 0.2957614064216614,
      "loss/reference_anchor": 0.3111433684825897,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.295354127883911,
      "rewards/margins": 1.0461337566375732,
      "rewards/rejected": 1.2492201328277588,
      "step": 745
    },
    {
      "drift/chosen_abs": 0.22382231056690216,
      "drift/rejected_abs": 0.1400088369846344,
      "epoch": 0.7165890361877463,
      "grad_norm": 7872.0,
      "learning_rate": 2.0831217881543557e-08,
      "logits/chosen": -2.2578673362731934,
      "logits/rejected": -2.194847583770752,
      "logps/chosen": -0.27832141518592834,
      "logps/rejected": -0.30063265562057495,
      "loss": 4433.3857421875,
      "loss/core": 4433.3818359375,
      "loss/preference_sft": 0.27832141518592834,
      "loss/reference_anchor": 0.1458483338356018,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.2344486713409424,
      "rewards/margins": 0.8346776962280273,
      "rewards/rejected": 1.3997710943222046,
      "step": 750
    },
    {
      "drift/chosen_abs": 0.3602936863899231,
      "drift/rejected_abs": 0.19181835651397705,
      "epoch": 0.721366296428998,
      "grad_norm": 560.0,
      "learning_rate": 1.951099629193366e-08,
      "logits/chosen": -2.169501781463623,
      "logits/rejected": -2.14741587638855,
      "logps/chosen": -0.2755396068096161,
      "logps/rejected": -0.2857043743133545,
      "loss": 4422.30126953125,
      "loss/core": 4422.2890625,
      "loss/preference_sft": 0.2755396068096161,
      "loss/reference_anchor": 0.6152486801147461,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.602374315261841,
      "rewards/margins": 1.6893268823623657,
      "rewards/rejected": 1.9130470752716064,
      "step": 755
    },
    {
      "drift/chosen_abs": 0.3550851047039032,
      "drift/rejected_abs": 0.20769810676574707,
      "epoch": 0.7261435566702497,
      "grad_norm": 8768.0,
      "learning_rate": 1.823044480243431e-08,
      "logits/chosen": -2.0105481147766113,
      "logits/rejected": -2.006004810333252,
      "logps/chosen": -0.3070371150970459,
      "logps/rejected": -0.3485947549343109,
      "loss": 4425.2255859375,
      "loss/core": 4425.2138671875,
      "loss/preference_sft": 0.3070371150970459,
      "loss/reference_anchor": 0.5617225170135498,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 3.5491435527801514,
      "rewards/margins": 1.4741640090942383,
      "rewards/rejected": 2.074979305267334,
      "step": 760
    },
    {
      "drift/chosen_abs": 0.16385112702846527,
      "drift/rejected_abs": 0.08402819186449051,
      "epoch": 0.7309208169115012,
      "grad_norm": 844.0,
      "learning_rate": 1.699004497608994e-08,
      "logits/chosen": -2.3256075382232666,
      "logits/rejected": -2.332451105117798,
      "logps/chosen": -0.27758467197418213,
      "logps/rejected": -0.26881900429725647,
      "loss": 4433.8310546875,
      "loss/core": 4433.8291015625,
      "loss/preference_sft": 0.27758467197418213,
      "loss/reference_anchor": 0.07551290094852448,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.637289047241211,
      "rewards/margins": 0.7998763918876648,
      "rewards/rejected": 0.8374127149581909,
      "step": 765
    },
    {
      "drift/chosen_abs": 0.2287888526916504,
      "drift/rejected_abs": 0.07531949877738953,
      "epoch": 0.7356980771527529,
      "grad_norm": 11264.0,
      "learning_rate": 1.5790263276546658e-08,
      "logits/chosen": -2.220702648162842,
      "logits/rejected": -2.295063018798828,
      "logps/chosen": -0.286711722612381,
      "logps/rejected": -0.2794603705406189,
      "loss": 4424.37353515625,
      "loss/core": 4424.3681640625,
      "loss/preference_sft": 0.286711722612381,
      "loss/reference_anchor": 0.25186899304389954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.287888526916504,
      "rewards/margins": 1.5352530479431152,
      "rewards/rejected": 0.7526354789733887,
      "step": 770
    },
    {
      "drift/chosen_abs": 0.18583203852176666,
      "drift/rejected_abs": 0.1396988481283188,
      "epoch": 0.7404753373940045,
      "grad_norm": 528.0,
      "learning_rate": 1.4631550892634126e-08,
      "logits/chosen": -2.1028919219970703,
      "logits/rejected": -1.9989187717437744,
      "logps/chosen": -0.2728358209133148,
      "logps/rejected": -0.2678207755088806,
      "loss": 4438.3876953125,
      "loss/core": 4438.38330078125,
      "loss/preference_sft": 0.2728358209133148,
      "loss/reference_anchor": 0.1961844116449356,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.8562612533569336,
      "rewards/margins": 0.4593871533870697,
      "rewards/rejected": 1.3968740701675415,
      "step": 775
    },
    {
      "drift/chosen_abs": 0.2970121502876282,
      "drift/rejected_abs": 0.19716815650463104,
      "epoch": 0.7452525976352562,
      "grad_norm": 29184.0,
      "learning_rate": 1.3514343568692132e-08,
      "logits/chosen": -2.24552845954895,
      "logits/rejected": -2.210707426071167,
      "logps/chosen": -0.29537197947502136,
      "logps/rejected": -0.2902478277683258,
      "loss": 4431.3583984375,
      "loss/core": 4431.33984375,
      "loss/preference_sft": 0.29537197947502136,
      "loss/reference_anchor": 0.8959586024284363,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9684689044952393,
      "rewards/margins": 0.9983173608779907,
      "rewards/rejected": 1.9701515436172485,
      "step": 780
    },
    {
      "drift/chosen_abs": 0.19076788425445557,
      "drift/rejected_abs": 0.07851524651050568,
      "epoch": 0.7500298578765078,
      "grad_norm": 944.0,
      "learning_rate": 1.2439061440704724e-08,
      "logits/chosen": -2.2142930030822754,
      "logits/rejected": -2.226604700088501,
      "logps/chosen": -0.2813388407230377,
      "logps/rejected": -0.281820148229599,
      "loss": 4430.03515625,
      "loss/core": 4430.0283203125,
      "loss/preference_sft": 0.2813388407230377,
      "loss/reference_anchor": 0.3390761911869049,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.9072763919830322,
      "rewards/margins": 1.1225610971450806,
      "rewards/rejected": 0.7847151756286621,
      "step": 785
    },
    {
      "drift/chosen_abs": 0.33144018054008484,
      "drift/rejected_abs": 0.13096961379051208,
      "epoch": 0.7548071181177595,
      "grad_norm": 3488.0,
      "learning_rate": 1.1406108878304468e-08,
      "logits/chosen": -2.039895534515381,
      "logits/rejected": -2.0326807498931885,
      "logps/chosen": -0.2768157422542572,
      "logps/rejected": -0.27031582593917847,
      "loss": 4417.72802734375,
      "loss/core": 4417.72021484375,
      "loss/preference_sft": 0.2768157422542572,
      "loss/reference_anchor": 0.3354681134223938,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 3.3139610290527344,
      "rewards/margins": 2.033888339996338,
      "rewards/rejected": 1.2800720930099487,
      "step": 790
    },
    {
      "drift/chosen_abs": 0.3740524649620056,
      "drift/rejected_abs": 0.09586691856384277,
      "epoch": 0.7595843783590112,
      "grad_norm": 8256.0,
      "learning_rate": 1.0415874332705381e-08,
      "logits/chosen": -2.1573445796966553,
      "logits/rejected": -2.1065096855163574,
      "logps/chosen": -0.2745102345943451,
      "logps/rejected": -0.2804674804210663,
      "loss": 4408.5126953125,
      "loss/core": 4408.4970703125,
      "loss/preference_sft": 0.2745102345943451,
      "loss/reference_anchor": 0.7442028522491455,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.7369015216827393,
      "rewards/margins": 2.7862377166748047,
      "rewards/rejected": 0.9506635665893555,
      "step": 795
    },
    {
      "drift/chosen_abs": 0.2516661584377289,
      "drift/rejected_abs": 0.14492499828338623,
      "epoch": 0.7643616386002627,
      "grad_norm": 1168.0,
      "learning_rate": 9.468730190622484e-09,
      "logits/chosen": -2.473392963409424,
      "logits/rejected": -2.458265781402588,
      "logps/chosen": -0.2965652346611023,
      "logps/rejected": -0.3087556064128876,
      "loss": 4430.37841796875,
      "loss/core": 4430.365234375,
      "loss/preference_sft": 0.2965652346611023,
      "loss/reference_anchor": 0.6160001158714294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.5161986351013184,
      "rewards/margins": 1.0671333074569702,
      "rewards/rejected": 1.4490649700164795,
      "step": 800
    },
    {
      "drift/chosen_abs": 0.09642498195171356,
      "drift/rejected_abs": 0.06358600407838821,
      "epoch": 0.7691388988415144,
      "grad_norm": 772.0,
      "learning_rate": 8.565032634232194e-09,
      "logits/chosen": -2.1253273487091064,
      "logits/rejected": -2.0759212970733643,
      "logps/chosen": -0.3012627065181732,
      "logps/rejected": -0.3006402850151062,
      "loss": 4440.068359375,
      "loss/core": 4440.0673828125,
      "loss/preference_sft": 0.3012627065181732,
      "loss/reference_anchor": 0.021201241761446,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.962838351726532,
      "rewards/margins": 0.3286899924278259,
      "rewards/rejected": 0.634148359298706,
      "step": 805
    },
    {
      "drift/chosen_abs": 0.1828320473432541,
      "drift/rejected_abs": 0.1643926501274109,
      "epoch": 0.773916159082766,
      "grad_norm": 3008.0,
      "learning_rate": 7.70512150722702e-09,
      "logits/chosen": -2.1040894985198975,
      "logits/rejected": -2.042335033416748,
      "logps/chosen": -0.2662808299064636,
      "logps/rejected": -0.26375871896743774,
      "loss": 4442.34619140625,
      "loss/core": 4442.3408203125,
      "loss/preference_sft": 0.2662808299064636,
      "loss/reference_anchor": 0.245298832654953,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8268150091171265,
      "rewards/margins": 0.18445713818073273,
      "rewards/rejected": 1.6423580646514893,
      "step": 810
    },
    {
      "drift/chosen_abs": 0.24483975768089294,
      "drift/rejected_abs": 0.10785166919231415,
      "epoch": 0.7786934193240177,
      "grad_norm": 588.0,
      "learning_rate": 6.8893201870139915e-09,
      "logits/chosen": -2.097686290740967,
      "logits/rejected": -2.2861804962158203,
      "logps/chosen": -0.31777235865592957,
      "logps/rejected": -0.3054574131965637,
      "loss": 4426.5830078125,
      "loss/core": 4426.57666015625,
      "loss/preference_sft": 0.31777235865592957,
      "loss/reference_anchor": 0.284587562084198,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.445316791534424,
      "rewards/margins": 1.3685089349746704,
      "rewards/rejected": 1.0768078565597534,
      "step": 815
    },
    {
      "drift/chosen_abs": 0.26204735040664673,
      "drift/rejected_abs": 0.20677991211414337,
      "epoch": 0.7834706795652693,
      "grad_norm": 2288.0,
      "learning_rate": 6.117935463105808e-09,
      "logits/chosen": -1.823381781578064,
      "logits/rejected": -1.8026411533355713,
      "logps/chosen": -0.2800712585449219,
      "logps/rejected": -0.3262674808502197,
      "loss": 4437.53466796875,
      "loss/core": 4437.5244140625,
      "loss/preference_sft": 0.2800712585449219,
      "loss/reference_anchor": 0.490020215511322,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.620026111602783,
      "rewards/margins": 0.566874086856842,
      "rewards/rejected": 2.053152084350586,
      "step": 820
    },
    {
      "drift/chosen_abs": 0.35397663712501526,
      "drift/rejected_abs": 0.16934973001480103,
      "epoch": 0.788247939806521,
      "grad_norm": 1288.0,
      "learning_rate": 5.391257421749826e-09,
      "logits/chosen": -2.1389248371124268,
      "logits/rejected": -2.1224210262298584,
      "logps/chosen": -0.29372864961624146,
      "logps/rejected": -0.2752332091331482,
      "loss": 4420.16455078125,
      "loss/core": 4420.1484375,
      "loss/preference_sft": 0.29372864961624146,
      "loss/reference_anchor": 0.7682027816772461,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.5389609336853027,
      "rewards/margins": 1.84674072265625,
      "rewards/rejected": 1.6922200918197632,
      "step": 825
    },
    {
      "drift/chosen_abs": 0.28098636865615845,
      "drift/rejected_abs": 0.12216347455978394,
      "epoch": 0.7930252000477725,
      "grad_norm": 7328.0,
      "learning_rate": 4.709559336838462e-09,
      "logits/chosen": -1.9067840576171875,
      "logits/rejected": -1.9582593441009521,
      "logps/chosen": -0.3030111491680145,
      "logps/rejected": -0.3043147027492523,
      "loss": 4423.7431640625,
      "loss/core": 4423.7333984375,
      "loss/preference_sft": 0.3030111491680145,
      "loss/reference_anchor": 0.462859570980072,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.809863567352295,
      "rewards/margins": 1.590742588043213,
      "rewards/rejected": 1.2191212177276611,
      "step": 830
    },
    {
      "drift/chosen_abs": 0.16951501369476318,
      "drift/rejected_abs": 0.10462959110736847,
      "epoch": 0.7978024602890242,
      "grad_norm": 596.0,
      "learning_rate": 4.073097567142025e-09,
      "logits/chosen": -2.2416045665740967,
      "logits/rejected": -2.323671817779541,
      "logps/chosen": -0.29937443137168884,
      "logps/rejected": -0.28850868344306946,
      "loss": 4435.8212890625,
      "loss/core": 4435.818359375,
      "loss/preference_sft": 0.29937443137168884,
      "loss/reference_anchor": 0.1115211695432663,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6946293115615845,
      "rewards/margins": 0.649151086807251,
      "rewards/rejected": 1.0454782247543335,
      "step": 835
    },
    {
      "drift/chosen_abs": 0.20362646877765656,
      "drift/rejected_abs": 0.15938568115234375,
      "epoch": 0.8025797205302759,
      "grad_norm": 1040.0,
      "learning_rate": 3.482111459902695e-09,
      "logits/chosen": -2.0392162799835205,
      "logits/rejected": -2.133927822113037,
      "logps/chosen": -0.2889602780342102,
      "logps/rejected": -0.3033998906612396,
      "loss": 4439.00927734375,
      "loss/core": 4439.0029296875,
      "loss/preference_sft": 0.2889602780342102,
      "loss/reference_anchor": 0.33250418305397034,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.036264657974243,
      "rewards/margins": 0.44385433197021484,
      "rewards/rejected": 1.5924100875854492,
      "step": 840
    },
    {
      "drift/chosen_abs": 0.46066784858703613,
      "drift/rejected_abs": 0.22359898686408997,
      "epoch": 0.8073569807715275,
      "grad_norm": 1544.0,
      "learning_rate": 2.9368232608258797e-09,
      "logits/chosen": -2.173532485961914,
      "logits/rejected": -2.1856465339660645,
      "logps/chosen": -0.2729622721672058,
      "logps/rejected": -0.27251607179641724,
      "loss": 4414.25390625,
      "loss/core": 4414.23193359375,
      "loss/preference_sft": 0.2729622721672058,
      "loss/reference_anchor": 1.0718512535095215,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.606678009033203,
      "rewards/margins": 2.375056028366089,
      "rewards/rejected": 2.2316224575042725,
      "step": 845
    },
    {
      "drift/chosen_abs": 0.17466862499713898,
      "drift/rejected_abs": 0.06862124055624008,
      "epoch": 0.8121342410127792,
      "grad_norm": 424.0,
      "learning_rate": 2.4374380305025866e-09,
      "logits/chosen": -2.2522521018981934,
      "logits/rejected": -2.309323787689209,
      "logps/chosen": -0.2542715072631836,
      "logps/rejected": -0.2512281537055969,
      "loss": 4430.23974609375,
      "loss/core": 4430.2373046875,
      "loss/preference_sft": 0.2542715072631836,
      "loss/reference_anchor": 0.06204066798090935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.7456880807876587,
      "rewards/margins": 1.0702584981918335,
      "rewards/rejected": 0.6754294633865356,
      "step": 850
    },
    {
      "drift/chosen_abs": 0.23169496655464172,
      "drift/rejected_abs": 0.19141283631324768,
      "epoch": 0.8169115012540308,
      "grad_norm": 2560.0,
      "learning_rate": 1.984143567294594e-09,
      "logits/chosen": -2.089970350265503,
      "logits/rejected": -2.082474946975708,
      "logps/chosen": -0.2673383355140686,
      "logps/rejected": -0.2954670786857605,
      "loss": 4439.4287109375,
      "loss/core": 4439.421875,
      "loss/preference_sft": 0.2673383355140686,
      "loss/reference_anchor": 0.32883888483047485,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.314150333404541,
      "rewards/margins": 0.4091102182865143,
      "rewards/rejected": 1.9050400257110596,
      "step": 855
    },
    {
      "drift/chosen_abs": 0.19237197935581207,
      "drift/rejected_abs": 0.10806970298290253,
      "epoch": 0.8216887614952825,
      "grad_norm": 544.0,
      "learning_rate": 1.577110336711096e-09,
      "logits/chosen": -2.185058116912842,
      "logits/rejected": -2.2582504749298096,
      "logps/chosen": -0.29970070719718933,
      "logps/rejected": -0.30701369047164917,
      "loss": 4433.2958984375,
      "loss/core": 4433.291015625,
      "loss/preference_sft": 0.29970070719718933,
      "loss/reference_anchor": 0.2221534699201584,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.921847939491272,
      "rewards/margins": 0.8417887687683105,
      "rewards/rejected": 1.0800589323043823,
      "step": 860
    },
    {
      "drift/chosen_abs": 0.26305049657821655,
      "drift/rejected_abs": 0.12108287960290909,
      "epoch": 0.826466021736534,
      "grad_norm": 736.0,
      "learning_rate": 1.2164914073037048e-09,
      "logits/chosen": -2.147150754928589,
      "logits/rejected": -2.2282862663269043,
      "logps/chosen": -0.29672080278396606,
      "logps/rejected": -0.28275054693222046,
      "loss": 4426.12451171875,
      "loss/core": 4426.11181640625,
      "loss/preference_sft": 0.29672080278396606,
      "loss/reference_anchor": 0.6080988049507141,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.629103183746338,
      "rewards/margins": 1.4199514389038086,
      "rewards/rejected": 1.2091517448425293,
      "step": 865
    },
    {
      "drift/chosen_abs": 0.4451891779899597,
      "drift/rejected_abs": 0.21213407814502716,
      "epoch": 0.8312432819777857,
      "grad_norm": 2024.0,
      "learning_rate": 9.024223931035357e-10,
      "logits/chosen": -1.9941089153289795,
      "logits/rejected": -1.9594297409057617,
      "logps/chosen": -0.2970310151576996,
      "logps/rejected": -0.2791312336921692,
      "loss": 4414.5986328125,
      "loss/core": 4414.580078125,
      "loss/preference_sft": 0.2970310151576996,
      "loss/reference_anchor": 0.8715806007385254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.448572158813477,
      "rewards/margins": 2.3311219215393066,
      "rewards/rejected": 2.1174495220184326,
      "step": 870
    },
    {
      "drift/chosen_abs": 0.19540363550186157,
      "drift/rejected_abs": 0.13800457119941711,
      "epoch": 0.8360205422190374,
      "grad_norm": 440.0,
      "learning_rate": 6.350214026223516e-10,
      "logits/chosen": -2.0932259559631348,
      "logits/rejected": -2.0954394340515137,
      "logps/chosen": -0.30296754837036133,
      "logps/rejected": -0.3049096465110779,
      "loss": 4436.95068359375,
      "loss/core": 4436.94580078125,
      "loss/preference_sft": 0.30296754837036133,
      "loss/reference_anchor": 0.24102571606636047,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.953322172164917,
      "rewards/margins": 0.5735519528388977,
      "rewards/rejected": 1.379770278930664,
      "step": 875
    },
    {
      "drift/chosen_abs": 0.37028616666793823,
      "drift/rejected_abs": 0.15220406651496887,
      "epoch": 0.840797802460289,
      "grad_norm": 360.0,
      "learning_rate": 4.143889944367429e-10,
      "logits/chosen": -2.00821590423584,
      "logits/rejected": -1.9785652160644531,
      "logps/chosen": -0.28579357266426086,
      "logps/rejected": -0.30298155546188354,
      "loss": 4416.18603515625,
      "loss/core": 4416.17138671875,
      "loss/preference_sft": 0.28579357266426086,
      "loss/reference_anchor": 0.7266721725463867,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.702655076980591,
      "rewards/margins": 2.1888763904571533,
      "rewards/rejected": 1.5137783288955688,
      "step": 880
    },
    {
      "drift/chosen_abs": 0.15521343052387238,
      "drift/rejected_abs": 0.07931456714868546,
      "epoch": 0.8455750627015407,
      "grad_norm": 2080.0,
      "learning_rate": 2.406081393722531e-10,
      "logits/chosen": -2.103982448577881,
      "logits/rejected": -2.132598876953125,
      "logps/chosen": -0.29826053977012634,
      "logps/rejected": -0.3063696026802063,
      "loss": 4434.3544921875,
      "loss/core": 4434.3525390625,
      "loss/preference_sft": 0.29826053977012634,
      "loss/reference_anchor": 0.05555032566189766,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5512726306915283,
      "rewards/margins": 0.7596304416656494,
      "rewards/rejected": 0.7916421890258789,
      "step": 885
    },
    {
      "drift/chosen_abs": 0.18553657829761505,
      "drift/rejected_abs": 0.1623484194278717,
      "epoch": 0.8503523229427923,
      "grad_norm": 1152.0,
      "learning_rate": 1.1374418930135133e-10,
      "logits/chosen": -2.1390254497528076,
      "logits/rejected": -2.191925525665283,
      "logps/chosen": -0.26982808113098145,
      "logps/rejected": -0.28849366307258606,
      "loss": 4441.68017578125,
      "loss/core": 4441.67431640625,
      "loss/preference_sft": 0.26982808113098145,
      "loss/reference_anchor": 0.2661302983760834,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8522002696990967,
      "rewards/margins": 0.22871632874011993,
      "rewards/rejected": 1.6234838962554932,
      "step": 890
    },
    {
      "drift/chosen_abs": 0.21298833191394806,
      "drift/rejected_abs": 0.09839890152215958,
      "epoch": 0.855129583184044,
      "grad_norm": 5824.0,
      "learning_rate": 3.3844852567285756e-11,
      "logits/chosen": -2.102201223373413,
      "logits/rejected": -2.060621738433838,
      "logps/chosen": -0.29058605432510376,
      "logps/rejected": -0.2914868891239166,
      "loss": 4429.2578125,
      "loss/core": 4429.2548828125,
      "loss/preference_sft": 0.29058605432510376,
      "loss/reference_anchor": 0.13424192368984222,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.129883289337158,
      "rewards/margins": 1.1480109691619873,
      "rewards/rejected": 0.9818723797798157,
      "step": 895
    },
    {
      "drift/chosen_abs": 0.29029321670532227,
      "drift/rejected_abs": 0.14265631139278412,
      "epoch": 0.8599068434252956,
      "grad_norm": 7520.0,
      "learning_rate": 9.401760429905703e-13,
      "logits/chosen": -2.05942964553833,
      "logits/rejected": -2.0243592262268066,
      "logps/chosen": -0.2692040205001831,
      "logps/rejected": -0.27301639318466187,
      "loss": 4425.076171875,
      "loss/core": 4425.06787109375,
      "loss/preference_sft": 0.2692040205001831,
      "loss/reference_anchor": 0.38637253642082214,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9029033184051514,
      "rewards/margins": 1.476595163345337,
      "rewards/rejected": 1.4263079166412354,
      "step": 900
    },
    {
      "epoch": 0.8599068434252956,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 4429.236100260417,
      "train_runtime": 7053.367,
      "train_samples_per_second": 2.042,
      "train_steps_per_second": 0.128
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}