{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.134796238244514,
  "eval_steps": 500,
  "global_step": 2000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047021943573667714,
      "grad_norm": 1.1235875661288681,
      "learning_rate": 9.999801114386507e-06,
      "logits/chosen": 0.11724523454904556,
      "logits/rejected": 0.2596210241317749,
      "logps/chosen": -1.0975141525268555,
      "logps/rejected": -0.9545950889587402,
      "loss": 1.7296,
      "odds_ratio_loss": 6.3209943771362305,
      "rewards/accuracies": 0.24635416269302368,
      "rewards/chosen": -0.10975142568349838,
      "rewards/margins": -0.014291909523308277,
      "rewards/rejected": -0.09545952081680298,
      "sft_loss": 1.0975141525268555,
      "step": 30
    },
    {
      "epoch": 0.09404388714733543,
      "grad_norm": 1.086092490147137,
      "learning_rate": 9.996265810285894e-06,
      "logits/chosen": 0.06234961748123169,
      "logits/rejected": 0.22102031111717224,
      "logps/chosen": -0.9451744556427002,
      "logps/rejected": -1.1230847835540771,
      "loss": 1.4772,
      "odds_ratio_loss": 5.320559501647949,
      "rewards/accuracies": 0.7739583849906921,
      "rewards/chosen": -0.09451745450496674,
      "rewards/margins": 0.017791030928492546,
      "rewards/rejected": -0.11230848729610443,
      "sft_loss": 0.9451744556427002,
      "step": 60
    },
    {
      "epoch": 0.14106583072100312,
      "grad_norm": 1.1108478150041545,
      "learning_rate": 9.988314422702716e-06,
      "logits/chosen": 0.12033864855766296,
      "logits/rejected": 0.11232076585292816,
      "logps/chosen": -0.9296269416809082,
      "logps/rejected": -5.353595733642578,
      "loss": 1.4246,
      "odds_ratio_loss": 4.9492692947387695,
      "rewards/accuracies": 0.9562500715255737,
      "rewards/chosen": -0.09296268969774246,
      "rewards/margins": 0.4423968493938446,
      "rewards/rejected": -0.5353595018386841,
      "sft_loss": 0.9296269416809082,
      "step": 90
    },
    {
      "epoch": 0.18808777429467086,
      "grad_norm": 1.0395955342939103,
      "learning_rate": 9.975953979684062e-06,
      "logits/chosen": -0.059552229940891266,
      "logits/rejected": -0.010672621428966522,
      "logps/chosen": -0.9207684397697449,
      "logps/rejected": -24.890125274658203,
      "loss": 1.3819,
      "odds_ratio_loss": 4.610863208770752,
      "rewards/accuracies": 0.9979166984558105,
      "rewards/chosen": -0.0920768529176712,
      "rewards/margins": 2.3969357013702393,
      "rewards/rejected": -2.4890124797821045,
      "sft_loss": 0.9207684397697449,
      "step": 120
    },
    {
      "epoch": 0.23510971786833856,
      "grad_norm": 1.0746539201985001,
      "learning_rate": 9.959195406338866e-06,
      "logits/chosen": -0.24456781148910522,
      "logits/rejected": -0.25657919049263,
      "logps/chosen": -0.9097563028335571,
      "logps/rejected": -33.72811508178711,
      "loss": 1.3651,
      "odds_ratio_loss": 4.553545951843262,
      "rewards/accuracies": 0.9973958730697632,
      "rewards/chosen": -0.09097563475370407,
      "rewards/margins": 3.2818357944488525,
      "rewards/rejected": -3.3728113174438477,
      "sft_loss": 0.9097563028335571,
      "step": 150
    },
    {
      "epoch": 0.28213166144200624,
      "grad_norm": 1.0528834041448145,
      "learning_rate": 9.938053515181465e-06,
      "logits/chosen": -0.19424977898597717,
      "logits/rejected": -0.6158410310745239,
      "logps/chosen": -0.9048575162887573,
      "logps/rejected": -39.44026184082031,
      "loss": 1.3581,
      "odds_ratio_loss": 4.532027721405029,
      "rewards/accuracies": 0.9979166984558105,
      "rewards/chosen": -0.09048575162887573,
      "rewards/margins": 3.853541135787964,
      "rewards/rejected": -3.9440269470214844,
      "sft_loss": 0.9048575162887573,
      "step": 180
    },
    {
      "epoch": 0.329153605015674,
      "grad_norm": 1.0415419362116665,
      "learning_rate": 9.912546993039157e-06,
      "logits/chosen": -0.02191479504108429,
      "logits/rejected": -0.8679313063621521,
      "logps/chosen": -0.89907306432724,
      "logps/rejected": -41.08076477050781,
      "loss": 1.3498,
      "odds_ratio_loss": 4.506892681121826,
      "rewards/accuracies": 0.9937500357627869,
      "rewards/chosen": -0.08990731090307236,
      "rewards/margins": 4.018168926239014,
      "rewards/rejected": -4.108077049255371,
      "sft_loss": 0.89907306432724,
      "step": 210
    },
    {
      "epoch": 0.3761755485893417,
      "grad_norm": 1.105436137299816,
      "learning_rate": 9.882698384535345e-06,
      "logits/chosen": -0.02618134766817093,
      "logits/rejected": -0.9593080878257751,
      "logps/chosen": -0.8983286619186401,
      "logps/rejected": -40.29463195800781,
      "loss": 1.3482,
      "odds_ratio_loss": 4.498676300048828,
      "rewards/accuracies": 0.9968750476837158,
      "rewards/chosen": -0.0898328572511673,
      "rewards/margins": 3.9396305084228516,
      "rewards/rejected": -4.029463291168213,
      "sft_loss": 0.8983286619186401,
      "step": 240
    },
    {
      "epoch": 0.4231974921630094,
      "grad_norm": 1.0565241888624548,
      "learning_rate": 9.848534072162876e-06,
      "logits/chosen": -0.3394937217235565,
      "logits/rejected": -0.8553600311279297,
      "logps/chosen": -0.898851752281189,
      "logps/rejected": -22.5789852142334,
      "loss": 1.3715,
      "odds_ratio_loss": 4.726236343383789,
      "rewards/accuracies": 0.9609376192092896,
      "rewards/chosen": -0.0898851826786995,
      "rewards/margins": 2.168013334274292,
      "rewards/rejected": -2.2578983306884766,
      "sft_loss": 0.898851752281189,
      "step": 270
    },
    {
      "epoch": 0.4702194357366771,
      "grad_norm": 1.0100518691076887,
      "learning_rate": 9.810084252965162e-06,
      "logits/chosen": -0.5359396934509277,
      "logits/rejected": -0.7189143896102905,
      "logps/chosen": -0.8881340622901917,
      "logps/rejected": -5.904677391052246,
      "loss": 1.3596,
      "odds_ratio_loss": 4.7141828536987305,
      "rewards/accuracies": 0.9843750596046448,
      "rewards/chosen": -0.0888134092092514,
      "rewards/margins": 0.501654326915741,
      "rewards/rejected": -0.5904676914215088,
      "sft_loss": 0.8881340622901917,
      "step": 300
    },
    {
      "epoch": 0.5172413793103449,
      "grad_norm": 0.9690370862954015,
      "learning_rate": 9.767382911845711e-06,
      "logits/chosen": -0.6090168356895447,
      "logits/rejected": -1.3381540775299072,
      "logps/chosen": -0.8794980049133301,
      "logps/rejected": -26.965978622436523,
      "loss": 1.3202,
      "odds_ratio_loss": 4.406744956970215,
      "rewards/accuracies": 0.9963542222976685,
      "rewards/chosen": -0.08794979751110077,
      "rewards/margins": 2.6086480617523193,
      "rewards/rejected": -2.6965980529785156,
      "sft_loss": 0.8794980049133301,
      "step": 330
    },
    {
      "epoch": 0.5642633228840125,
      "grad_norm": 0.9969800996732315,
      "learning_rate": 9.720467791529659e-06,
      "logits/chosen": -0.5933035612106323,
      "logits/rejected": -1.2798489332199097,
      "logps/chosen": -0.877310037612915,
      "logps/rejected": -29.602815628051758,
      "loss": 1.3168,
      "odds_ratio_loss": 4.394427299499512,
      "rewards/accuracies": 0.9963542222976685,
      "rewards/chosen": -0.0877310037612915,
      "rewards/margins": 2.8725509643554688,
      "rewards/rejected": -2.9602818489074707,
      "sft_loss": 0.877310037612915,
      "step": 360
    },
    {
      "epoch": 0.6112852664576802,
      "grad_norm": 0.9419202894251142,
      "learning_rate": 9.669380359203862e-06,
      "logits/chosen": -0.7155650854110718,
      "logits/rejected": -1.3223919868469238,
      "logps/chosen": -0.8825143575668335,
      "logps/rejected": -31.081226348876953,
      "loss": 1.3245,
      "odds_ratio_loss": 4.420156478881836,
      "rewards/accuracies": 0.9968750476837158,
      "rewards/chosen": -0.08825143426656723,
      "rewards/margins": 3.0198709964752197,
      "rewards/rejected": -3.1081225872039795,
      "sft_loss": 0.8825143575668335,
      "step": 390
    },
    {
      "epoch": 0.658307210031348,
      "grad_norm": 1.0278725212806465,
      "learning_rate": 9.614165769864993e-06,
      "logits/chosen": -0.8998770713806152,
      "logits/rejected": -1.4382452964782715,
      "logps/chosen": -0.862067461013794,
      "logps/rejected": -32.7113151550293,
      "loss": 1.294,
      "odds_ratio_loss": 4.319062232971191,
      "rewards/accuracies": 0.9968750476837158,
      "rewards/chosen": -0.08620674908161163,
      "rewards/margins": 3.184924602508545,
      "rewards/rejected": -3.2711315155029297,
      "sft_loss": 0.862067461013794,
      "step": 420
    },
    {
      "epoch": 0.7053291536050157,
      "grad_norm": 0.9866461928420249,
      "learning_rate": 9.554872826408102e-06,
      "logits/chosen": -0.9998570084571838,
      "logits/rejected": -1.401207685470581,
      "logps/chosen": -0.8621182441711426,
      "logps/rejected": -33.86608123779297,
      "loss": 1.294,
      "odds_ratio_loss": 4.3188796043396,
      "rewards/accuracies": 0.9963542222976685,
      "rewards/chosen": -0.08621183037757874,
      "rewards/margins": 3.300396203994751,
      "rewards/rejected": -3.386608123779297,
      "sft_loss": 0.8621182441711426,
      "step": 450
    },
    {
      "epoch": 0.7523510971786834,
      "grad_norm": 0.9659912223798843,
      "learning_rate": 9.49155393649086e-06,
      "logits/chosen": -0.9582356810569763,
      "logits/rejected": -1.3586877584457397,
      "logps/chosen": -0.8651710748672485,
      "logps/rejected": -34.725868225097656,
      "loss": 1.2983,
      "odds_ratio_loss": 4.3313446044921875,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.0865171030163765,
      "rewards/margins": 3.3860700130462646,
      "rewards/rejected": -3.4725871086120605,
      "sft_loss": 0.8651710748672485,
      "step": 480
    },
    {
      "epoch": 0.799373040752351,
      "grad_norm": 1.023178923568536,
      "learning_rate": 9.424265066211657e-06,
      "logits/chosen": -0.979565441608429,
      "logits/rejected": -1.3727631568908691,
      "logps/chosen": -0.8631203770637512,
      "logps/rejected": -35.165767669677734,
      "loss": 1.2958,
      "odds_ratio_loss": 4.326568603515625,
      "rewards/accuracies": 0.9942708611488342,
      "rewards/chosen": -0.08631204068660736,
      "rewards/margins": 3.430264711380005,
      "rewards/rejected": -3.5165767669677734,
      "sft_loss": 0.8631203770637512,
      "step": 510
    },
    {
      "epoch": 0.8463949843260188,
      "grad_norm": 0.9266864933596661,
      "learning_rate": 9.353065690642453e-06,
      "logits/chosen": -0.9409717917442322,
      "logits/rejected": -1.401993751525879,
      "logps/chosen": -0.8574859499931335,
      "logps/rejected": -35.339935302734375,
      "loss": 1.2869,
      "odds_ratio_loss": 4.294265270233154,
      "rewards/accuracies": 0.9968750476837158,
      "rewards/chosen": -0.085748590528965,
      "rewards/margins": 3.448244571685791,
      "rewards/rejected": -3.5339932441711426,
      "sft_loss": 0.8574859499931335,
      "step": 540
    },
    {
      "epoch": 0.8934169278996865,
      "grad_norm": 0.991116661066985,
      "learning_rate": 9.278018741260144e-06,
      "logits/chosen": -0.9096575379371643,
      "logits/rejected": -1.3825011253356934,
      "logps/chosen": -0.8583310842514038,
      "logps/rejected": -36.20825958251953,
      "loss": 1.288,
      "odds_ratio_loss": 4.296856880187988,
      "rewards/accuracies": 0.9968750476837158,
      "rewards/chosen": -0.0858331024646759,
      "rewards/margins": 3.5349931716918945,
      "rewards/rejected": -3.620826244354248,
      "sft_loss": 0.8583310842514038,
      "step": 570
    },
    {
      "epoch": 0.9404388714733543,
      "grad_norm": 0.9532041283621853,
      "learning_rate": 9.199190550322895e-06,
      "logits/chosen": -1.046135425567627,
      "logits/rejected": -1.4427109956741333,
      "logps/chosen": -0.8530829548835754,
      "logps/rejected": -36.00242614746094,
      "loss": 1.2804,
      "odds_ratio_loss": 4.273621082305908,
      "rewards/accuracies": 0.9973958730697632,
      "rewards/chosen": -0.08530830591917038,
      "rewards/margins": 3.5149343013763428,
      "rewards/rejected": -3.600242853164673,
      "sft_loss": 0.8530829548835754,
      "step": 600
    },
    {
      "epoch": 0.987460815047022,
      "grad_norm": 0.9736159250338984,
      "learning_rate": 9.116650792240592e-06,
      "logits/chosen": -1.136152982711792,
      "logits/rejected": -1.4546109437942505,
      "logps/chosen": -0.8512221574783325,
      "logps/rejected": -36.04446029663086,
      "loss": 1.2774,
      "odds_ratio_loss": 4.2616400718688965,
      "rewards/accuracies": 0.9979166984558105,
      "rewards/chosen": -0.08512221276760101,
      "rewards/margins": 3.5193233489990234,
      "rewards/rejected": -3.604445695877075,
      "sft_loss": 0.8512221574783325,
      "step": 630
    },
    {
      "epoch": 1.0344827586206897,
      "grad_norm": 0.9519135160773134,
      "learning_rate": 9.030472421991243e-06,
      "logits/chosen": -1.4093230962753296,
      "logits/rejected": -1.5503480434417725,
      "logps/chosen": -0.67490553855896,
      "logps/rejected": -37.34769058227539,
      "loss": 1.0127,
      "odds_ratio_loss": 3.377951145172119,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06749054789543152,
      "rewards/margins": 3.66727876663208,
      "rewards/rejected": -3.734768867492676,
      "sft_loss": 0.67490553855896,
      "step": 660
    },
    {
      "epoch": 1.0815047021943573,
      "grad_norm": 1.0315297815628177,
      "learning_rate": 8.940731610637772e-06,
      "logits/chosen": -1.5676097869873047,
      "logits/rejected": -1.540395975112915,
      "logps/chosen": -0.6025470495223999,
      "logps/rejected": -37.542171478271484,
      "loss": 0.9043,
      "odds_ratio_loss": 3.0179269313812256,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.06025470420718193,
      "rewards/margins": 3.693962574005127,
      "rewards/rejected": -3.7542171478271484,
      "sft_loss": 0.6025470495223999,
      "step": 690
    },
    {
      "epoch": 1.1285266457680252,
      "grad_norm": 1.0247278224991563,
      "learning_rate": 8.847507678002177e-06,
      "logits/chosen": -1.4729366302490234,
      "logits/rejected": -1.5617899894714355,
      "logps/chosen": -0.6058939695358276,
      "logps/rejected": -38.69839096069336,
      "loss": 0.9094,
      "odds_ratio_loss": 3.0350539684295654,
      "rewards/accuracies": 0.9979166984558105,
      "rewards/chosen": -0.060589395463466644,
      "rewards/margins": 3.8092496395111084,
      "rewards/rejected": -3.8698391914367676,
      "sft_loss": 0.6058939695358276,
      "step": 720
    },
    {
      "epoch": 1.1755485893416928,
      "grad_norm": 1.0213178135656136,
      "learning_rate": 8.75088302255658e-06,
      "logits/chosen": -1.4907186031341553,
      "logits/rejected": -1.5485866069793701,
      "logps/chosen": -0.6068739295005798,
      "logps/rejected": -38.33498001098633,
      "loss": 0.9111,
      "odds_ratio_loss": 3.041877508163452,
      "rewards/accuracies": 0.9963542222976685,
      "rewards/chosen": -0.06068739295005798,
      "rewards/margins": 3.772810697555542,
      "rewards/rejected": -3.8334977626800537,
      "sft_loss": 0.6068739295005798,
      "step": 750
    },
    {
      "epoch": 1.2225705329153604,
      "grad_norm": 1.0161893086023928,
      "learning_rate": 8.650943048593135e-06,
      "logits/chosen": -1.5544003248214722,
      "logits/rejected": -1.540315866470337,
      "logps/chosen": -0.6121038794517517,
      "logps/rejected": -38.208885192871094,
      "loss": 0.9187,
      "odds_ratio_loss": 3.065685749053955,
      "rewards/accuracies": 0.9979166984558105,
      "rewards/chosen": -0.06121038645505905,
      "rewards/margins": 3.759678363800049,
      "rewards/rejected": -3.8208887577056885,
      "sft_loss": 0.6121038794517517,
      "step": 780
    },
    {
      "epoch": 1.2695924764890283,
      "grad_norm": 1.0772783309503091,
      "learning_rate": 8.547776090737143e-06,
      "logits/chosen": -1.4951478242874146,
      "logits/rejected": -1.542786955833435,
      "logps/chosen": -0.6139963269233704,
      "logps/rejected": -38.818702697753906,
      "loss": 0.9213,
      "odds_ratio_loss": 3.073441505432129,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06139963120222092,
      "rewards/margins": 3.8204703330993652,
      "rewards/rejected": -3.8818702697753906,
      "sft_loss": 0.6139963269233704,
      "step": 810
    },
    {
      "epoch": 1.316614420062696,
      "grad_norm": 0.9900333449019929,
      "learning_rate": 8.441473335870134e-06,
      "logits/chosen": -1.4873510599136353,
      "logits/rejected": -1.5388774871826172,
      "logps/chosen": -0.6097849607467651,
      "logps/rejected": -39.01841354370117,
      "loss": 0.915,
      "odds_ratio_loss": 3.0520551204681396,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.060978494584560394,
      "rewards/margins": 3.840862989425659,
      "rewards/rejected": -3.901841402053833,
      "sft_loss": 0.6097849607467651,
      "step": 840
    },
    {
      "epoch": 1.3636363636363638,
      "grad_norm": 1.0446900620996404,
      "learning_rate": 8.332128742531895e-06,
      "logits/chosen": -1.5400224924087524,
      "logits/rejected": -1.5519053936004639,
      "logps/chosen": -0.6121001243591309,
      "logps/rejected": -39.485267639160156,
      "loss": 0.9186,
      "odds_ratio_loss": 3.0645432472229004,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": -0.061210013926029205,
      "rewards/margins": 3.887317180633545,
      "rewards/rejected": -3.9485275745391846,
      "sft_loss": 0.6121001243591309,
      "step": 870
    },
    {
      "epoch": 1.4106583072100314,
      "grad_norm": 1.0584585285332069,
      "learning_rate": 8.219838957872695e-06,
      "logits/chosen": -1.5601458549499512,
      "logits/rejected": -1.5774872303009033,
      "logps/chosen": -0.612558126449585,
      "logps/rejected": -39.039276123046875,
      "loss": 0.9193,
      "odds_ratio_loss": 3.0671072006225586,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.061255816370248795,
      "rewards/margins": 3.842672109603882,
      "rewards/rejected": -3.903927803039551,
      "sft_loss": 0.612558126449585,
      "step": 900
    },
    {
      "epoch": 1.457680250783699,
      "grad_norm": 1.03154453551069,
      "learning_rate": 8.10470323222911e-06,
      "logits/chosen": -1.5531983375549316,
      "logits/rejected": -1.5607452392578125,
      "logps/chosen": -0.6180831789970398,
      "logps/rejected": -39.3043212890625,
      "loss": 0.9278,
      "odds_ratio_loss": 3.0967118740081787,
      "rewards/accuracies": 0.9979166984558105,
      "rewards/chosen": -0.061808325350284576,
      "rewards/margins": 3.868623733520508,
      "rewards/rejected": -3.9304323196411133,
      "sft_loss": 0.6180831789970398,
      "step": 930
    },
    {
      "epoch": 1.5047021943573666,
      "grad_norm": 1.1180953361782457,
      "learning_rate": 7.98682333139895e-06,
      "logits/chosen": -1.55546236038208,
      "logits/rejected": -1.5939733982086182,
      "logps/chosen": -0.6205478310585022,
      "logps/rejected": -39.821533203125,
      "loss": 0.9313,
      "odds_ratio_loss": 3.1071197986602783,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.06205478310585022,
      "rewards/margins": 3.9200985431671143,
      "rewards/rejected": -3.9821529388427734,
      "sft_loss": 0.6205478310585022,
      "step": 960
    },
    {
      "epoch": 1.5517241379310345,
      "grad_norm": 1.0650244622896785,
      "learning_rate": 7.866303446692838e-06,
      "logits/chosen": -1.5789411067962646,
      "logits/rejected": -1.6112656593322754,
      "logps/chosen": -0.6237287521362305,
      "logps/rejected": -39.89510726928711,
      "loss": 0.9359,
      "odds_ratio_loss": 3.1217641830444336,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06237287074327469,
      "rewards/margins": 3.927137851715088,
      "rewards/rejected": -3.989511013031006,
      "sft_loss": 0.6237287521362305,
      "step": 990
    },
    {
      "epoch": 1.5987460815047023,
      "grad_norm": 1.0533917228283542,
      "learning_rate": 7.74325010284192e-06,
      "logits/chosen": -1.448456883430481,
      "logits/rejected": -1.5946437120437622,
      "logps/chosen": -0.6208869218826294,
      "logps/rejected": -40.297550201416016,
      "loss": 0.9319,
      "odds_ratio_loss": 3.1104769706726074,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": -0.06208869069814682,
      "rewards/margins": 3.9676666259765625,
      "rewards/rejected": -4.029755115509033,
      "sft_loss": 0.6208869218826294,
      "step": 1020
    },
    {
      "epoch": 1.64576802507837,
      "grad_norm": 1.0981490419572077,
      "learning_rate": 7.617772063843125e-06,
      "logits/chosen": -1.3358079195022583,
      "logits/rejected": -1.6043446063995361,
      "logps/chosen": -0.6198667883872986,
      "logps/rejected": -40.07209014892578,
      "loss": 0.9302,
      "odds_ratio_loss": 3.103623867034912,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06198667734861374,
      "rewards/margins": 3.9452221393585205,
      "rewards/rejected": -4.007209300994873,
      "sft_loss": 0.6198667883872986,
      "step": 1050
    },
    {
      "epoch": 1.6927899686520376,
      "grad_norm": 1.002702342649321,
      "learning_rate": 7.489980236825194e-06,
      "logits/chosen": -1.4426370859146118,
      "logits/rejected": -1.573171615600586,
      "logps/chosen": -0.6225786805152893,
      "logps/rejected": -40.08259582519531,
      "loss": 0.9343,
      "odds_ratio_loss": 3.1171209812164307,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06225787475705147,
      "rewards/margins": 3.9460020065307617,
      "rewards/rejected": -4.0082597732543945,
      "sft_loss": 0.6225786805152893,
      "step": 1080
    },
    {
      "epoch": 1.7398119122257052,
      "grad_norm": 1.045644355148547,
      "learning_rate": 7.3599875740204405e-06,
      "logits/chosen": -1.3436530828475952,
      "logits/rejected": -1.578587532043457,
      "logps/chosen": -0.6230885982513428,
      "logps/rejected": -40.49763870239258,
      "loss": 0.935,
      "odds_ratio_loss": 3.1195812225341797,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.062308862805366516,
      "rewards/margins": 3.987455129623413,
      "rewards/rejected": -4.0497636795043945,
      "sft_loss": 0.6230885982513428,
      "step": 1110
    },
    {
      "epoch": 1.786833855799373,
      "grad_norm": 1.0927200855751504,
      "learning_rate": 7.227908972928892e-06,
      "logits/chosen": -1.3714853525161743,
      "logits/rejected": -1.6073870658874512,
      "logps/chosen": -0.6151573657989502,
      "logps/rejected": -39.90351867675781,
      "loss": 0.9232,
      "odds_ratio_loss": 3.079930543899536,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06151573359966278,
      "rewards/margins": 3.928835868835449,
      "rewards/rejected": -3.990351676940918,
      "sft_loss": 0.6151573657989502,
      "step": 1140
    },
    {
      "epoch": 1.8338557993730409,
      "grad_norm": 1.042619184198925,
      "learning_rate": 7.0938611747630496e-06,
      "logits/chosen": -1.4685415029525757,
      "logits/rejected": -1.5880926847457886,
      "logps/chosen": -0.6156089305877686,
      "logps/rejected": -40.116119384765625,
      "loss": 0.9239,
      "odds_ratio_loss": 3.082674741744995,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": -0.061560891568660736,
      "rewards/margins": 3.9500508308410645,
      "rewards/rejected": -4.0116119384765625,
      "sft_loss": 0.6156089305877686,
      "step": 1170
    },
    {
      "epoch": 1.8808777429467085,
      "grad_norm": 1.0318644739056106,
      "learning_rate": 6.957962661263043e-06,
      "logits/chosen": -1.4960691928863525,
      "logits/rejected": -1.6382880210876465,
      "logps/chosen": -0.6132216453552246,
      "logps/rejected": -40.39379119873047,
      "loss": 0.9202,
      "odds_ratio_loss": 3.0698792934417725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0613221675157547,
      "rewards/margins": 3.9780571460723877,
      "rewards/rejected": -4.039379119873047,
      "sft_loss": 0.6132216453552246,
      "step": 1200
    },
    {
      "epoch": 1.9278996865203761,
      "grad_norm": 0.9587286583895697,
      "learning_rate": 6.820333549973367e-06,
      "logits/chosen": -1.4510760307312012,
      "logits/rejected": -1.6273317337036133,
      "logps/chosen": -0.6141430139541626,
      "logps/rejected": -40.484169006347656,
      "loss": 0.9216,
      "odds_ratio_loss": 3.074673652648926,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06141430139541626,
      "rewards/margins": 3.9870028495788574,
      "rewards/rejected": -4.048417091369629,
      "sft_loss": 0.6141430139541626,
      "step": 1230
    },
    {
      "epoch": 1.9749216300940438,
      "grad_norm": 0.9879114442885194,
      "learning_rate": 6.68109548807377e-06,
      "logits/chosen": -1.4274954795837402,
      "logits/rejected": -1.612345576286316,
      "logps/chosen": -0.6152713894844055,
      "logps/rejected": -38.77534484863281,
      "loss": 0.9233,
      "odds_ratio_loss": 3.0799922943115234,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.06152713671326637,
      "rewards/margins": 3.816007375717163,
      "rewards/rejected": -3.8775346279144287,
      "sft_loss": 0.6152713894844055,
      "step": 1260
    },
    {
      "epoch": 2.0219435736677114,
      "grad_norm": 1.0845364591851332,
      "learning_rate": 6.5403715448581284e-06,
      "logits/chosen": -1.7770729064941406,
      "logits/rejected": -1.7219361066818237,
      "logps/chosen": -0.5043900012969971,
      "logps/rejected": -38.080379486083984,
      "loss": 0.7567,
      "odds_ratio_loss": 2.52349853515625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05043900012969971,
      "rewards/margins": 3.757598876953125,
      "rewards/rejected": -3.808037757873535,
      "sft_loss": 0.5043900012969971,
      "step": 1290
    },
    {
      "epoch": 2.0689655172413794,
      "grad_norm": 1.0611444492838484,
      "learning_rate": 6.398286102956354e-06,
      "logits/chosen": -2.0708460807800293,
      "logits/rejected": -1.8045943975448608,
      "logps/chosen": -0.36996203660964966,
      "logps/rejected": -39.32792282104492,
      "loss": 0.5552,
      "odds_ratio_loss": 1.8524706363677979,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.03699620068073273,
      "rewards/margins": 3.895796298980713,
      "rewards/rejected": -3.9327926635742188,
      "sft_loss": 0.36996203660964966,
      "step": 1320
    },
    {
      "epoch": 2.115987460815047,
      "grad_norm": 1.0826753914388125,
      "learning_rate": 6.254964748395482e-06,
      "logits/chosen": -1.8300830125808716,
      "logits/rejected": -1.7398159503936768,
      "logps/chosen": -0.3730930685997009,
      "logps/rejected": -40.005470275878906,
      "loss": 0.5598,
      "odds_ratio_loss": 1.866748571395874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03730930760502815,
      "rewards/margins": 3.963238000869751,
      "rewards/rejected": -4.000547409057617,
      "sft_loss": 0.3730930685997009,
      "step": 1350
    },
    {
      "epoch": 2.1630094043887147,
      "grad_norm": 1.1611530323505992,
      "learning_rate": 6.110534159597075e-06,
      "logits/chosen": -1.4871913194656372,
      "logits/rejected": -1.6748030185699463,
      "logps/chosen": -0.37595564126968384,
      "logps/rejected": -40.809329986572266,
      "loss": 0.5642,
      "odds_ratio_loss": 1.882346749305725,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.03759556636214256,
      "rewards/margins": 4.043337821960449,
      "rewards/rejected": -4.080933094024658,
      "sft_loss": 0.37595564126968384,
      "step": 1380
    },
    {
      "epoch": 2.2100313479623823,
      "grad_norm": 1.0518829938359235,
      "learning_rate": 5.965121995409121e-06,
      "logits/chosen": -1.8022518157958984,
      "logits/rejected": -1.7283188104629517,
      "logps/chosen": -0.37588319182395935,
      "logps/rejected": -40.30139923095703,
      "loss": 0.5641,
      "odds_ratio_loss": 1.8821475505828857,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.037588316947221756,
      "rewards/margins": 3.992551326751709,
      "rewards/rejected": -4.030139923095703,
      "sft_loss": 0.37588319182395935,
      "step": 1410
    },
    {
      "epoch": 2.2570532915360504,
      "grad_norm": 1.078581003449218,
      "learning_rate": 5.818856782271325e-06,
      "logits/chosen": -1.7889518737792969,
      "logits/rejected": -1.7148984670639038,
      "logps/chosen": -0.3744273781776428,
      "logps/rejected": -41.171321868896484,
      "loss": 0.5618,
      "odds_ratio_loss": 1.8740653991699219,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.03744274005293846,
      "rewards/margins": 4.0796895027160645,
      "rewards/rejected": -4.117132186889648,
      "sft_loss": 0.3744273781776428,
      "step": 1440
    },
    {
      "epoch": 2.304075235109718,
      "grad_norm": 1.1352714624377223,
      "learning_rate": 5.671867800613582e-06,
      "logits/chosen": -1.654802680015564,
      "logits/rejected": -1.7102715969085693,
      "logps/chosen": -0.37380197644233704,
      "logps/rejected": -40.510955810546875,
      "loss": 0.5609,
      "odds_ratio_loss": 1.871164321899414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03738019987940788,
      "rewards/margins": 4.0137152671813965,
      "rewards/rejected": -4.051095962524414,
      "sft_loss": 0.37380197644233704,
      "step": 1470
    },
    {
      "epoch": 2.3510971786833856,
      "grad_norm": 1.0544550616593722,
      "learning_rate": 5.524284970587997e-06,
      "logits/chosen": -1.6725318431854248,
      "logits/rejected": -1.6666339635849,
      "logps/chosen": -0.37714874744415283,
      "logps/rejected": -41.287841796875,
      "loss": 0.5661,
      "odds_ratio_loss": 1.8892247676849365,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": -0.0377148762345314,
      "rewards/margins": 4.09106969833374,
      "rewards/rejected": -4.128784656524658,
      "sft_loss": 0.37714874744415283,
      "step": 1500
    },
    {
      "epoch": 2.3981191222570533,
      "grad_norm": 1.0647146692947609,
      "learning_rate": 5.376238737235469e-06,
      "logits/chosen": -1.7326326370239258,
      "logits/rejected": -1.692516803741455,
      "logps/chosen": -0.37957924604415894,
      "logps/rejected": -40.64752197265625,
      "loss": 0.5696,
      "odds_ratio_loss": 1.8999431133270264,
      "rewards/accuracies": 0.9989583492279053,
      "rewards/chosen": -0.03795792534947395,
      "rewards/margins": 4.026793956756592,
      "rewards/rejected": -4.064752101898193,
      "sft_loss": 0.37957924604415894,
      "step": 1530
    },
    {
      "epoch": 2.445141065830721,
      "grad_norm": 1.0668568570109873,
      "learning_rate": 5.227859955188355e-06,
      "logits/chosen": -1.7023158073425293,
      "logits/rejected": -1.704237937927246,
      "logps/chosen": -0.37669476866722107,
      "logps/rejected": -41.38389587402344,
      "loss": 0.5652,
      "odds_ratio_loss": 1.8851966857910156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03766947612166405,
      "rewards/margins": 4.100719928741455,
      "rewards/rejected": -4.138389587402344,
      "sft_loss": 0.37669476866722107,
      "step": 1560
    },
    {
      "epoch": 2.492163009404389,
      "grad_norm": 1.048799823874522,
      "learning_rate": 5.079279773011095e-06,
      "logits/chosen": -1.5848591327667236,
      "logits/rejected": -1.663787603378296,
      "logps/chosen": -0.38333389163017273,
      "logps/rejected": -42.06602096557617,
      "loss": 0.5752,
      "odds_ratio_loss": 1.9189646244049072,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.03833338990807533,
      "rewards/margins": 4.168269157409668,
      "rewards/rejected": -4.206602096557617,
      "sft_loss": 0.38333389163017273,
      "step": 1590
    },
    {
      "epoch": 2.5391849529780566,
      "grad_norm": 1.0924219426439041,
      "learning_rate": 4.930629517281028e-06,
      "logits/chosen": -1.6451200246810913,
      "logits/rejected": -1.6708714962005615,
      "logps/chosen": -0.37655967473983765,
      "logps/rejected": -40.94800567626953,
      "loss": 0.5651,
      "odds_ratio_loss": 1.8852598667144775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03765597194433212,
      "rewards/margins": 4.057145118713379,
      "rewards/rejected": -4.09480094909668,
      "sft_loss": 0.37655967473983765,
      "step": 1620
    },
    {
      "epoch": 2.586206896551724,
      "grad_norm": 1.0780617647053894,
      "learning_rate": 4.782040576511881e-06,
      "logits/chosen": -1.6605018377304077,
      "logits/rejected": -1.7122681140899658,
      "logps/chosen": -0.37766554951667786,
      "logps/rejected": -41.40230178833008,
      "loss": 0.5668,
      "odds_ratio_loss": 1.8910880088806152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.037766553461551666,
      "rewards/margins": 4.102463722229004,
      "rewards/rejected": -4.140230655670166,
      "sft_loss": 0.37766554951667786,
      "step": 1650
    },
    {
      "epoch": 2.633228840125392,
      "grad_norm": 1.0665283784627981,
      "learning_rate": 4.633644285022514e-06,
      "logits/chosen": -1.4954478740692139,
      "logits/rejected": -1.6388036012649536,
      "logps/chosen": -0.37818458676338196,
      "logps/rejected": -41.43574905395508,
      "loss": 0.5676,
      "odds_ratio_loss": 1.8939425945281982,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.037818457931280136,
      "rewards/margins": 4.105756759643555,
      "rewards/rejected": -4.1435747146606445,
      "sft_loss": 0.37818458676338196,
      "step": 1680
    },
    {
      "epoch": 2.6802507836990594,
      "grad_norm": 1.117909376672666,
      "learning_rate": 4.485571806853553e-06,
      "logits/chosen": -1.4043054580688477,
      "logits/rejected": -1.6599198579788208,
      "logps/chosen": -0.3815458118915558,
      "logps/rejected": -42.49534606933594,
      "loss": 0.5726,
      "odds_ratio_loss": 1.9101753234863281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03815457969903946,
      "rewards/margins": 4.2113800048828125,
      "rewards/rejected": -4.249535083770752,
      "sft_loss": 0.3815458118915558,
      "step": 1710
    },
    {
      "epoch": 2.7272727272727275,
      "grad_norm": 0.9957664276621001,
      "learning_rate": 4.337954019834537e-06,
      "logits/chosen": -1.382237195968628,
      "logits/rejected": -1.6407064199447632,
      "logps/chosen": -0.37606796622276306,
      "logps/rejected": -41.618125915527344,
      "loss": 0.5642,
      "odds_ratio_loss": 1.8813918828964233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.037606798112392426,
      "rewards/margins": 4.124205589294434,
      "rewards/rejected": -4.161812782287598,
      "sft_loss": 0.37606796622276306,
      "step": 1740
    },
    {
      "epoch": 2.774294670846395,
      "grad_norm": 1.0842791833086312,
      "learning_rate": 4.190921399904048e-06,
      "logits/chosen": -1.4750745296478271,
      "logits/rejected": -1.6820265054702759,
      "logps/chosen": -0.37413859367370605,
      "logps/rejected": -41.810272216796875,
      "loss": 0.5615,
      "odds_ratio_loss": 1.873866319656372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.037413861602544785,
      "rewards/margins": 4.143613815307617,
      "rewards/rejected": -4.181027412414551,
      "sft_loss": 0.37413859367370605,
      "step": 1770
    },
    {
      "epoch": 2.8213166144200628,
      "grad_norm": 1.0015237419368779,
      "learning_rate": 4.044603905785044e-06,
      "logits/chosen": -1.3445394039154053,
      "logits/rejected": -1.6244174242019653,
      "logps/chosen": -0.3810182809829712,
      "logps/rejected": -42.303462982177734,
      "loss": 0.5717,
      "odds_ratio_loss": 1.9063372611999512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03810183331370354,
      "rewards/margins": 4.192244529724121,
      "rewards/rejected": -4.2303466796875,
      "sft_loss": 0.3810182809829712,
      "step": 1800
    },
    {
      "epoch": 2.8683385579937304,
      "grad_norm": 1.1765923790147168,
      "learning_rate": 3.899130864117366e-06,
      "logits/chosen": -1.3384199142456055,
      "logits/rejected": -1.626448154449463,
      "logps/chosen": -0.382070928812027,
      "logps/rejected": -42.68085479736328,
      "loss": 0.5733,
      "odds_ratio_loss": 1.9123166799545288,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.03820709139108658,
      "rewards/margins": 4.229878902435303,
      "rewards/rejected": -4.268085479736328,
      "sft_loss": 0.382070928812027,
      "step": 1830
    },
    {
      "epoch": 2.915360501567398,
      "grad_norm": 1.090679323980196,
      "learning_rate": 3.754630855148914e-06,
      "logits/chosen": -1.3685989379882812,
      "logits/rejected": -1.5960211753845215,
      "logps/chosen": -0.3773561120033264,
      "logps/rejected": -42.771400451660156,
      "loss": 0.5662,
      "odds_ratio_loss": 1.8884392976760864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03773561120033264,
      "rewards/margins": 4.239404678344727,
      "rewards/rejected": -4.277140140533447,
      "sft_loss": 0.3773561120033264,
      "step": 1860
    },
    {
      "epoch": 2.962382445141066,
      "grad_norm": 1.0970942641955401,
      "learning_rate": 3.6112315990865543e-06,
      "logits/chosen": -1.268080472946167,
      "logits/rejected": -1.5860413312911987,
      "logps/chosen": -0.3762770891189575,
      "logps/rejected": -42.43409729003906,
      "loss": 0.5647,
      "odds_ratio_loss": 1.8838173151016235,
      "rewards/accuracies": 0.9994791746139526,
      "rewards/chosen": -0.03762771189212799,
      "rewards/margins": 4.205781936645508,
      "rewards/rejected": -4.243409633636475,
      "sft_loss": 0.3762770891189575,
      "step": 1890
    },
    {
      "epoch": 3.0094043887147337,
      "grad_norm": 1.2095236484058132,
      "learning_rate": 3.4690598432071793e-06,
      "logits/chosen": -1.1365634202957153,
      "logits/rejected": -1.5553410053253174,
      "logps/chosen": -0.3466207981109619,
      "logps/rejected": -42.42127227783203,
      "loss": 0.5201,
      "odds_ratio_loss": 1.7351850271224976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03466207906603813,
      "rewards/margins": 4.207465171813965,
      "rewards/rejected": -4.242127418518066,
      "sft_loss": 0.3466207981109619,
      "step": 1920
    },
    {
      "epoch": 3.0564263322884013,
      "grad_norm": 1.024347525310305,
      "learning_rate": 3.328241249828747e-06,
      "logits/chosen": -1.4228293895721436,
      "logits/rejected": -1.614565134048462,
      "logps/chosen": -0.21424469351768494,
      "logps/rejected": -41.821380615234375,
      "loss": 0.3216,
      "odds_ratio_loss": 1.073145866394043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021424470469355583,
      "rewards/margins": 4.160713195800781,
      "rewards/rejected": -4.182137966156006,
      "sft_loss": 0.21424469351768494,
      "step": 1950
    },
    {
      "epoch": 3.103448275862069,
      "grad_norm": 0.9863432232044451,
      "learning_rate": 3.1889002852402505e-06,
      "logits/chosen": -1.3458492755889893,
      "logits/rejected": -1.5922932624816895,
      "logps/chosen": -0.21549250185489655,
      "logps/rejected": -42.468841552734375,
      "loss": 0.3234,
      "odds_ratio_loss": 1.078763723373413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021549250930547714,
      "rewards/margins": 4.225335121154785,
      "rewards/rejected": -4.246884346008301,
      "sft_loss": 0.21549250185489655,
      "step": 1980
    }
  ],
  "log_save_evaluate_time": 457.3154537677765,
  "logging_steps": 30,
  "max_steps": 3190,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 5,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 680295014072320.0,
  "total_tokens": 0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}