{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.26171159382360637,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005234231876472127,
      "grad_norm": 132.6717987060547,
      "learning_rate": 0.0,
      "logits/chosen": -0.40118408203125,
      "logits/rejected": -0.41802978515625,
      "logps/chosen": -297.609375,
      "logps/rejected": -247.84375,
      "logps/weighted_chosen": -4.7568359375,
      "logps/weighted_rejected": -3.47998046875,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "rewards/weighted_accuracies": 0.0,
      "rewards/weighted_chosen": 0.0,
      "rewards/weighted_margins": 0.0,
      "rewards/weighted_rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005234231876472127,
      "grad_norm": 226.00839233398438,
      "learning_rate": 4.6875e-08,
      "logits/chosen": -0.3175845742225647,
      "logits/rejected": -0.3532341718673706,
      "logps/chosen": -275.5841979980469,
      "logps/rejected": -255.84548950195312,
      "logps/weighted_chosen": -2.651665687561035,
      "logps/weighted_rejected": -2.88427734375,
      "loss": 0.6921,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0731336772441864,
      "rewards/margins": -0.0670572891831398,
      "rewards/rejected": -0.006076388992369175,
      "rewards/weighted_accuracies": 0.2951388955116272,
      "rewards/weighted_chosen": -0.0015835232334211469,
      "rewards/weighted_margins": -0.0009324815473519266,
      "rewards/weighted_rejected": -0.0006510416860692203,
      "step": 10
    },
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 14.726158142089844,
      "learning_rate": 9.895833333333332e-08,
      "logits/chosen": -0.29781341552734375,
      "logits/rejected": -0.3136836886405945,
      "logps/chosen": -294.4296875,
      "logps/rejected": -272.58984375,
      "logps/weighted_chosen": -2.458728075027466,
      "logps/weighted_rejected": -2.455883741378784,
      "loss": 0.6924,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.14013671875,
      "rewards/margins": -0.13369140028953552,
      "rewards/rejected": -0.0064453124068677425,
      "rewards/weighted_accuracies": 0.34062498807907104,
      "rewards/weighted_chosen": -0.0013603210682049394,
      "rewards/weighted_margins": -0.0010925292735919356,
      "rewards/weighted_rejected": -0.0002677917364053428,
      "step": 20
    },
    {
      "epoch": 0.015702695629416383,
      "grad_norm": 77.6125259399414,
      "learning_rate": 1.5104166666666664e-07,
      "logits/chosen": -0.2918853759765625,
      "logits/rejected": -0.3377639651298523,
      "logps/chosen": -298.05859375,
      "logps/rejected": -268.0132751464844,
      "logps/weighted_chosen": -2.4350829124450684,
      "logps/weighted_rejected": -2.7343993186950684,
      "loss": 0.6926,
      "rewards/accuracies": 0.24062499403953552,
      "rewards/chosen": -0.06621094048023224,
      "rewards/margins": -0.1640625,
      "rewards/rejected": 0.09785155951976776,
      "rewards/weighted_accuracies": 0.3187499940395355,
      "rewards/weighted_chosen": 0.0014068603049963713,
      "rewards/weighted_margins": -0.0015777588123455644,
      "rewards/weighted_rejected": 0.0029846192337572575,
      "step": 30
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 30.666196823120117,
      "learning_rate": 2.03125e-07,
      "logits/chosen": -0.30072021484375,
      "logits/rejected": -0.3433845639228821,
      "logps/chosen": -278.68829345703125,
      "logps/rejected": -253.90780639648438,
      "logps/weighted_chosen": -2.506396532058716,
      "logps/weighted_rejected": -2.8416504859924316,
      "loss": 0.6908,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 0.063232421875,
      "rewards/margins": 0.04838867112994194,
      "rewards/rejected": 0.014843749813735485,
      "rewards/weighted_accuracies": 0.40312498807907104,
      "rewards/weighted_chosen": 0.0042968750931322575,
      "rewards/weighted_margins": 0.0019538879860192537,
      "rewards/weighted_rejected": 0.0023429871071130037,
      "step": 40
    },
    {
      "epoch": 0.02617115938236064,
      "grad_norm": 18.60569953918457,
      "learning_rate": 2.552083333333333e-07,
      "logits/chosen": -0.2819870114326477,
      "logits/rejected": -0.32059136033058167,
      "logps/chosen": -280.31951904296875,
      "logps/rejected": -267.4359436035156,
      "logps/weighted_chosen": -2.4267334938049316,
      "logps/weighted_rejected": -2.529711961746216,
      "loss": 0.6891,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.03535156324505806,
      "rewards/margins": -0.13984374701976776,
      "rewards/rejected": 0.1044921875,
      "rewards/weighted_accuracies": 0.3968749940395355,
      "rewards/weighted_chosen": 0.0039031982887536287,
      "rewards/weighted_margins": 0.005755615420639515,
      "rewards/weighted_rejected": -0.0018524170154705644,
      "step": 50
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 38.21036911010742,
      "learning_rate": 3.0729166666666665e-07,
      "logits/chosen": -0.31453245878219604,
      "logits/rejected": -0.30809077620506287,
      "logps/chosen": -277.66015625,
      "logps/rejected": -261.7445373535156,
      "logps/weighted_chosen": -2.8622069358825684,
      "logps/weighted_rejected": -2.7553467750549316,
      "loss": 0.6894,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": 0.04150390625,
      "rewards/margins": 0.08027343451976776,
      "rewards/rejected": -0.03876953199505806,
      "rewards/weighted_accuracies": 0.4312500059604645,
      "rewards/weighted_chosen": 0.0006561279296875,
      "rewards/weighted_margins": 0.006243896670639515,
      "rewards/weighted_rejected": -0.005587768740952015,
      "step": 60
    },
    {
      "epoch": 0.036639623135304895,
      "grad_norm": 69.19047546386719,
      "learning_rate": 3.59375e-07,
      "logits/chosen": -0.3177490234375,
      "logits/rejected": -0.3246749937534332,
      "logps/chosen": -289.76251220703125,
      "logps/rejected": -244.92578125,
      "logps/weighted_chosen": -2.3438963890075684,
      "logps/weighted_rejected": -2.7010498046875,
      "loss": 0.6841,
      "rewards/accuracies": 0.49687498807907104,
      "rewards/chosen": 0.29765623807907104,
      "rewards/margins": 0.4546875059604645,
      "rewards/rejected": -0.15703125298023224,
      "rewards/weighted_accuracies": 0.5406249761581421,
      "rewards/weighted_chosen": 0.01530532818287611,
      "rewards/weighted_margins": 0.01918792724609375,
      "rewards/weighted_rejected": -0.0038825988303869963,
      "step": 70
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 51.98476791381836,
      "learning_rate": 4.114583333333333e-07,
      "logits/chosen": -0.2850998044013977,
      "logits/rejected": -0.30662041902542114,
      "logps/chosen": -289.234375,
      "logps/rejected": -270.375,
      "logps/weighted_chosen": -2.5325684547424316,
      "logps/weighted_rejected": -2.796435594558716,
      "loss": 0.6747,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.512499988079071,
      "rewards/margins": 0.6001952886581421,
      "rewards/rejected": -0.08769531548023224,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": 0.036380767822265625,
      "rewards/weighted_margins": 0.04396667331457138,
      "rewards/weighted_rejected": -0.007586670108139515,
      "step": 80
    },
    {
      "epoch": 0.04710808688824915,
      "grad_norm": 30.52783203125,
      "learning_rate": 4.6354166666666664e-07,
      "logits/chosen": -0.3142959475517273,
      "logits/rejected": -0.3075408935546875,
      "logps/chosen": -280.11407470703125,
      "logps/rejected": -257.95233154296875,
      "logps/weighted_chosen": -2.719482421875,
      "logps/weighted_rejected": -2.88037109375,
      "loss": 0.6687,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.5205078125,
      "rewards/margins": 0.737109363079071,
      "rewards/rejected": -0.21660156548023224,
      "rewards/weighted_accuracies": 0.621874988079071,
      "rewards/weighted_chosen": 0.06780395656824112,
      "rewards/weighted_margins": 0.07340697944164276,
      "rewards/weighted_rejected": -0.0056396485306322575,
      "step": 90
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 69.397705078125,
      "learning_rate": 5.156249999999999e-07,
      "logits/chosen": -0.28213196992874146,
      "logits/rejected": -0.3543289303779602,
      "logps/chosen": -290.71875,
      "logps/rejected": -286.73126220703125,
      "logps/weighted_chosen": -2.2228636741638184,
      "logps/weighted_rejected": -2.8367552757263184,
      "loss": 0.6848,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": 0.24521484971046448,
      "rewards/margins": 1.0690429210662842,
      "rewards/rejected": -0.8238281011581421,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": 0.05242309719324112,
      "rewards/weighted_margins": 0.05032653734087944,
      "rewards/weighted_rejected": 0.0021240233909338713,
      "step": 100
    },
    {
      "epoch": 0.05757655064119341,
      "grad_norm": 36.600040435791016,
      "learning_rate": 5.677083333333333e-07,
      "logits/chosen": -0.33063429594039917,
      "logits/rejected": -0.319937139749527,
      "logps/chosen": -296.82501220703125,
      "logps/rejected": -262.2984313964844,
      "logps/weighted_chosen": -2.8468017578125,
      "logps/weighted_rejected": -2.9306397438049316,
      "loss": 0.6773,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.474609375,
      "rewards/margins": 1.053613305091858,
      "rewards/rejected": -1.528222680091858,
      "rewards/weighted_accuracies": 0.534375011920929,
      "rewards/weighted_chosen": 0.013439941219985485,
      "rewards/weighted_margins": 0.05541381984949112,
      "rewards/weighted_rejected": -0.04198913648724556,
      "step": 110
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 57.109580993652344,
      "learning_rate": 6.197916666666666e-07,
      "logits/chosen": -0.33633461594581604,
      "logits/rejected": -0.36155110597610474,
      "logps/chosen": -295.3687438964844,
      "logps/rejected": -256.1953125,
      "logps/weighted_chosen": -2.161865234375,
      "logps/weighted_rejected": -2.4251465797424316,
      "loss": 0.6791,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -0.72900390625,
      "rewards/margins": 1.641210913658142,
      "rewards/rejected": -2.3702149391174316,
      "rewards/weighted_accuracies": 0.5562499761581421,
      "rewards/weighted_chosen": 0.007176590152084827,
      "rewards/weighted_margins": 0.05286560207605362,
      "rewards/weighted_rejected": -0.04570160061120987,
      "step": 120
    },
    {
      "epoch": 0.06804501439413765,
      "grad_norm": 39.176841735839844,
      "learning_rate": 6.718749999999999e-07,
      "logits/chosen": -0.29625242948532104,
      "logits/rejected": -0.2914108335971832,
      "logps/chosen": -306.6781311035156,
      "logps/rejected": -280.15936279296875,
      "logps/weighted_chosen": -2.188079833984375,
      "logps/weighted_rejected": -2.5787596702575684,
      "loss": 0.6659,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.644238293170929,
      "rewards/margins": 1.972265601158142,
      "rewards/rejected": -2.616406202316284,
      "rewards/weighted_accuracies": 0.606249988079071,
      "rewards/weighted_chosen": 0.01349639892578125,
      "rewards/weighted_margins": 0.0841522216796875,
      "rewards/weighted_rejected": -0.07064209133386612,
      "step": 130
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 52.14993667602539,
      "learning_rate": 7.239583333333333e-07,
      "logits/chosen": -0.3304199278354645,
      "logits/rejected": -0.3464847505092621,
      "logps/chosen": -301.4390563964844,
      "logps/rejected": -277.9515686035156,
      "logps/weighted_chosen": -2.554003953933716,
      "logps/weighted_rejected": -2.881591796875,
      "loss": 0.6581,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -2.746875047683716,
      "rewards/margins": 2.744921922683716,
      "rewards/rejected": -5.491991996765137,
      "rewards/weighted_accuracies": 0.6312500238418579,
      "rewards/weighted_chosen": -0.02762756310403347,
      "rewards/weighted_margins": 0.11510010063648224,
      "rewards/weighted_rejected": -0.14276733994483948,
      "step": 140
    },
    {
      "epoch": 0.07851347814708191,
      "grad_norm": 22.611814498901367,
      "learning_rate": 7.760416666666666e-07,
      "logits/chosen": -0.2870376706123352,
      "logits/rejected": -0.2975311279296875,
      "logps/chosen": -287.859375,
      "logps/rejected": -257.54296875,
      "logps/weighted_chosen": -3.089892625808716,
      "logps/weighted_rejected": -3.1946043968200684,
      "loss": 0.6544,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.7095704078674316,
      "rewards/margins": 2.942578077316284,
      "rewards/rejected": -6.652148246765137,
      "rewards/weighted_accuracies": 0.6312500238418579,
      "rewards/weighted_chosen": -0.005145263858139515,
      "rewards/weighted_margins": 0.16416625678539276,
      "rewards/weighted_rejected": -0.16951599717140198,
      "step": 150
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 15.511767387390137,
      "learning_rate": 8.28125e-07,
      "logits/chosen": -0.3232177793979645,
      "logits/rejected": -0.3726806640625,
      "logps/chosen": -308.91796875,
      "logps/rejected": -282.15704345703125,
      "logps/weighted_chosen": -2.5903563499450684,
      "logps/weighted_rejected": -2.742602586746216,
      "loss": 0.6211,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -4.956835746765137,
      "rewards/margins": 3.9961915016174316,
      "rewards/rejected": -8.953222274780273,
      "rewards/weighted_accuracies": 0.6499999761581421,
      "rewards/weighted_chosen": -0.002410888671875,
      "rewards/weighted_margins": 0.23797607421875,
      "rewards/weighted_rejected": -0.24028320610523224,
      "step": 160
    },
    {
      "epoch": 0.08898194190002617,
      "grad_norm": 167.33956909179688,
      "learning_rate": 8.802083333333333e-07,
      "logits/chosen": -0.36021536588668823,
      "logits/rejected": -0.3597045838832855,
      "logps/chosen": -311.03045654296875,
      "logps/rejected": -270.46875,
      "logps/weighted_chosen": -2.8318848609924316,
      "logps/weighted_rejected": -3.139453172683716,
      "loss": 0.6949,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -8.0087890625,
      "rewards/margins": 4.345898628234863,
      "rewards/rejected": -12.354199409484863,
      "rewards/weighted_accuracies": 0.640625,
      "rewards/weighted_chosen": -0.01859130896627903,
      "rewards/weighted_margins": 0.20853272080421448,
      "rewards/weighted_rejected": -0.22731323540210724,
      "step": 170
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 64.57138061523438,
      "learning_rate": 9.322916666666666e-07,
      "logits/chosen": -0.33618468046188354,
      "logits/rejected": -0.3534431457519531,
      "logps/chosen": -284.2171936035156,
      "logps/rejected": -272.12969970703125,
      "logps/weighted_chosen": -2.694580078125,
      "logps/weighted_rejected": -3.225878953933716,
      "loss": 0.6814,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -10.43701171875,
      "rewards/margins": 5.353320121765137,
      "rewards/rejected": -15.7919921875,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.08297424018383026,
      "rewards/weighted_margins": 0.26459962129592896,
      "rewards/weighted_rejected": -0.347381591796875,
      "step": 180
    },
    {
      "epoch": 0.09945040565297043,
      "grad_norm": 49.0852165222168,
      "learning_rate": 9.84375e-07,
      "logits/chosen": -0.354086309671402,
      "logits/rejected": -0.38891831040382385,
      "logps/chosen": -319.17498779296875,
      "logps/rejected": -283.31561279296875,
      "logps/weighted_chosen": -2.5078492164611816,
      "logps/weighted_rejected": -3.016357421875,
      "loss": 0.6496,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -13.561426162719727,
      "rewards/margins": 5.937890529632568,
      "rewards/rejected": -19.498828887939453,
      "rewards/weighted_accuracies": 0.6343749761581421,
      "rewards/weighted_chosen": -0.16942748427391052,
      "rewards/weighted_margins": 0.24410399794578552,
      "rewards/weighted_rejected": -0.41356199979782104,
      "step": 190
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 53.46296691894531,
      "learning_rate": 9.99959085414323e-07,
      "logits/chosen": -0.37868577241897583,
      "logits/rejected": -0.4114578366279602,
      "logps/chosen": -324.7124938964844,
      "logps/rejected": -279.72967529296875,
      "logps/weighted_chosen": -2.8757567405700684,
      "logps/weighted_rejected": -3.3623046875,
      "loss": 0.639,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -15.428125381469727,
      "rewards/margins": 6.552148342132568,
      "rewards/rejected": -21.975000381469727,
      "rewards/weighted_accuracies": 0.6343749761581421,
      "rewards/weighted_chosen": -0.18135985732078552,
      "rewards/weighted_margins": 0.29008787870407104,
      "rewards/weighted_rejected": -0.471527099609375,
      "step": 200
    },
    {
      "epoch": 0.10991886940591468,
      "grad_norm": 24.815481185913086,
      "learning_rate": 9.997587035630105e-07,
      "logits/chosen": -0.3853309750556946,
      "logits/rejected": -0.4257049560546875,
      "logps/chosen": -302.82891845703125,
      "logps/rejected": -308.671875,
      "logps/weighted_chosen": -2.632519483566284,
      "logps/weighted_rejected": -3.3669190406799316,
      "loss": 0.6558,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -15.814453125,
      "rewards/margins": 8.331445693969727,
      "rewards/rejected": -24.146093368530273,
      "rewards/weighted_accuracies": 0.659375011920929,
      "rewards/weighted_chosen": -0.225901797413826,
      "rewards/weighted_margins": 0.23236694931983948,
      "rewards/weighted_rejected": -0.45829468965530396,
      "step": 210
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 24.175745010375977,
      "learning_rate": 9.99391406364405e-07,
      "logits/chosen": -0.37365952134132385,
      "logits/rejected": -0.3758789002895355,
      "logps/chosen": -309.34686279296875,
      "logps/rejected": -293.98126220703125,
      "logps/weighted_chosen": -3.002514600753784,
      "logps/weighted_rejected": -3.453906297683716,
      "loss": 0.6732,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -17.203418731689453,
      "rewards/margins": 7.933203220367432,
      "rewards/rejected": -25.137109756469727,
      "rewards/weighted_accuracies": 0.6156250238418579,
      "rewards/weighted_chosen": -0.259225457906723,
      "rewards/weighted_margins": 0.29540252685546875,
      "rewards/weighted_rejected": -0.5546798706054688,
      "step": 220
    },
    {
      "epoch": 0.12038733315885894,
      "grad_norm": 85.15988159179688,
      "learning_rate": 9.988573164927884e-07,
      "logits/chosen": -0.3097473084926605,
      "logits/rejected": -0.3477935791015625,
      "logps/chosen": -286.5078125,
      "logps/rejected": -281.8453063964844,
      "logps/weighted_chosen": -2.66943359375,
      "logps/weighted_rejected": -3.1229491233825684,
      "loss": 0.6646,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -17.690723419189453,
      "rewards/margins": 12.424609184265137,
      "rewards/rejected": -30.110157012939453,
      "rewards/weighted_accuracies": 0.65625,
      "rewards/weighted_chosen": -0.2716217041015625,
      "rewards/weighted_margins": 0.32661741971969604,
      "rewards/weighted_rejected": -0.5983597040176392,
      "step": 230
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 26.17377471923828,
      "learning_rate": 9.98156612329838e-07,
      "logits/chosen": -0.39516907930374146,
      "logits/rejected": -0.44511109590530396,
      "logps/chosen": -286.74884033203125,
      "logps/rejected": -318.22735595703125,
      "logps/weighted_chosen": -2.6696534156799316,
      "logps/weighted_rejected": -3.4151854515075684,
      "loss": 0.643,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -21.990428924560547,
      "rewards/margins": 14.028905868530273,
      "rewards/rejected": -36.013282775878906,
      "rewards/weighted_accuracies": 0.6625000238418579,
      "rewards/weighted_chosen": -0.2329559326171875,
      "rewards/weighted_margins": 0.3950134217739105,
      "rewards/weighted_rejected": -0.6281493902206421,
      "step": 240
    },
    {
      "epoch": 0.13085579691180318,
      "grad_norm": 56.73057174682617,
      "learning_rate": 9.97289527905053e-07,
      "logits/chosen": -0.40631332993507385,
      "logits/rejected": -0.4203124940395355,
      "logps/chosen": -290.1703186035156,
      "logps/rejected": -291.6328125,
      "logps/weighted_chosen": -3.051513671875,
      "logps/weighted_rejected": -3.3163819313049316,
      "loss": 0.677,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -25.742870330810547,
      "rewards/margins": 9.973828315734863,
      "rewards/rejected": -35.72148513793945,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.2856689393520355,
      "rewards/weighted_margins": 0.253326416015625,
      "rewards/weighted_rejected": -0.5388733148574829,
      "step": 250
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 17.766258239746094,
      "learning_rate": 9.962563528175875e-07,
      "logits/chosen": -0.3611465394496918,
      "logits/rejected": -0.39628905057907104,
      "logps/chosen": -324.36639404296875,
      "logps/rejected": -297.765625,
      "logps/weighted_chosen": -2.652392625808716,
      "logps/weighted_rejected": -3.535571336746216,
      "loss": 0.6414,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -25.621288299560547,
      "rewards/margins": 11.306055068969727,
      "rewards/rejected": -36.93359375,
      "rewards/weighted_accuracies": 0.637499988079071,
      "rewards/weighted_chosen": -0.2533508241176605,
      "rewards/weighted_margins": 0.2956604063510895,
      "rewards/weighted_rejected": -0.5490142703056335,
      "step": 260
    },
    {
      "epoch": 0.14132426066474746,
      "grad_norm": 17.552453994750977,
      "learning_rate": 9.950574321395277e-07,
      "logits/chosen": -0.41735154390335083,
      "logits/rejected": -0.441476434469223,
      "logps/chosen": -314.5093688964844,
      "logps/rejected": -295.7093811035156,
      "logps/weighted_chosen": -2.864941358566284,
      "logps/weighted_rejected": -3.25732421875,
      "loss": 0.661,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -25.1123046875,
      "rewards/margins": 7.519726753234863,
      "rewards/rejected": -32.62890625,
      "rewards/weighted_accuracies": 0.609375,
      "rewards/weighted_chosen": -0.3035888671875,
      "rewards/weighted_margins": 0.2833190858364105,
      "rewards/weighted_rejected": -0.5868393182754517,
      "step": 270
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 47.66518020629883,
      "learning_rate": 9.936931663006413e-07,
      "logits/chosen": -0.4760284423828125,
      "logits/rejected": -0.46795654296875,
      "logps/chosen": -323.48126220703125,
      "logps/rejected": -313.2875061035156,
      "logps/weighted_chosen": -2.794970750808716,
      "logps/weighted_rejected": -3.3581910133361816,
      "loss": 0.6169,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -20.707616806030273,
      "rewards/margins": 13.166601181030273,
      "rewards/rejected": -33.86640548706055,
      "rewards/weighted_accuracies": 0.7093750238418579,
      "rewards/weighted_chosen": -0.10174255073070526,
      "rewards/weighted_margins": 0.34544676542282104,
      "rewards/weighted_rejected": -0.447021484375,
      "step": 280
    },
    {
      "epoch": 0.1517927244176917,
      "grad_norm": 32.503883361816406,
      "learning_rate": 9.921640109546357e-07,
      "logits/chosen": -0.44742050766944885,
      "logits/rejected": -0.5166229009628296,
      "logps/chosen": -292.1796875,
      "logps/rejected": -289.6234436035156,
      "logps/weighted_chosen": -2.7469239234924316,
      "logps/weighted_rejected": -3.9541258811950684,
      "loss": 0.6249,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -25.293359756469727,
      "rewards/margins": 12.698633193969727,
      "rewards/rejected": -37.994529724121094,
      "rewards/weighted_accuracies": 0.628125011920929,
      "rewards/weighted_chosen": -0.15215758979320526,
      "rewards/weighted_margins": 0.4393859803676605,
      "rewards/weighted_rejected": -0.5915588140487671,
      "step": 290
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 17.32170867919922,
      "learning_rate": 9.90470476826975e-07,
      "logits/chosen": -0.5146636962890625,
      "logits/rejected": -0.515917956829071,
      "logps/chosen": -302.3570251464844,
      "logps/rejected": -313.68438720703125,
      "logps/weighted_chosen": -2.6830201148986816,
      "logps/weighted_rejected": -3.202099561691284,
      "loss": 0.6526,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": -32.978126525878906,
      "rewards/margins": 13.435937881469727,
      "rewards/rejected": -46.408203125,
      "rewards/weighted_accuracies": 0.621874988079071,
      "rewards/weighted_chosen": -0.23505249619483948,
      "rewards/weighted_margins": 0.33623045682907104,
      "rewards/weighted_rejected": -0.5710296630859375,
      "step": 300
    },
    {
      "epoch": 0.16226118817063595,
      "grad_norm": 25.855854034423828,
      "learning_rate": 9.886131295443002e-07,
      "logits/chosen": -0.6332122683525085,
      "logits/rejected": -0.6879852414131165,
      "logps/chosen": -315.02264404296875,
      "logps/rejected": -296.54998779296875,
      "logps/weighted_chosen": -2.8891844749450684,
      "logps/weighted_rejected": -3.3497071266174316,
      "loss": 0.6099,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": -33.83808517456055,
      "rewards/margins": 12.542577743530273,
      "rewards/rejected": -46.39081954956055,
      "rewards/weighted_accuracies": 0.668749988079071,
      "rewards/weighted_chosen": -0.20775146782398224,
      "rewards/weighted_margins": 0.507794201374054,
      "rewards/weighted_rejected": -0.715728759765625,
      "step": 310
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 19.11484718322754,
      "learning_rate": 9.865925894455166e-07,
      "logits/chosen": -0.730267345905304,
      "logits/rejected": -0.746167004108429,
      "logps/chosen": -338.2242126464844,
      "logps/rejected": -307.18280029296875,
      "logps/weighted_chosen": -2.9883790016174316,
      "logps/weighted_rejected": -3.5892090797424316,
      "loss": 0.6942,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -38.233009338378906,
      "rewards/margins": 11.564062118530273,
      "rewards/rejected": -49.80937576293945,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.3507080078125,
      "rewards/weighted_margins": 0.3366760313510895,
      "rewards/weighted_rejected": -0.6871337890625,
      "step": 320
    },
    {
      "epoch": 0.17272965192358022,
      "grad_norm": 57.19697570800781,
      "learning_rate": 9.84409531374603e-07,
      "logits/chosen": -0.6843910217285156,
      "logits/rejected": -0.6659576296806335,
      "logps/chosen": -345.46875,
      "logps/rejected": -316.2515563964844,
      "logps/weighted_chosen": -3.05517578125,
      "logps/weighted_rejected": -3.5519776344299316,
      "loss": 0.6569,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -40.732032775878906,
      "rewards/margins": 12.651952743530273,
      "rewards/rejected": -53.38984298706055,
      "rewards/weighted_accuracies": 0.6468750238418579,
      "rewards/weighted_chosen": -0.3262878358364105,
      "rewards/weighted_margins": 0.346893310546875,
      "rewards/weighted_rejected": -0.6730865240097046,
      "step": 330
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 52.49288558959961,
      "learning_rate": 9.820646844552219e-07,
      "logits/chosen": -0.6993133425712585,
      "logits/rejected": -0.7529846429824829,
      "logps/chosen": -313.59295654296875,
      "logps/rejected": -322.1499938964844,
      "logps/weighted_chosen": -3.0488524436950684,
      "logps/weighted_rejected": -3.440136671066284,
      "loss": 0.6287,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -37.06660079956055,
      "rewards/margins": 19.494531631469727,
      "rewards/rejected": -56.556640625,
      "rewards/weighted_accuracies": 0.6968749761581421,
      "rewards/weighted_chosen": -0.30719298124313354,
      "rewards/weighted_margins": 0.448944091796875,
      "rewards/weighted_rejected": -0.755999743938446,
      "step": 340
    },
    {
      "epoch": 0.18319811567652447,
      "grad_norm": 15.657389640808105,
      "learning_rate": 9.795588318471964e-07,
      "logits/chosen": -0.7813507318496704,
      "logits/rejected": -0.7874206304550171,
      "logps/chosen": -299.80157470703125,
      "logps/rejected": -331.4375,
      "logps/weighted_chosen": -2.84619140625,
      "logps/weighted_rejected": -3.315380811691284,
      "loss": 0.6405,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.32304763793945,
      "rewards/margins": 14.830663681030273,
      "rewards/rejected": -55.15625,
      "rewards/weighted_accuracies": 0.6468750238418579,
      "rewards/weighted_chosen": -0.315826416015625,
      "rewards/weighted_margins": 0.386627197265625,
      "rewards/weighted_rejected": -0.702471911907196,
      "step": 350
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 16.19976806640625,
      "learning_rate": 9.768928104849415e-07,
      "logits/chosen": -0.801177978515625,
      "logits/rejected": -0.799664318561554,
      "logps/chosen": -323.5171813964844,
      "logps/rejected": -305.046875,
      "logps/weighted_chosen": -3.1164307594299316,
      "logps/weighted_rejected": -3.3475098609924316,
      "loss": 0.6865,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -40.823829650878906,
      "rewards/margins": 15.389843940734863,
      "rewards/rejected": -56.216407775878906,
      "rewards/weighted_accuracies": 0.659375011920929,
      "rewards/weighted_chosen": -0.31828004121780396,
      "rewards/weighted_margins": 0.3831420838832855,
      "rewards/weighted_rejected": -0.7014526128768921,
      "step": 360
    },
    {
      "epoch": 0.19366657942946872,
      "grad_norm": 89.87427520751953,
      "learning_rate": 9.740675107979355e-07,
      "logits/chosen": -0.7640800476074219,
      "logits/rejected": -0.7867538332939148,
      "logps/chosen": -361.13751220703125,
      "logps/rejected": -334.97967529296875,
      "logps/weighted_chosen": -2.5084471702575684,
      "logps/weighted_rejected": -3.4689698219299316,
      "loss": 0.6531,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -46.098045349121094,
      "rewards/margins": 14.188085556030273,
      "rewards/rejected": -60.26640701293945,
      "rewards/weighted_accuracies": 0.671875,
      "rewards/weighted_chosen": -0.36387938261032104,
      "rewards/weighted_margins": 0.3567260801792145,
      "rewards/weighted_rejected": -0.720538318157196,
      "step": 370
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 22.484216690063477,
      "learning_rate": 9.71083876413323e-07,
      "logits/chosen": -0.7209137082099915,
      "logits/rejected": -0.7318176031112671,
      "logps/chosen": -353.6031188964844,
      "logps/rejected": -339.16485595703125,
      "logps/weighted_chosen": -2.70361328125,
      "logps/weighted_rejected": -3.5843749046325684,
      "loss": 0.6589,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -51.237892150878906,
      "rewards/margins": 18.424999237060547,
      "rewards/rejected": -69.64530944824219,
      "rewards/weighted_accuracies": 0.6312500238418579,
      "rewards/weighted_chosen": -0.4475158751010895,
      "rewards/weighted_margins": 0.3267761170864105,
      "rewards/weighted_rejected": -0.7747405767440796,
      "step": 380
    },
    {
      "epoch": 0.204135043182413,
      "grad_norm": 21.885372161865234,
      "learning_rate": 9.67942903840751e-07,
      "logits/chosen": -0.7708206176757812,
      "logits/rejected": -0.8207153081893921,
      "logps/chosen": -355.18438720703125,
      "logps/rejected": -350.47186279296875,
      "logps/weighted_chosen": -2.8836669921875,
      "logps/weighted_rejected": -3.5904297828674316,
      "loss": 0.6028,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -50.973045349121094,
      "rewards/margins": 25.190038681030273,
      "rewards/rejected": -76.1617202758789,
      "rewards/weighted_accuracies": 0.7093750238418579,
      "rewards/weighted_chosen": -0.39097899198532104,
      "rewards/weighted_margins": 0.4941650331020355,
      "rewards/weighted_rejected": -0.884967029094696,
      "step": 390
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 26.357742309570312,
      "learning_rate": 9.646456421395447e-07,
      "logits/chosen": -0.805267333984375,
      "logits/rejected": -0.8178039789199829,
      "logps/chosen": -377.52813720703125,
      "logps/rejected": -392.0296936035156,
      "logps/weighted_chosen": -2.7947998046875,
      "logps/weighted_rejected": -3.697582960128784,
      "loss": 0.6296,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -59.2109375,
      "rewards/margins": 29.121875762939453,
      "rewards/rejected": -88.32890319824219,
      "rewards/weighted_accuracies": 0.653124988079071,
      "rewards/weighted_chosen": -0.41761475801467896,
      "rewards/weighted_margins": 0.38171082735061646,
      "rewards/weighted_rejected": -0.7994705438613892,
      "step": 400
    },
    {
      "epoch": 0.21460350693535724,
      "grad_norm": 21.382999420166016,
      "learning_rate": 9.611931925683266e-07,
      "logits/chosen": -0.7703964114189148,
      "logits/rejected": -0.808850109577179,
      "logps/chosen": -367.3140563964844,
      "logps/rejected": -348.0687561035156,
      "logps/weighted_chosen": -2.711962938308716,
      "logps/weighted_rejected": -3.4615721702575684,
      "loss": 0.5758,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -61.02734375,
      "rewards/margins": 23.316797256469727,
      "rewards/rejected": -84.34687805175781,
      "rewards/weighted_accuracies": 0.6937500238418579,
      "rewards/weighted_chosen": -0.40519410371780396,
      "rewards/weighted_margins": 0.521441638469696,
      "rewards/weighted_rejected": -0.9261535406112671,
      "step": 410
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 23.030996322631836,
      "learning_rate": 9.575867082172085e-07,
      "logits/chosen": -0.7789466977119446,
      "logits/rejected": -0.8260132074356079,
      "logps/chosen": -372.22344970703125,
      "logps/rejected": -367.0171813964844,
      "logps/weighted_chosen": -3.114550828933716,
      "logps/weighted_rejected": -3.364208936691284,
      "loss": 0.6211,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -67.16816711425781,
      "rewards/margins": 29.731639862060547,
      "rewards/rejected": -96.90156555175781,
      "rewards/weighted_accuracies": 0.659375011920929,
      "rewards/weighted_chosen": -0.5122925043106079,
      "rewards/weighted_margins": 0.5218566656112671,
      "rewards/weighted_rejected": -1.0339782238006592,
      "step": 420
    },
    {
      "epoch": 0.22507197068830148,
      "grad_norm": 16.442333221435547,
      "learning_rate": 9.538273936226673e-07,
      "logits/chosen": -0.830523669719696,
      "logits/rejected": -0.8667358160018921,
      "logps/chosen": -328.4546813964844,
      "logps/rejected": -347.9593811035156,
      "logps/weighted_chosen": -3.373584032058716,
      "logps/weighted_rejected": -3.832958936691284,
      "loss": 0.6425,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -62.920310974121094,
      "rewards/margins": 20.668750762939453,
      "rewards/rejected": -83.5894546508789,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.459890753030777,
      "rewards/weighted_margins": 0.39284056425094604,
      "rewards/weighted_rejected": -0.8525451421737671,
      "step": 430
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 21.955875396728516,
      "learning_rate": 9.499165043652391e-07,
      "logits/chosen": -0.8598114252090454,
      "logits/rejected": -0.868182361125946,
      "logps/chosen": -358.21563720703125,
      "logps/rejected": -356.26251220703125,
      "logps/weighted_chosen": -3.4171142578125,
      "logps/weighted_rejected": -3.6997313499450684,
      "loss": 0.624,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -66.96504211425781,
      "rewards/margins": 22.563282012939453,
      "rewards/rejected": -89.5503921508789,
      "rewards/weighted_accuracies": 0.6625000238418579,
      "rewards/weighted_chosen": -0.5841079950332642,
      "rewards/weighted_margins": 0.434326171875,
      "rewards/weighted_rejected": -1.018707275390625,
      "step": 440
    },
    {
      "epoch": 0.23554043444124576,
      "grad_norm": 75.56902313232422,
      "learning_rate": 9.458553466501665e-07,
      "logits/chosen": -0.9330536127090454,
      "logits/rejected": -0.9642333984375,
      "logps/chosen": -352.6187438964844,
      "logps/rejected": -336.0218811035156,
      "logps/weighted_chosen": -3.4129395484924316,
      "logps/weighted_rejected": -3.74462890625,
      "loss": 0.6566,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -66.412109375,
      "rewards/margins": 25.757617950439453,
      "rewards/rejected": -92.181640625,
      "rewards/weighted_accuracies": 0.6875,
      "rewards/weighted_chosen": -0.689007580280304,
      "rewards/weighted_margins": 0.4539245665073395,
      "rewards/weighted_rejected": -1.143212914466858,
      "step": 450
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 19.516427993774414,
      "learning_rate": 9.416452768711366e-07,
      "logits/chosen": -0.945111095905304,
      "logits/rejected": -0.9787231683731079,
      "logps/chosen": -369.3671875,
      "logps/rejected": -358.9624938964844,
      "logps/weighted_chosen": -3.1959471702575684,
      "logps/weighted_rejected": -3.948193311691284,
      "loss": 0.6392,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -77.17265319824219,
      "rewards/margins": 21.916015625,
      "rewards/rejected": -99.0796890258789,
      "rewards/weighted_accuracies": 0.668749988079071,
      "rewards/weighted_chosen": -0.67822265625,
      "rewards/weighted_margins": 0.532788097858429,
      "rewards/weighted_rejected": -1.2112305164337158,
      "step": 460
    },
    {
      "epoch": 0.24600889819419,
      "grad_norm": 19.182979583740234,
      "learning_rate": 9.372877011572557e-07,
      "logits/chosen": -0.9224609136581421,
      "logits/rejected": -0.9388214349746704,
      "logps/chosen": -391.6937561035156,
      "logps/rejected": -377.0625,
      "logps/weighted_chosen": -3.224560499191284,
      "logps/weighted_rejected": -3.783252000808716,
      "loss": 0.6162,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -80.95976257324219,
      "rewards/margins": 21.617578506469727,
      "rewards/rejected": -102.59883117675781,
      "rewards/weighted_accuracies": 0.6781250238418579,
      "rewards/weighted_chosen": -0.684155285358429,
      "rewards/weighted_margins": 0.5555480718612671,
      "rewards/weighted_rejected": -1.23956298828125,
      "step": 470
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 31.75469970703125,
      "learning_rate": 9.327840749034141e-07,
      "logits/chosen": -0.969561755657196,
      "logits/rejected": -0.998791515827179,
      "logps/chosen": -362.1859436035156,
      "logps/rejected": -385.29998779296875,
      "logps/weighted_chosen": -3.0771241188049316,
      "logps/weighted_rejected": -4.388257026672363,
      "loss": 0.6296,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -75.1123046875,
      "rewards/margins": 33.66425704956055,
      "rewards/rejected": -108.75,
      "rewards/weighted_accuracies": 0.6781250238418579,
      "rewards/weighted_chosen": -0.606555163860321,
      "rewards/weighted_margins": 0.612231433391571,
      "rewards/weighted_rejected": -1.2182190418243408,
      "step": 480
    },
    {
      "epoch": 0.2564773619471343,
      "grad_norm": 37.024818420410156,
      "learning_rate": 9.281359022841965e-07,
      "logits/chosen": -0.846588134765625,
      "logits/rejected": -0.859790027141571,
      "logps/chosen": -352.46405029296875,
      "logps/rejected": -355.24688720703125,
      "logps/weighted_chosen": -3.219531297683716,
      "logps/weighted_rejected": -4.648681640625,
      "loss": 0.5897,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -77.3949203491211,
      "rewards/margins": 32.93046951293945,
      "rewards/rejected": -110.32890319824219,
      "rewards/weighted_accuracies": 0.703125,
      "rewards/weighted_chosen": -0.7288268804550171,
      "rewards/weighted_margins": 0.6741577386856079,
      "rewards/weighted_rejected": -1.403161644935608,
      "step": 490
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 17.740766525268555,
      "learning_rate": 9.233447357514989e-07,
      "logits/chosen": -0.8205505609512329,
      "logits/rejected": -0.863543689250946,
      "logps/chosen": -375.52032470703125,
      "logps/rejected": -378.3500061035156,
      "logps/weighted_chosen": -3.53125,
      "logps/weighted_rejected": -4.106689453125,
      "loss": 0.6305,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -77.591796875,
      "rewards/margins": 30.978906631469727,
      "rewards/rejected": -108.54609680175781,
      "rewards/weighted_accuracies": 0.668749988079071,
      "rewards/weighted_chosen": -0.6407378911972046,
      "rewards/weighted_margins": 0.6662231683731079,
      "rewards/weighted_rejected": -1.30645751953125,
      "step": 500
    },
    {
      "epoch": 0.26171159382360637,
      "eval_logits/chosen": -0.9400458931922913,
      "eval_logits/rejected": -0.955981433391571,
      "eval_logps/chosen": -371.72900390625,
      "eval_logps/rejected": -379.6419982910156,
      "eval_logps/weighted_chosen": -3.214712381362915,
      "eval_logps/weighted_rejected": -4.0158867835998535,
      "eval_loss": 0.6316163539886475,
      "eval_rewards/accuracies": 0.6349999904632568,
      "eval_rewards/chosen": -82.98784637451172,
      "eval_rewards/margins": 28.939437866210938,
      "eval_rewards/rejected": -111.93875122070312,
      "eval_rewards/weighted_accuracies": 0.6725000143051147,
      "eval_rewards/weighted_chosen": -0.6669993996620178,
      "eval_rewards/weighted_margins": 0.5506796836853027,
      "eval_rewards/weighted_rejected": -1.2176789045333862,
      "eval_runtime": 1162.5522,
      "eval_samples_per_second": 1.72,
      "eval_steps_per_second": 0.43,
      "step": 500
    }
  ],
  "logging_steps": 10,
  "max_steps": 1911,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}