{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5234231876472127,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005234231876472127,
      "grad_norm": 124.9239730834961,
      "learning_rate": 0.0,
      "logits/chosen": -0.4007568359375,
      "logits/rejected": -0.41790771484375,
      "logps/chosen": -297.796875,
      "logps/rejected": -248.078125,
      "logps/weighted_chosen": -4.70263671875,
      "logps/weighted_rejected": -3.35546875,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "rewards/weighted_accuracies": 0.0,
      "rewards/weighted_chosen": 0.0,
      "rewards/weighted_margins": 0.0,
      "rewards/weighted_rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005234231876472127,
      "grad_norm": 265.8483581542969,
      "learning_rate": 4.6875e-08,
      "logits/chosen": -0.3176540732383728,
      "logits/rejected": -0.3532511293888092,
      "logps/chosen": -275.5720520019531,
      "logps/rejected": -255.88714599609375,
      "logps/weighted_chosen": -2.5362956523895264,
      "logps/weighted_rejected": -2.774468421936035,
      "loss": 0.6903,
      "rewards/accuracies": 0.2743055522441864,
      "rewards/chosen": 0.0078125,
      "rewards/margins": 0.02973090298473835,
      "rewards/rejected": -0.02191840298473835,
      "rewards/weighted_accuracies": 0.3055555522441864,
      "rewards/weighted_chosen": 0.0025702582206577063,
      "rewards/weighted_margins": 0.0017564562149345875,
      "rewards/weighted_rejected": 0.0008138020639307797,
      "step": 10
    },
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 17.298032760620117,
      "learning_rate": 9.895833333333332e-08,
      "logits/chosen": -0.2979484498500824,
      "logits/rejected": -0.3138488829135895,
      "logps/chosen": -294.40625,
      "logps/rejected": -272.69061279296875,
      "logps/weighted_chosen": -2.3424315452575684,
      "logps/weighted_rejected": -2.307080030441284,
      "loss": 0.6912,
      "rewards/accuracies": 0.296875,
      "rewards/chosen": 0.007617187686264515,
      "rewards/margins": 0.05781250074505806,
      "rewards/rejected": -0.05019531399011612,
      "rewards/weighted_accuracies": 0.3531250059604645,
      "rewards/weighted_chosen": 0.0003913879336323589,
      "rewards/weighted_margins": 0.00116729736328125,
      "rewards/weighted_rejected": -0.0007759094005450606,
      "step": 20
    },
    {
      "epoch": 0.015702695629416383,
      "grad_norm": 79.90486907958984,
      "learning_rate": 1.5104166666666664e-07,
      "logits/chosen": -0.2918685972690582,
      "logits/rejected": -0.3376312255859375,
      "logps/chosen": -298.02032470703125,
      "logps/rejected": -268.0210876464844,
      "logps/weighted_chosen": -2.290698289871216,
      "logps/weighted_rejected": -2.578540086746216,
      "loss": 0.6921,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": 0.0751953125,
      "rewards/margins": 0.10107421875,
      "rewards/rejected": -0.02587890625,
      "rewards/weighted_accuracies": 0.375,
      "rewards/weighted_chosen": 0.0023864745162427425,
      "rewards/weighted_margins": -9.1552734375e-05,
      "rewards/weighted_rejected": 0.0024780272506177425,
      "step": 30
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 32.818607330322266,
      "learning_rate": 2.03125e-07,
      "logits/chosen": -0.3010299801826477,
      "logits/rejected": -0.3435005247592926,
      "logps/chosen": -278.60235595703125,
      "logps/rejected": -253.9250030517578,
      "logps/weighted_chosen": -2.3670654296875,
      "logps/weighted_rejected": -2.760937452316284,
      "loss": 0.6923,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.12041015923023224,
      "rewards/margins": 0.13857421278953552,
      "rewards/rejected": -0.01816406287252903,
      "rewards/weighted_accuracies": 0.4156250059604645,
      "rewards/weighted_chosen": 0.0007911681896075606,
      "rewards/weighted_margins": -0.00026035308837890625,
      "rewards/weighted_rejected": 0.0010513305896893144,
      "step": 40
    },
    {
      "epoch": 0.02617115938236064,
      "grad_norm": 17.478111267089844,
      "learning_rate": 2.552083333333333e-07,
      "logits/chosen": -0.2825515866279602,
      "logits/rejected": -0.3209354281425476,
      "logps/chosen": -280.23126220703125,
      "logps/rejected": -267.45001220703125,
      "logps/weighted_chosen": -2.329760789871216,
      "logps/weighted_rejected": -2.414306640625,
      "loss": 0.6877,
      "rewards/accuracies": 0.30937498807907104,
      "rewards/chosen": 0.01835937425494194,
      "rewards/margins": -0.06035156175494194,
      "rewards/rejected": 0.07871093600988388,
      "rewards/weighted_accuracies": 0.41874998807907104,
      "rewards/weighted_chosen": 0.0070404051803052425,
      "rewards/weighted_margins": 0.00885009765625,
      "rewards/weighted_rejected": -0.0018066406482830644,
      "step": 50
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 39.04142761230469,
      "learning_rate": 3.0729166666666665e-07,
      "logits/chosen": -0.3157821595668793,
      "logits/rejected": -0.30972975492477417,
      "logps/chosen": -277.58123779296875,
      "logps/rejected": -261.6187438964844,
      "logps/weighted_chosen": -2.753100633621216,
      "logps/weighted_rejected": -2.6282715797424316,
      "loss": 0.6846,
      "rewards/accuracies": 0.34687501192092896,
      "rewards/chosen": 0.13593749701976776,
      "rewards/margins": 0.02968749962747097,
      "rewards/rejected": 0.10625000298023224,
      "rewards/weighted_accuracies": 0.4937500059604645,
      "rewards/weighted_chosen": 0.02093353308737278,
      "rewards/weighted_margins": 0.015999604016542435,
      "rewards/weighted_rejected": 0.0049346922896802425,
      "step": 60
    },
    {
      "epoch": 0.036639623135304895,
      "grad_norm": 73.14978790283203,
      "learning_rate": 3.59375e-07,
      "logits/chosen": -0.32021790742874146,
      "logits/rejected": -0.32699280977249146,
      "logps/chosen": -289.6148376464844,
      "logps/rejected": -244.8367156982422,
      "logps/weighted_chosen": -2.221527099609375,
      "logps/weighted_rejected": -2.581958055496216,
      "loss": 0.6775,
      "rewards/accuracies": 0.49687498807907104,
      "rewards/chosen": 0.43046873807907104,
      "rewards/margins": 0.5244140625,
      "rewards/rejected": -0.09394530951976776,
      "rewards/weighted_accuracies": 0.53125,
      "rewards/weighted_chosen": 0.030600737780332565,
      "rewards/weighted_margins": 0.034238435328006744,
      "rewards/weighted_rejected": -0.0036373138427734375,
      "step": 70
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 65.60453796386719,
      "learning_rate": 4.114583333333333e-07,
      "logits/chosen": -0.28860169649124146,
      "logits/rejected": -0.3099655210971832,
      "logps/chosen": -289.33984375,
      "logps/rejected": -270.3453063964844,
      "logps/weighted_chosen": -2.456127882003784,
      "logps/weighted_rejected": -2.6986327171325684,
      "loss": 0.6733,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 0.509765625,
      "rewards/margins": 0.56396484375,
      "rewards/rejected": -0.05419921875,
      "rewards/weighted_accuracies": 0.5625,
      "rewards/weighted_chosen": 0.04326324537396431,
      "rewards/weighted_margins": 0.05183257907629013,
      "rewards/weighted_rejected": -0.008570862002670765,
      "step": 80
    },
    {
      "epoch": 0.04710808688824915,
      "grad_norm": 30.41583824157715,
      "learning_rate": 4.6354166666666664e-07,
      "logits/chosen": -0.3197769224643707,
      "logits/rejected": -0.3126419186592102,
      "logps/chosen": -280.0874938964844,
      "logps/rejected": -257.95782470703125,
      "logps/weighted_chosen": -2.6199097633361816,
      "logps/weighted_rejected": -2.705517530441284,
      "loss": 0.6743,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.5191406011581421,
      "rewards/margins": 0.6358398199081421,
      "rewards/rejected": -0.11669921875,
      "rewards/weighted_accuracies": 0.5718749761581421,
      "rewards/weighted_chosen": 0.07943420112133026,
      "rewards/weighted_margins": 0.07568969577550888,
      "rewards/weighted_rejected": 0.003753662109375,
      "step": 90
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 61.700382232666016,
      "learning_rate": 5.156249999999999e-07,
      "logits/chosen": -0.2895751893520355,
      "logits/rejected": -0.3621200621128082,
      "logps/chosen": -290.88592529296875,
      "logps/rejected": -286.73126220703125,
      "logps/weighted_chosen": -2.083447217941284,
      "logps/weighted_rejected": -2.6861205101013184,
      "loss": 0.6862,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": 0.10849609225988388,
      "rewards/margins": 0.94482421875,
      "rewards/rejected": -0.8363281488418579,
      "rewards/weighted_accuracies": 0.574999988079071,
      "rewards/weighted_chosen": 0.05786743015050888,
      "rewards/weighted_margins": 0.058258056640625,
      "rewards/weighted_rejected": -0.0002998351992573589,
      "step": 100
    },
    {
      "epoch": 0.05757655064119341,
      "grad_norm": 33.45664978027344,
      "learning_rate": 5.677083333333333e-07,
      "logits/chosen": -0.3412017822265625,
      "logits/rejected": -0.3312934935092926,
      "logps/chosen": -296.8671875,
      "logps/rejected": -262.3460998535156,
      "logps/weighted_chosen": -2.776904344558716,
      "logps/weighted_rejected": -2.8023438453674316,
      "loss": 0.6835,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.4873046875,
      "rewards/margins": 1.0051758289337158,
      "rewards/rejected": -1.4924805164337158,
      "rewards/weighted_accuracies": 0.5062500238418579,
      "rewards/weighted_chosen": 0.01709899865090847,
      "rewards/weighted_margins": 0.05614929273724556,
      "rewards/weighted_rejected": -0.03909149020910263,
      "step": 110
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 61.74055862426758,
      "learning_rate": 6.197916666666666e-07,
      "logits/chosen": -0.3482359051704407,
      "logits/rejected": -0.3736562728881836,
      "logps/chosen": -295.49530029296875,
      "logps/rejected": -256.08282470703125,
      "logps/weighted_chosen": -2.066760301589966,
      "logps/weighted_rejected": -2.2858643531799316,
      "loss": 0.6833,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8194335699081421,
      "rewards/margins": 1.502539038658142,
      "rewards/rejected": -2.321972608566284,
      "rewards/weighted_accuracies": 0.5874999761581421,
      "rewards/weighted_chosen": 0.01521911658346653,
      "rewards/weighted_margins": 0.05682678148150444,
      "rewards/weighted_rejected": -0.04169006273150444,
      "step": 120
    },
    {
      "epoch": 0.06804501439413765,
      "grad_norm": 45.78140640258789,
      "learning_rate": 6.718749999999999e-07,
      "logits/chosen": -0.3164505064487457,
      "logits/rejected": -0.3121093809604645,
      "logps/chosen": -306.4761657714844,
      "logps/rejected": -279.87030029296875,
      "logps/weighted_chosen": -2.1032347679138184,
      "logps/weighted_rejected": -2.432788133621216,
      "loss": 0.6718,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.4637207090854645,
      "rewards/margins": 1.8840820789337158,
      "rewards/rejected": -2.347851514816284,
      "rewards/weighted_accuracies": 0.559374988079071,
      "rewards/weighted_chosen": 0.01874847337603569,
      "rewards/weighted_margins": 0.07195129245519638,
      "rewards/weighted_rejected": -0.05319824069738388,
      "step": 130
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 47.97172546386719,
      "learning_rate": 7.239583333333333e-07,
      "logits/chosen": -0.36130523681640625,
      "logits/rejected": -0.3767889142036438,
      "logps/chosen": -300.2398376464844,
      "logps/rejected": -276.47344970703125,
      "logps/weighted_chosen": -2.402172803878784,
      "logps/weighted_rejected": -2.704345703125,
      "loss": 0.6601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6025390625,
      "rewards/margins": 2.2777342796325684,
      "rewards/rejected": -3.8802733421325684,
      "rewards/weighted_accuracies": 0.628125011920929,
      "rewards/weighted_chosen": 0.0066894530318677425,
      "rewards/weighted_margins": 0.10413207858800888,
      "rewards/weighted_rejected": -0.0973968505859375,
      "step": 140
    },
    {
      "epoch": 0.07851347814708191,
      "grad_norm": 18.48399543762207,
      "learning_rate": 7.760416666666666e-07,
      "logits/chosen": -0.3260757327079773,
      "logits/rejected": -0.33597564697265625,
      "logps/chosen": -286.6578063964844,
      "logps/rejected": -255.8078155517578,
      "logps/weighted_chosen": -2.939697265625,
      "logps/weighted_rejected": -2.9384522438049316,
      "loss": 0.6693,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.329296827316284,
      "rewards/margins": 2.532031297683716,
      "rewards/rejected": -4.861328125,
      "rewards/weighted_accuracies": 0.625,
      "rewards/weighted_chosen": 0.01940460130572319,
      "rewards/weighted_margins": 0.12819823622703552,
      "rewards/weighted_rejected": -0.10869445651769638,
      "step": 150
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 16.847576141357422,
      "learning_rate": 8.28125e-07,
      "logits/chosen": -0.37299805879592896,
      "logits/rejected": -0.4207763671875,
      "logps/chosen": -306.01873779296875,
      "logps/rejected": -279.0289001464844,
      "logps/weighted_chosen": -2.4432616233825684,
      "logps/weighted_rejected": -2.4922118186950684,
      "loss": 0.631,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -2.1587891578674316,
      "rewards/margins": 3.69091796875,
      "rewards/rejected": -5.849707126617432,
      "rewards/weighted_accuracies": 0.637499988079071,
      "rewards/weighted_chosen": 0.04796142503619194,
      "rewards/weighted_margins": 0.21007385849952698,
      "rewards/weighted_rejected": -0.1621856689453125,
      "step": 160
    },
    {
      "epoch": 0.08898194190002617,
      "grad_norm": 237.514404296875,
      "learning_rate": 8.802083333333333e-07,
      "logits/chosen": -0.42098236083984375,
      "logits/rejected": -0.4231323301792145,
      "logps/chosen": -306.859375,
      "logps/rejected": -266.03125,
      "logps/weighted_chosen": -2.6180663108825684,
      "logps/weighted_rejected": -2.9256348609924316,
      "loss": 0.709,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -3.73828125,
      "rewards/margins": 4.098242282867432,
      "rewards/rejected": -7.836328029632568,
      "rewards/weighted_accuracies": 0.6031249761581421,
      "rewards/weighted_chosen": 0.06289978325366974,
      "rewards/weighted_margins": 0.186981201171875,
      "rewards/weighted_rejected": -0.12393493950366974,
      "step": 170
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 52.46952819824219,
      "learning_rate": 9.322916666666666e-07,
      "logits/chosen": -0.38860780000686646,
      "logits/rejected": -0.40771445631980896,
      "logps/chosen": -280.2789001464844,
      "logps/rejected": -267.69219970703125,
      "logps/weighted_chosen": -2.579449415206909,
      "logps/weighted_rejected": -3.0037841796875,
      "loss": 0.6853,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -6.609179496765137,
      "rewards/margins": 4.819726467132568,
      "rewards/rejected": -11.427831649780273,
      "rewards/weighted_accuracies": 0.6031249761581421,
      "rewards/weighted_chosen": -0.04149780422449112,
      "rewards/weighted_margins": 0.20274047553539276,
      "rewards/weighted_rejected": -0.24451598525047302,
      "step": 180
    },
    {
      "epoch": 0.09945040565297043,
      "grad_norm": 37.741477966308594,
      "learning_rate": 9.84375e-07,
      "logits/chosen": -0.4061637818813324,
      "logits/rejected": -0.4410812258720398,
      "logps/chosen": -314.60626220703125,
      "logps/rejected": -278.87811279296875,
      "logps/weighted_chosen": -2.378369092941284,
      "logps/weighted_rejected": -2.7929444313049316,
      "loss": 0.6517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -8.988476753234863,
      "rewards/margins": 6.061327934265137,
      "rewards/rejected": -15.050390243530273,
      "rewards/weighted_accuracies": 0.621874988079071,
      "rewards/weighted_chosen": -0.11425475776195526,
      "rewards/weighted_margins": 0.22973021864891052,
      "rewards/weighted_rejected": -0.3441314697265625,
      "step": 190
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 76.11390686035156,
      "learning_rate": 9.99959085414323e-07,
      "logits/chosen": -0.4347488284111023,
      "logits/rejected": -0.46932679414749146,
      "logps/chosen": -319.34686279296875,
      "logps/rejected": -274.1890563964844,
      "logps/weighted_chosen": -2.6759276390075684,
      "logps/weighted_rejected": -3.110583543777466,
      "loss": 0.6539,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -10.064160346984863,
      "rewards/margins": 6.332421779632568,
      "rewards/rejected": -16.395898818969727,
      "rewards/weighted_accuracies": 0.612500011920929,
      "rewards/weighted_chosen": -0.08293457329273224,
      "rewards/weighted_margins": 0.25919800996780396,
      "rewards/weighted_rejected": -0.34211426973342896,
      "step": 200
    },
    {
      "epoch": 0.10991886940591468,
      "grad_norm": 19.709537506103516,
      "learning_rate": 9.997587035630105e-07,
      "logits/chosen": -0.4373016357421875,
      "logits/rejected": -0.47842711210250854,
      "logps/chosen": -298.6578063964844,
      "logps/rejected": -305.07733154296875,
      "logps/weighted_chosen": -2.4120116233825684,
      "logps/weighted_rejected": -3.137524366378784,
      "loss": 0.6448,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -11.547070503234863,
      "rewards/margins": 8.876855850219727,
      "rewards/rejected": -20.425390243530273,
      "rewards/weighted_accuracies": 0.659375011920929,
      "rewards/weighted_chosen": -0.12418060004711151,
      "rewards/weighted_margins": 0.24833527207374573,
      "rewards/weighted_rejected": -0.37247925996780396,
      "step": 210
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 39.1947135925293,
      "learning_rate": 9.99391406364405e-07,
      "logits/chosen": -0.424050897359848,
      "logits/rejected": -0.4263656735420227,
      "logps/chosen": -306.5453186035156,
      "logps/rejected": -291.21405029296875,
      "logps/weighted_chosen": -2.8793702125549316,
      "logps/weighted_rejected": -3.2673583030700684,
      "loss": 0.6969,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -14.446874618530273,
      "rewards/margins": 7.897851467132568,
      "rewards/rejected": -22.343358993530273,
      "rewards/weighted_accuracies": 0.606249988079071,
      "rewards/weighted_chosen": -0.2553466856479645,
      "rewards/weighted_margins": 0.21260376274585724,
      "rewards/weighted_rejected": -0.46800535917282104,
      "step": 220
    },
    {
      "epoch": 0.12038733315885894,
      "grad_norm": 437.028564453125,
      "learning_rate": 9.988573164927884e-07,
      "logits/chosen": -0.3824310302734375,
      "logits/rejected": -0.4174056947231293,
      "logps/chosen": -282.2710876464844,
      "logps/rejected": -277.2734375,
      "logps/weighted_chosen": -2.547900438308716,
      "logps/weighted_rejected": -2.85693359375,
      "loss": 0.7367,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -13.438867568969727,
      "rewards/margins": 12.128320693969727,
      "rewards/rejected": -25.565820693969727,
      "rewards/weighted_accuracies": 0.621874988079071,
      "rewards/weighted_chosen": -0.24434813857078552,
      "rewards/weighted_margins": 0.2164306640625,
      "rewards/weighted_rejected": -0.4607299864292145,
      "step": 230
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 139.88975524902344,
      "learning_rate": 9.98156612329838e-07,
      "logits/chosen": -0.4804244935512543,
      "logits/rejected": -0.527056872844696,
      "logps/chosen": -279.6773376464844,
      "logps/rejected": -310.64532470703125,
      "logps/weighted_chosen": -2.464794874191284,
      "logps/weighted_rejected": -3.058520555496216,
      "loss": 0.6735,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -14.966699600219727,
      "rewards/margins": 13.450976371765137,
      "rewards/rejected": -28.419336318969727,
      "rewards/weighted_accuracies": 0.6625000238418579,
      "rewards/weighted_chosen": -0.11341552436351776,
      "rewards/weighted_margins": 0.26569825410842896,
      "rewards/weighted_rejected": -0.37919312715530396,
      "step": 240
    },
    {
      "epoch": 0.13085579691180318,
      "grad_norm": 939.5078125,
      "learning_rate": 9.97289527905053e-07,
      "logits/chosen": -0.4899856448173523,
      "logits/rejected": -0.4979141354560852,
      "logps/chosen": -281.9039001464844,
      "logps/rejected": -282.93438720703125,
      "logps/weighted_chosen": -2.8607420921325684,
      "logps/weighted_rejected": -3.0235595703125,
      "loss": 0.7231,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -17.384668350219727,
      "rewards/margins": 9.600194931030273,
      "rewards/rejected": -26.986328125,
      "rewards/weighted_accuracies": 0.5843750238418579,
      "rewards/weighted_chosen": -0.20379944145679474,
      "rewards/weighted_margins": 0.18174438178539276,
      "rewards/weighted_rejected": -0.3856750428676605,
      "step": 250
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 24.300647735595703,
      "learning_rate": 9.962563528175875e-07,
      "logits/chosen": -0.4584365785121918,
      "logits/rejected": -0.4911254942417145,
      "logps/chosen": -317.06329345703125,
      "logps/rejected": -289.06561279296875,
      "logps/weighted_chosen": -2.4532713890075684,
      "logps/weighted_rejected": -3.330212354660034,
      "loss": 0.6636,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -18.153711318969727,
      "rewards/margins": 10.150195121765137,
      "rewards/rejected": -28.3046875,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.18357238173484802,
      "rewards/weighted_margins": 0.269927978515625,
      "rewards/weighted_rejected": -0.4535202085971832,
      "step": 260
    },
    {
      "epoch": 0.14132426066474746,
      "grad_norm": 228.46092224121094,
      "learning_rate": 9.950574321395277e-07,
      "logits/chosen": -0.4871993958950043,
      "logits/rejected": -0.5071777105331421,
      "logps/chosen": -310.2124938964844,
      "logps/rejected": -291.5687561035156,
      "logps/weighted_chosen": -2.747607469558716,
      "logps/weighted_rejected": -3.0499024391174316,
      "loss": 0.6525,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -20.7822265625,
      "rewards/margins": 7.598730564117432,
      "rewards/rejected": -28.382030487060547,
      "rewards/weighted_accuracies": 0.6499999761581421,
      "rewards/weighted_chosen": -0.2510986328125,
      "rewards/weighted_margins": 0.2596893310546875,
      "rewards/weighted_rejected": -0.5105804204940796,
      "step": 270
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 713.1409912109375,
      "learning_rate": 9.936931663006413e-07,
      "logits/chosen": -0.4856887757778168,
      "logits/rejected": -0.4791274964809418,
      "logps/chosen": -323.875,
      "logps/rejected": -312.1109313964844,
      "logps/weighted_chosen": -2.7660155296325684,
      "logps/weighted_rejected": -3.293591260910034,
      "loss": 0.6155,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -20.981639862060547,
      "rewards/margins": 11.869140625,
      "rewards/rejected": -32.850975036621094,
      "rewards/weighted_accuracies": 0.675000011920929,
      "rewards/weighted_chosen": -0.13939818739891052,
      "rewards/weighted_margins": 0.34244996309280396,
      "rewards/weighted_rejected": -0.48178404569625854,
      "step": 280
    },
    {
      "epoch": 0.1517927244176917,
      "grad_norm": 10845.4873046875,
      "learning_rate": 9.921640109546357e-07,
      "logits/chosen": -0.433615118265152,
      "logits/rejected": -0.5038871765136719,
      "logps/chosen": -285.6937561035156,
      "logps/rejected": -280.76251220703125,
      "logps/weighted_chosen": -2.63427734375,
      "logps/weighted_rejected": -3.711669921875,
      "loss": 0.65,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -18.829492568969727,
      "rewards/margins": 10.345703125,
      "rewards/rejected": -29.1728515625,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.12302856147289276,
      "rewards/weighted_margins": 0.3319152891635895,
      "rewards/weighted_rejected": -0.45494383573532104,
      "step": 290
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 47.70283889770508,
      "learning_rate": 9.90470476826975e-07,
      "logits/chosen": -0.4355606138706207,
      "logits/rejected": -0.4278717041015625,
      "logps/chosen": -288.78045654296875,
      "logps/rejected": -296.9765625,
      "logps/weighted_chosen": -2.532849073410034,
      "logps/weighted_rejected": -3.0221924781799316,
      "loss": 0.6674,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -19.346094131469727,
      "rewards/margins": 10.255078315734863,
      "rewards/rejected": -29.59765625,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.21412047743797302,
      "rewards/weighted_margins": 0.2544006407260895,
      "rewards/weighted_rejected": -0.4681999087333679,
      "step": 300
    },
    {
      "epoch": 0.16226118817063595,
      "grad_norm": 644.2051391601562,
      "learning_rate": 9.886131295443002e-07,
      "logits/chosen": -0.462362676858902,
      "logits/rejected": -0.5182906985282898,
      "logps/chosen": -301.2421875,
      "logps/rejected": -280.125,
      "logps/weighted_chosen": -2.5293211936950684,
      "logps/weighted_rejected": -2.9417481422424316,
      "loss": 0.6339,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -19.940235137939453,
      "rewards/margins": 9.952539443969727,
      "rewards/rejected": -29.892187118530273,
      "rewards/weighted_accuracies": 0.65625,
      "rewards/weighted_chosen": 0.0126800537109375,
      "rewards/weighted_margins": 0.4334716796875,
      "rewards/weighted_rejected": -0.42060548067092896,
      "step": 310
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 42.43592834472656,
      "learning_rate": 9.865925894455166e-07,
      "logits/chosen": -0.538470447063446,
      "logits/rejected": -0.5473419427871704,
      "logps/chosen": -323.84686279296875,
      "logps/rejected": -290.7406311035156,
      "logps/weighted_chosen": -2.8351807594299316,
      "logps/weighted_rejected": -3.2470703125,
      "loss": 0.6762,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -23.906444549560547,
      "rewards/margins": 9.430078506469727,
      "rewards/rejected": -33.3359375,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.23797301948070526,
      "rewards/weighted_margins": 0.272064208984375,
      "rewards/weighted_rejected": -0.5102294683456421,
      "step": 320
    },
    {
      "epoch": 0.17272965192358022,
      "grad_norm": 46.229000091552734,
      "learning_rate": 9.84409531374603e-07,
      "logits/chosen": -0.5096954107284546,
      "logits/rejected": -0.4907272458076477,
      "logps/chosen": -332.18280029296875,
      "logps/rejected": -298.5718688964844,
      "logps/weighted_chosen": -2.775341749191284,
      "logps/weighted_rejected": -3.165026903152466,
      "loss": 0.6741,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -27.408203125,
      "rewards/margins": 8.212109565734863,
      "rewards/rejected": -35.61640548706055,
      "rewards/weighted_accuracies": 0.6312500238418579,
      "rewards/weighted_chosen": -0.17764969170093536,
      "rewards/weighted_margins": 0.241455078125,
      "rewards/weighted_rejected": -0.41897353529930115,
      "step": 330
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 70.134521484375,
      "learning_rate": 9.820646844552219e-07,
      "logits/chosen": -0.5079803466796875,
      "logits/rejected": -0.5622657537460327,
      "logps/chosen": -301.3257751464844,
      "logps/rejected": -303.1468811035156,
      "logps/weighted_chosen": -2.852612257003784,
      "logps/weighted_rejected": -3.0445556640625,
      "loss": 0.6485,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -24.6875,
      "rewards/margins": 12.933984756469727,
      "rewards/rejected": -37.61992263793945,
      "rewards/weighted_accuracies": 0.65625,
      "rewards/weighted_chosen": -0.19311293959617615,
      "rewards/weighted_margins": 0.3020080626010895,
      "rewards/weighted_rejected": -0.4951171875,
      "step": 340
    },
    {
      "epoch": 0.18319811567652447,
      "grad_norm": 37.5682373046875,
      "learning_rate": 9.795588318471964e-07,
      "logits/chosen": -0.5787566900253296,
      "logits/rejected": -0.5869846343994141,
      "logps/chosen": -283.0796813964844,
      "logps/rejected": -310.00469970703125,
      "logps/weighted_chosen": -2.5658936500549316,
      "logps/weighted_rejected": -2.95947265625,
      "loss": 0.6307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -23.71875,
      "rewards/margins": 9.896288871765137,
      "rewards/rejected": -33.61249923706055,
      "rewards/weighted_accuracies": 0.6468750238418579,
      "rewards/weighted_chosen": -0.1531631499528885,
      "rewards/weighted_margins": 0.32664793729782104,
      "rewards/weighted_rejected": -0.479653924703598,
      "step": 350
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 145.69015502929688,
      "learning_rate": 9.768928104849415e-07,
      "logits/chosen": -0.6053832769393921,
      "logits/rejected": -0.6036437749862671,
      "logps/chosen": -306.9140625,
      "logps/rejected": -283.77655029296875,
      "logps/weighted_chosen": -2.835888624191284,
      "logps/weighted_rejected": -2.9885010719299316,
      "loss": 0.6741,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -24.237499237060547,
      "rewards/margins": 10.624218940734863,
      "rewards/rejected": -34.85859298706055,
      "rewards/weighted_accuracies": 0.640625,
      "rewards/weighted_chosen": -0.1663818359375,
      "rewards/weighted_margins": 0.3080078065395355,
      "rewards/weighted_rejected": -0.4744934141635895,
      "step": 360
    },
    {
      "epoch": 0.19366657942946872,
      "grad_norm": 1347.0242919921875,
      "learning_rate": 9.740675107979355e-07,
      "logits/chosen": -0.5699676275253296,
      "logits/rejected": -0.5878998041152954,
      "logps/chosen": -342.36248779296875,
      "logps/rejected": -311.7203063964844,
      "logps/weighted_chosen": -2.303784132003784,
      "logps/weighted_rejected": -3.164013624191284,
      "loss": 0.6624,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -27.253124237060547,
      "rewards/margins": 9.823046684265137,
      "rewards/rejected": -37.07597732543945,
      "rewards/weighted_accuracies": 0.643750011920929,
      "rewards/weighted_chosen": -0.24745483696460724,
      "rewards/weighted_margins": 0.27247315645217896,
      "rewards/weighted_rejected": -0.5200088620185852,
      "step": 370
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 1152.541748046875,
      "learning_rate": 9.71083876413323e-07,
      "logits/chosen": -0.5426742434501648,
      "logits/rejected": -0.5510879755020142,
      "logps/chosen": -331.25390625,
      "logps/rejected": -310.33123779296875,
      "logps/weighted_chosen": -2.4026856422424316,
      "logps/weighted_rejected": -3.172229051589966,
      "loss": 0.6676,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -28.869922637939453,
      "rewards/margins": 11.865234375,
      "rewards/rejected": -40.73320388793945,
      "rewards/weighted_accuracies": 0.6187499761581421,
      "rewards/weighted_chosen": -0.24112549424171448,
      "rewards/weighted_margins": 0.2662292420864105,
      "rewards/weighted_rejected": -0.5070465207099915,
      "step": 380
    },
    {
      "epoch": 0.204135043182413,
      "grad_norm": 69.06751251220703,
      "learning_rate": 9.67942903840751e-07,
      "logits/chosen": -0.5844222903251648,
      "logits/rejected": -0.6357940435409546,
      "logps/chosen": -331.53436279296875,
      "logps/rejected": -316.50469970703125,
      "logps/weighted_chosen": -2.523181200027466,
      "logps/weighted_rejected": -3.170581102371216,
      "loss": 0.6281,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -27.312108993530273,
      "rewards/margins": 14.989062309265137,
      "rewards/rejected": -42.309181213378906,
      "rewards/weighted_accuracies": 0.675000011920929,
      "rewards/weighted_chosen": -0.17550353705883026,
      "rewards/weighted_margins": 0.357177734375,
      "rewards/weighted_rejected": -0.532818615436554,
      "step": 390
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 64.58200073242188,
      "learning_rate": 9.646456421395447e-07,
      "logits/chosen": -0.573455810546875,
      "logits/rejected": -0.5838836431503296,
      "logps/chosen": -348.1484375,
      "logps/rejected": -349.8687438964844,
      "logps/weighted_chosen": -2.479809522628784,
      "logps/weighted_rejected": -3.226489305496216,
      "loss": 0.6551,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -29.957422256469727,
      "rewards/margins": 16.478321075439453,
      "rewards/rejected": -46.43476486206055,
      "rewards/weighted_accuracies": 0.640625,
      "rewards/weighted_chosen": -0.203724667429924,
      "rewards/weighted_margins": 0.254974365234375,
      "rewards/weighted_rejected": -0.4586502015590668,
      "step": 400
    },
    {
      "epoch": 0.21460350693535724,
      "grad_norm": 19.509164810180664,
      "learning_rate": 9.611931925683266e-07,
      "logits/chosen": -0.5067977905273438,
      "logits/rejected": -0.5407257080078125,
      "logps/chosen": -337.97967529296875,
      "logps/rejected": -311.12811279296875,
      "logps/weighted_chosen": -2.369091749191284,
      "logps/weighted_rejected": -2.994799852371216,
      "loss": 0.5919,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -31.741016387939453,
      "rewards/margins": 15.541796684265137,
      "rewards/rejected": -47.293357849121094,
      "rewards/weighted_accuracies": 0.6937500238418579,
      "rewards/weighted_chosen": -0.17744140326976776,
      "rewards/weighted_margins": 0.386505126953125,
      "rewards/weighted_rejected": -0.563507080078125,
      "step": 410
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 29.50520133972168,
      "learning_rate": 9.575867082172085e-07,
      "logits/chosen": -0.5286194086074829,
      "logits/rejected": -0.5688308477401733,
      "logps/chosen": -344.96563720703125,
      "logps/rejected": -328.64532470703125,
      "logps/weighted_chosen": -2.8730225563049316,
      "logps/weighted_rejected": -2.930432081222534,
      "loss": 0.6442,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -40.091796875,
      "rewards/margins": 18.596485137939453,
      "rewards/rejected": -58.685935974121094,
      "rewards/weighted_accuracies": 0.628125011920929,
      "rewards/weighted_chosen": -0.38405150175094604,
      "rewards/weighted_margins": 0.39207762479782104,
      "rewards/weighted_rejected": -0.7763046026229858,
      "step": 420
    },
    {
      "epoch": 0.22507197068830148,
      "grad_norm": 15.081392288208008,
      "learning_rate": 9.538273936226673e-07,
      "logits/chosen": -0.5923309326171875,
      "logits/rejected": -0.625653088092804,
      "logps/chosen": -301.640625,
      "logps/rejected": -313.77032470703125,
      "logps/weighted_chosen": -3.0785889625549316,
      "logps/weighted_rejected": -3.5872559547424316,
      "loss": 0.668,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -36.12675857543945,
      "rewards/margins": 13.111328125,
      "rewards/rejected": -49.236328125,
      "rewards/weighted_accuracies": 0.6312500238418579,
      "rewards/weighted_chosen": -0.28452759981155396,
      "rewards/weighted_margins": 0.3635620176792145,
      "rewards/weighted_rejected": -0.64788818359375,
      "step": 430
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 23.0412540435791,
      "learning_rate": 9.499165043652391e-07,
      "logits/chosen": -0.5760772824287415,
      "logits/rejected": -0.5774017572402954,
      "logps/chosen": -328.1015625,
      "logps/rejected": -318.84454345703125,
      "logps/weighted_chosen": -3.0999755859375,
      "logps/weighted_rejected": -3.371142625808716,
      "loss": 0.6452,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -36.97431564331055,
      "rewards/margins": 15.057812690734863,
      "rewards/rejected": -52.0390625,
      "rewards/weighted_accuracies": 0.628125011920929,
      "rewards/weighted_chosen": -0.39061737060546875,
      "rewards/weighted_margins": 0.3595825135707855,
      "rewards/weighted_rejected": -0.7500244379043579,
      "step": 440
    },
    {
      "epoch": 0.23554043444124576,
      "grad_norm": 22.85993003845215,
      "learning_rate": 9.458553466501665e-07,
      "logits/chosen": -0.6301513910293579,
      "logits/rejected": -0.6610687375068665,
      "logps/chosen": -324.77032470703125,
      "logps/rejected": -298.57501220703125,
      "logps/weighted_chosen": -3.1078124046325684,
      "logps/weighted_rejected": -3.38037109375,
      "loss": 0.6501,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -38.617774963378906,
      "rewards/margins": 16.141407012939453,
      "rewards/rejected": -54.763671875,
      "rewards/weighted_accuracies": 0.653124988079071,
      "rewards/weighted_chosen": -0.4693542420864105,
      "rewards/weighted_margins": 0.4438629150390625,
      "rewards/weighted_rejected": -0.9127838015556335,
      "step": 450
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 11.338366508483887,
      "learning_rate": 9.416452768711366e-07,
      "logits/chosen": -0.6437957882881165,
      "logits/rejected": -0.674633800983429,
      "logps/chosen": -333.87188720703125,
      "logps/rejected": -318.10467529296875,
      "logps/weighted_chosen": -2.8603515625,
      "logps/weighted_rejected": -3.472119092941284,
      "loss": 0.6881,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -41.862693786621094,
      "rewards/margins": 16.437694549560547,
      "rewards/rejected": -58.305076599121094,
      "rewards/weighted_accuracies": 0.653124988079071,
      "rewards/weighted_chosen": -0.43403321504592896,
      "rewards/weighted_margins": 0.4345336854457855,
      "rewards/weighted_rejected": -0.8688415288925171,
      "step": 460
    },
    {
      "epoch": 0.24600889819419,
      "grad_norm": 35.14228057861328,
      "learning_rate": 9.372877011572557e-07,
      "logits/chosen": -0.5738860964775085,
      "logits/rejected": -0.5953949093818665,
      "logps/chosen": -350.23046875,
      "logps/rejected": -329.484375,
      "logps/weighted_chosen": -2.7630372047424316,
      "logps/weighted_rejected": -3.2092041969299316,
      "loss": 0.6424,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -39.640037536621094,
      "rewards/margins": 15.155468940734863,
      "rewards/rejected": -54.796485900878906,
      "rewards/weighted_accuracies": 0.671875,
      "rewards/weighted_chosen": -0.338470458984375,
      "rewards/weighted_margins": 0.4430099427700043,
      "rewards/weighted_rejected": -0.781768798828125,
      "step": 470
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 55.78861618041992,
      "learning_rate": 9.327840749034141e-07,
      "logits/chosen": -0.6301849484443665,
      "logits/rejected": -0.6710245013237,
      "logps/chosen": -324.8578186035156,
      "logps/rejected": -336.734375,
      "logps/weighted_chosen": -2.5625243186950684,
      "logps/weighted_rejected": -3.774975538253784,
      "loss": 0.636,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -37.658203125,
      "rewards/margins": 22.687694549560547,
      "rewards/rejected": -60.357421875,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.2594970762729645,
      "rewards/weighted_margins": 0.48378294706344604,
      "rewards/weighted_rejected": -0.7435302734375,
      "step": 480
    },
    {
      "epoch": 0.2564773619471343,
      "grad_norm": 38.52030563354492,
      "learning_rate": 9.281359022841965e-07,
      "logits/chosen": -0.5341835021972656,
      "logits/rejected": -0.550585925579071,
      "logps/chosen": -320.76171875,
      "logps/rejected": -314.70391845703125,
      "logps/weighted_chosen": -2.91943359375,
      "logps/weighted_rejected": -4.169653415679932,
      "loss": 0.6375,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": -45.831443786621094,
      "rewards/margins": 24.026952743530273,
      "rewards/rejected": -69.88359069824219,
      "rewards/weighted_accuracies": 0.668749988079071,
      "rewards/weighted_chosen": -0.516644299030304,
      "rewards/weighted_margins": 0.518634021282196,
      "rewards/weighted_rejected": -1.0354125499725342,
      "step": 490
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 15.318764686584473,
      "learning_rate": 9.233447357514989e-07,
      "logits/chosen": -0.5592590570449829,
      "logits/rejected": -0.6022475957870483,
      "logps/chosen": -347.77423095703125,
      "logps/rejected": -342.21875,
      "logps/weighted_chosen": -3.2605223655700684,
      "logps/weighted_rejected": -3.667773485183716,
      "loss": 0.6494,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -49.72968673706055,
      "rewards/margins": 22.665430068969727,
      "rewards/rejected": -72.4097671508789,
      "rewards/weighted_accuracies": 0.65625,
      "rewards/weighted_chosen": -0.4633804261684418,
      "rewards/weighted_margins": 0.527294933795929,
      "rewards/weighted_rejected": -0.990936279296875,
      "step": 500
    },
    {
      "epoch": 0.26171159382360637,
      "eval_logits/chosen": -0.6649160385131836,
      "eval_logits/rejected": -0.682785153388977,
      "eval_logps/chosen": -345.551513671875,
      "eval_logps/rejected": -345.9989929199219,
      "eval_logps/weighted_chosen": -2.992426872253418,
      "eval_logps/weighted_rejected": -3.637266159057617,
      "eval_loss": 0.6663314700126648,
      "eval_rewards/accuracies": 0.6305000185966492,
      "eval_rewards/chosen": -56.88743591308594,
      "eval_rewards/margins": 21.448062896728516,
      "eval_rewards/rejected": -78.32637786865234,
      "eval_rewards/weighted_accuracies": 0.6480000019073486,
      "eval_rewards/weighted_chosen": -0.5315031409263611,
      "eval_rewards/weighted_margins": 0.43671876192092896,
      "eval_rewards/weighted_rejected": -0.9682218432426453,
      "eval_runtime": 1455.1048,
      "eval_samples_per_second": 1.374,
      "eval_steps_per_second": 0.344,
      "step": 500
    },
    {
      "epoch": 0.2669458257000785,
      "grad_norm": 51.472145080566406,
      "learning_rate": 9.184121755160232e-07,
      "logits/chosen": -0.659375011920929,
      "logits/rejected": -0.689471423625946,
      "logps/chosen": -355.8125,
      "logps/rejected": -371.6640625,
      "logps/weighted_chosen": -3.25238037109375,
      "logps/weighted_rejected": -3.708740234375,
      "loss": 0.6929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -54.556640625,
      "rewards/margins": 28.4453125,
      "rewards/rejected": -83.0054702758789,
      "rewards/weighted_accuracies": 0.6625000238418579,
      "rewards/weighted_chosen": -0.532073974609375,
      "rewards/weighted_margins": 0.402008056640625,
      "rewards/weighted_rejected": -0.934326171875,
      "step": 510
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 39.522377014160156,
      "learning_rate": 9.133398690128193e-07,
      "logits/chosen": -0.668170154094696,
      "logits/rejected": -0.691680908203125,
      "logps/chosen": -380.31561279296875,
      "logps/rejected": -371.16094970703125,
      "logps/weighted_chosen": -2.9228272438049316,
      "logps/weighted_rejected": -3.671313524246216,
      "loss": 0.6462,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -58.849220275878906,
      "rewards/margins": 31.808202743530273,
      "rewards/rejected": -90.65234375,
      "rewards/weighted_accuracies": 0.7281249761581421,
      "rewards/weighted_chosen": -0.3155883848667145,
      "rewards/weighted_margins": 0.47456055879592896,
      "rewards/weighted_rejected": -0.790509045124054,
      "step": 520
    },
    {
      "epoch": 0.27741428945302277,
      "grad_norm": 40.427040100097656,
      "learning_rate": 9.081295103510554e-07,
      "logits/chosen": -0.5946930050849915,
      "logits/rejected": -0.64569091796875,
      "logps/chosen": -355.3265686035156,
      "logps/rejected": -368.6781311035156,
      "logps/weighted_chosen": -2.624804735183716,
      "logps/weighted_rejected": -3.721435546875,
      "loss": 0.5869,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -71.3091812133789,
      "rewards/margins": 27.795703887939453,
      "rewards/rejected": -99.08203125,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.41926270723342896,
      "rewards/weighted_margins": 0.5740417242050171,
      "rewards/weighted_rejected": -0.993090808391571,
      "step": 530
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 21.890594482421875,
      "learning_rate": 9.027828397481989e-07,
      "logits/chosen": -0.5646041631698608,
      "logits/rejected": -0.5952346920967102,
      "logps/chosen": -338.10467529296875,
      "logps/rejected": -359.58282470703125,
      "logps/weighted_chosen": -3.3427734375,
      "logps/weighted_rejected": -4.061425685882568,
      "loss": 0.6384,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -77.6187515258789,
      "rewards/margins": 26.623828887939453,
      "rewards/rejected": -104.2367172241211,
      "rewards/weighted_accuracies": 0.65625,
      "rewards/weighted_chosen": -0.63751220703125,
      "rewards/weighted_margins": 0.47407835721969604,
      "rewards/weighted_rejected": -1.1118652820587158,
      "step": 540
    },
    {
      "epoch": 0.287882753205967,
      "grad_norm": 28.308740615844727,
      "learning_rate": 8.973016429487988e-07,
      "logits/chosen": -0.6230270266532898,
      "logits/rejected": -0.630328357219696,
      "logps/chosen": -362.5726623535156,
      "logps/rejected": -369.64373779296875,
      "logps/weighted_chosen": -3.282421827316284,
      "logps/weighted_rejected": -3.598876953125,
      "loss": 0.6423,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -80.73359680175781,
      "rewards/margins": 34.60234451293945,
      "rewards/rejected": -115.34687805175781,
      "rewards/weighted_accuracies": 0.684374988079071,
      "rewards/weighted_chosen": -0.597552478313446,
      "rewards/weighted_margins": 0.46974486112594604,
      "rewards/weighted_rejected": -1.067834496498108,
      "step": 550
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 108.37396240234375,
      "learning_rate": 8.916877506280601e-07,
      "logits/chosen": -0.6662933230400085,
      "logits/rejected": -0.657055675983429,
      "logps/chosen": -369.0171813964844,
      "logps/rejected": -373.078125,
      "logps/weighted_chosen": -3.2555909156799316,
      "logps/weighted_rejected": -3.583251953125,
      "loss": 0.6414,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -88.1117172241211,
      "rewards/margins": 30.770702362060547,
      "rewards/rejected": -118.8863296508789,
      "rewards/weighted_accuracies": 0.684374988079071,
      "rewards/weighted_chosen": -0.57525634765625,
      "rewards/weighted_margins": 0.5091766119003296,
      "rewards/weighted_rejected": -1.084588646888733,
      "step": 560
    },
    {
      "epoch": 0.29835121695891126,
      "grad_norm": 20.634323120117188,
      "learning_rate": 8.85943037780415e-07,
      "logits/chosen": -0.7054427862167358,
      "logits/rejected": -0.7076568603515625,
      "logps/chosen": -373.78125,
      "logps/rejected": -348.30780029296875,
      "logps/weighted_chosen": -3.1310057640075684,
      "logps/weighted_rejected": -3.516772508621216,
      "loss": 0.624,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -87.689453125,
      "rewards/margins": 18.8154296875,
      "rewards/rejected": -106.50468444824219,
      "rewards/weighted_accuracies": 0.6781250238418579,
      "rewards/weighted_chosen": -0.5498199462890625,
      "rewards/weighted_margins": 0.480499267578125,
      "rewards/weighted_rejected": -1.030615210533142,
      "step": 570
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 19.36369514465332,
      "learning_rate": 8.800694230932884e-07,
      "logits/chosen": -0.6282256841659546,
      "logits/rejected": -0.6369781494140625,
      "logps/chosen": -372.3421936035156,
      "logps/rejected": -367.71563720703125,
      "logps/weighted_chosen": -2.7366943359375,
      "logps/weighted_rejected": -3.2843260765075684,
      "loss": 0.6563,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -82.69844055175781,
      "rewards/margins": 19.826171875,
      "rewards/rejected": -102.53633117675781,
      "rewards/weighted_accuracies": 0.6312500238418579,
      "rewards/weighted_chosen": -0.4037841856479645,
      "rewards/weighted_margins": 0.385406494140625,
      "rewards/weighted_rejected": -0.7893310785293579,
      "step": 580
    },
    {
      "epoch": 0.30881968071185556,
      "grad_norm": 17.115406036376953,
      "learning_rate": 8.740688683062723e-07,
      "logits/chosen": -0.7163940668106079,
      "logits/rejected": -0.729107677936554,
      "logps/chosen": -413.54608154296875,
      "logps/rejected": -381.59375,
      "logps/weighted_chosen": -2.6946043968200684,
      "logps/weighted_rejected": -3.377270460128784,
      "loss": 0.6482,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -84.58671569824219,
      "rewards/margins": 21.25390625,
      "rewards/rejected": -105.85078430175781,
      "rewards/weighted_accuracies": 0.675000011920929,
      "rewards/weighted_chosen": -0.3194335997104645,
      "rewards/weighted_margins": 0.46089476346969604,
      "rewards/weighted_rejected": -0.780108630657196,
      "step": 590
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 21.21268081665039,
      "learning_rate": 8.679433775559215e-07,
      "logits/chosen": -0.6942993402481079,
      "logits/rejected": -0.7400848269462585,
      "logps/chosen": -414.60467529296875,
      "logps/rejected": -400.4125061035156,
      "logps/weighted_chosen": -2.6637206077575684,
      "logps/weighted_rejected": -3.6212401390075684,
      "loss": 0.6532,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -89.740234375,
      "rewards/margins": 25.157617568969727,
      "rewards/rejected": -114.8902359008789,
      "rewards/weighted_accuracies": 0.6499999761581421,
      "rewards/weighted_chosen": -0.4604248106479645,
      "rewards/weighted_margins": 0.40590208768844604,
      "rewards/weighted_rejected": -0.8660339117050171,
      "step": 600
    },
    {
      "epoch": 0.3192881444647998,
      "grad_norm": 52.20835876464844,
      "learning_rate": 8.616949967063871e-07,
      "logits/chosen": -0.6779068112373352,
      "logits/rejected": -0.7114807367324829,
      "logps/chosen": -354.9515686035156,
      "logps/rejected": -375.41094970703125,
      "logps/weighted_chosen": -2.974072217941284,
      "logps/weighted_rejected": -3.5394530296325684,
      "loss": 0.6742,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -86.57109069824219,
      "rewards/margins": 28.051952362060547,
      "rewards/rejected": -114.6156234741211,
      "rewards/weighted_accuracies": 0.653124988079071,
      "rewards/weighted_chosen": -0.4645446836948395,
      "rewards/weighted_margins": 0.36915284395217896,
      "rewards/weighted_rejected": -0.8335021734237671,
      "step": 610
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 22.18308448791504,
      "learning_rate": 8.553258126661154e-07,
      "logits/chosen": -0.7243606448173523,
      "logits/rejected": -0.7389724850654602,
      "logps/chosen": -374.96563720703125,
      "logps/rejected": -378.7953186035156,
      "logps/weighted_chosen": -3.1138672828674316,
      "logps/weighted_rejected": -3.898486375808716,
      "loss": 0.6444,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -95.001953125,
      "rewards/margins": 27.532812118530273,
      "rewards/rejected": -122.50508117675781,
      "rewards/weighted_accuracies": 0.65625,
      "rewards/weighted_chosen": -0.5153747797012329,
      "rewards/weighted_margins": 0.505627453327179,
      "rewards/weighted_rejected": -1.0211060047149658,
      "step": 620
    },
    {
      "epoch": 0.32975660821774405,
      "grad_norm": 40.515403747558594,
      "learning_rate": 8.488379526908368e-07,
      "logits/chosen": -0.719134509563446,
      "logits/rejected": -0.729626476764679,
      "logps/chosen": -393.68359375,
      "logps/rejected": -405.1734313964844,
      "logps/weighted_chosen": -3.021240234375,
      "logps/weighted_rejected": -3.6431641578674316,
      "loss": 0.6548,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -102.9828109741211,
      "rewards/margins": 33.127342224121094,
      "rewards/rejected": -136.0828094482422,
      "rewards/weighted_accuracies": 0.6781250238418579,
      "rewards/weighted_chosen": -0.5511718988418579,
      "rewards/weighted_margins": 0.46650999784469604,
      "rewards/weighted_rejected": -1.018225073814392,
      "step": 630
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 62.75144958496094,
      "learning_rate": 8.422335836730802e-07,
      "logits/chosen": -0.7076660394668579,
      "logits/rejected": -0.7095870971679688,
      "logps/chosen": -371.7562561035156,
      "logps/rejected": -414.140625,
      "logps/weighted_chosen": -2.911572217941284,
      "logps/weighted_rejected": -3.5963377952575684,
      "loss": 0.6254,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -98.0855484008789,
      "rewards/margins": 38.46601486206055,
      "rewards/rejected": -136.51718139648438,
      "rewards/weighted_accuracies": 0.690625011920929,
      "rewards/weighted_chosen": -0.5848175287246704,
      "rewards/weighted_margins": 0.512927234172821,
      "rewards/weighted_rejected": -1.097833275794983,
      "step": 640
    },
    {
      "epoch": 0.3402250719706883,
      "grad_norm": 41.57294845581055,
      "learning_rate": 8.355149114184485e-07,
      "logits/chosen": -0.7601822018623352,
      "logits/rejected": -0.7432464361190796,
      "logps/chosen": -409.87188720703125,
      "logps/rejected": -429.62188720703125,
      "logps/weighted_chosen": -3.059814453125,
      "logps/weighted_rejected": -3.5130615234375,
      "loss": 0.5909,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": -102.44218444824219,
      "rewards/margins": 44.93242263793945,
      "rewards/rejected": -147.43515014648438,
      "rewards/weighted_accuracies": 0.6968749761581421,
      "rewards/weighted_chosen": -0.5836563110351562,
      "rewards/weighted_margins": 0.5829010009765625,
      "rewards/weighted_rejected": -1.166589379310608,
      "step": 650
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 45.57954025268555,
      "learning_rate": 8.286841799088963e-07,
      "logits/chosen": -0.802441418170929,
      "logits/rejected": -0.7937591671943665,
      "logps/chosen": -387.9624938964844,
      "logps/rejected": -394.9937438964844,
      "logps/weighted_chosen": -2.703906297683716,
      "logps/weighted_rejected": -3.4778809547424316,
      "loss": 0.6382,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -103.9437484741211,
      "rewards/margins": 27.896678924560547,
      "rewards/rejected": -131.8410186767578,
      "rewards/weighted_accuracies": 0.675000011920929,
      "rewards/weighted_chosen": -0.571490466594696,
      "rewards/weighted_margins": 0.549023449420929,
      "rewards/weighted_rejected": -1.12017822265625,
      "step": 660
    },
    {
      "epoch": 0.35069353572363254,
      "grad_norm": 37.978858947753906,
      "learning_rate": 8.217436705532599e-07,
      "logits/chosen": -0.7893584966659546,
      "logits/rejected": -0.8069244623184204,
      "logps/chosen": -414.5484313964844,
      "logps/rejected": -398.55780029296875,
      "logps/weighted_chosen": -2.7833495140075684,
      "logps/weighted_rejected": -3.48388671875,
      "loss": 0.5742,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -111.7906265258789,
      "rewards/margins": 23.006250381469727,
      "rewards/rejected": -134.8093719482422,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.4879394471645355,
      "rewards/weighted_margins": 0.6075531244277954,
      "rewards/weighted_rejected": -1.095251441001892,
      "step": 670
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 731.5437622070312,
      "learning_rate": 8.14695701425284e-07,
      "logits/chosen": -0.7789825201034546,
      "logits/rejected": -0.8169800043106079,
      "logps/chosen": -417.2015686035156,
      "logps/rejected": -403.9624938964844,
      "logps/weighted_chosen": -2.839160203933716,
      "logps/weighted_rejected": -3.607421875,
      "loss": 0.6329,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -111.77265930175781,
      "rewards/margins": 29.793750762939453,
      "rewards/rejected": -141.5343780517578,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.5055176019668579,
      "rewards/weighted_margins": 0.5288635492324829,
      "rewards/weighted_rejected": -1.034582495689392,
      "step": 680
    },
    {
      "epoch": 0.3611619994765768,
      "grad_norm": 23.96302604675293,
      "learning_rate": 8.075426264894046e-07,
      "logits/chosen": -0.7564483880996704,
      "logits/rejected": -0.785125732421875,
      "logps/chosen": -418.2562561035156,
      "logps/rejected": -427.62030029296875,
      "logps/weighted_chosen": -2.916186571121216,
      "logps/weighted_rejected": -4.193115234375,
      "loss": 0.5335,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -112.79512023925781,
      "rewards/margins": 37.262107849121094,
      "rewards/rejected": -150.0929718017578,
      "rewards/weighted_accuracies": 0.737500011920929,
      "rewards/weighted_chosen": -0.5042785406112671,
      "rewards/weighted_margins": 0.7850341796875,
      "rewards/weighted_rejected": -1.2886931896209717,
      "step": 690
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 30.07147789001465,
      "learning_rate": 8.002868348145435e-07,
      "logits/chosen": -0.7395874261856079,
      "logits/rejected": -0.7387515902519226,
      "logps/chosen": -413.5406188964844,
      "logps/rejected": -407.9750061035156,
      "logps/weighted_chosen": -2.937329053878784,
      "logps/weighted_rejected": -3.2009034156799316,
      "loss": 0.6394,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -113.3121109008789,
      "rewards/margins": 26.630273818969727,
      "rewards/rejected": -139.94921875,
      "rewards/weighted_accuracies": 0.675000011920929,
      "rewards/weighted_chosen": -0.5579589605331421,
      "rewards/weighted_margins": 0.46031492948532104,
      "rewards/weighted_rejected": -1.017907738685608,
      "step": 700
    },
    {
      "epoch": 0.3716304632295211,
      "grad_norm": 38.97393035888672,
      "learning_rate": 7.92930749776179e-07,
      "logits/chosen": -0.7464812994003296,
      "logits/rejected": -0.7725540399551392,
      "logps/chosen": -388.9750061035156,
      "logps/rejected": -398.98126220703125,
      "logps/weighted_chosen": -3.270263671875,
      "logps/weighted_rejected": -3.752880811691284,
      "loss": 0.6589,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -117.39530944824219,
      "rewards/margins": 21.184764862060547,
      "rewards/rejected": -138.5859375,
      "rewards/weighted_accuracies": 0.621874988079071,
      "rewards/weighted_chosen": -0.5594116449356079,
      "rewards/weighted_margins": 0.44041746854782104,
      "rewards/weighted_rejected": -0.9994751214981079,
      "step": 710
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 21.152732849121094,
      "learning_rate": 7.854768282469582e-07,
      "logits/chosen": -0.7833511233329773,
      "logits/rejected": -0.8182953000068665,
      "logps/chosen": -379.1953125,
      "logps/rejected": -416.29998779296875,
      "logps/weighted_chosen": -2.8000245094299316,
      "logps/weighted_rejected": -3.4732422828674316,
      "loss": 0.6116,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -101.853515625,
      "rewards/margins": 36.69355392456055,
      "rewards/rejected": -138.53866577148438,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.414450079202652,
      "rewards/weighted_margins": 0.53369140625,
      "rewards/weighted_rejected": -0.948101818561554,
      "step": 720
    },
    {
      "epoch": 0.38209892698246534,
      "grad_norm": 267.910400390625,
      "learning_rate": 7.779275597761215e-07,
      "logits/chosen": -0.7315715551376343,
      "logits/rejected": -0.778778076171875,
      "logps/chosen": -380.20623779296875,
      "logps/rejected": -408.5703125,
      "logps/weighted_chosen": -2.9701170921325684,
      "logps/weighted_rejected": -3.582080125808716,
      "loss": 0.5809,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -104.14375305175781,
      "rewards/margins": 35.924217224121094,
      "rewards/rejected": -140.08749389648438,
      "rewards/weighted_accuracies": 0.7093750238418579,
      "rewards/weighted_chosen": -0.3895111083984375,
      "rewards/weighted_margins": 0.6164184808731079,
      "rewards/weighted_rejected": -1.0058715343475342,
      "step": 730
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 67.87287902832031,
      "learning_rate": 7.702854657580126e-07,
      "logits/chosen": -0.7619155645370483,
      "logits/rejected": -0.772930920124054,
      "logps/chosen": -405.0328063964844,
      "logps/rejected": -391.39373779296875,
      "logps/weighted_chosen": -2.9206786155700684,
      "logps/weighted_rejected": -3.8096680641174316,
      "loss": 0.6077,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -110.5816421508789,
      "rewards/margins": 26.568944931030273,
      "rewards/rejected": -137.11093139648438,
      "rewards/weighted_accuracies": 0.671875,
      "rewards/weighted_chosen": -0.441131591796875,
      "rewards/weighted_margins": 0.5765472650527954,
      "rewards/weighted_rejected": -1.0178649425506592,
      "step": 740
    },
    {
      "epoch": 0.3925673907354096,
      "grad_norm": 46.017822265625,
      "learning_rate": 7.625530985899547e-07,
      "logits/chosen": -0.722912609577179,
      "logits/rejected": -0.7346221804618835,
      "logps/chosen": -379.953125,
      "logps/rejected": -388.6000061035156,
      "logps/weighted_chosen": -2.8084716796875,
      "logps/weighted_rejected": -3.8277587890625,
      "loss": 0.5923,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -113.12422180175781,
      "rewards/margins": 30.504297256469727,
      "rewards/rejected": -143.63516235351562,
      "rewards/weighted_accuracies": 0.6937500238418579,
      "rewards/weighted_chosen": -0.549816906452179,
      "rewards/weighted_margins": 0.5676025152206421,
      "rewards/weighted_rejected": -1.1176025867462158,
      "step": 750
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 29.748411178588867,
      "learning_rate": 7.547330408197694e-07,
      "logits/chosen": -0.7272213101387024,
      "logits/rejected": -0.7657715082168579,
      "logps/chosen": -423.21875,
      "logps/rejected": -402.828125,
      "logps/weighted_chosen": -2.842236280441284,
      "logps/weighted_rejected": -3.7777099609375,
      "loss": 0.6328,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -127.716796875,
      "rewards/margins": 18.883983612060547,
      "rewards/rejected": -146.58828735351562,
      "rewards/weighted_accuracies": 0.671875,
      "rewards/weighted_chosen": -0.5538665652275085,
      "rewards/weighted_margins": 0.509082019329071,
      "rewards/weighted_rejected": -1.063262939453125,
      "step": 760
    },
    {
      "epoch": 0.40303585448835383,
      "grad_norm": 36.96773147583008,
      "learning_rate": 7.468279042832271e-07,
      "logits/chosen": -0.73992919921875,
      "logits/rejected": -0.757153332233429,
      "logps/chosen": -398.8296813964844,
      "logps/rejected": -451.765625,
      "logps/weighted_chosen": -2.8755125999450684,
      "logps/weighted_rejected": -3.5738768577575684,
      "loss": 0.6346,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": -119.18359375,
      "rewards/margins": 43.133201599121094,
      "rewards/rejected": -162.3484344482422,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.599902331829071,
      "rewards/weighted_margins": 0.4711242616176605,
      "rewards/weighted_rejected": -1.0711486339569092,
      "step": 770
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 31.340259552001953,
      "learning_rate": 7.388403292317154e-07,
      "logits/chosen": -0.7140892148017883,
      "logits/rejected": -0.7637573480606079,
      "logps/chosen": -424.03436279296875,
      "logps/rejected": -419.3531188964844,
      "logps/weighted_chosen": -2.7535400390625,
      "logps/weighted_rejected": -3.53564453125,
      "loss": 0.5954,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -122.986328125,
      "rewards/margins": 30.427734375,
      "rewards/rejected": -153.4421844482422,
      "rewards/weighted_accuracies": 0.699999988079071,
      "rewards/weighted_chosen": -0.5587218999862671,
      "rewards/weighted_margins": 0.592761218547821,
      "rewards/weighted_rejected": -1.1515014171600342,
      "step": 780
    },
    {
      "epoch": 0.4135043182412981,
      "grad_norm": 22.861244201660156,
      "learning_rate": 7.307729834504154e-07,
      "logits/chosen": -0.7405792474746704,
      "logits/rejected": -0.7957550287246704,
      "logps/chosen": -408.3984375,
      "logps/rejected": -426.87811279296875,
      "logps/weighted_chosen": -2.751171827316284,
      "logps/weighted_rejected": -3.7800049781799316,
      "loss": 0.6317,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -126.57695007324219,
      "rewards/margins": 34.237892150878906,
      "rewards/rejected": -160.8195343017578,
      "rewards/weighted_accuracies": 0.699999988079071,
      "rewards/weighted_chosen": -0.590838611125946,
      "rewards/weighted_margins": 0.5351654291152954,
      "rewards/weighted_rejected": -1.1267273426055908,
      "step": 790
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 47.63352966308594,
      "learning_rate": 7.226285613672847e-07,
      "logits/chosen": -0.6944122314453125,
      "logits/rejected": -0.7281158566474915,
      "logps/chosen": -399.3265686035156,
      "logps/rejected": -450.2906188964844,
      "logps/weighted_chosen": -2.772753953933716,
      "logps/weighted_rejected": -3.867480516433716,
      "loss": 0.604,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -121.8121109008789,
      "rewards/margins": 49.55859375,
      "rewards/rejected": -171.38046264648438,
      "rewards/weighted_accuracies": 0.7093750238418579,
      "rewards/weighted_chosen": -0.5663421750068665,
      "rewards/weighted_margins": 0.5819091796875,
      "rewards/weighted_rejected": -1.148461937904358,
      "step": 800
    },
    {
      "epoch": 0.4239727819942423,
      "grad_norm": 19.353519439697266,
      "learning_rate": 7.144097831531398e-07,
      "logits/chosen": -0.6676399111747742,
      "logits/rejected": -0.6886535882949829,
      "logps/chosen": -402.2984313964844,
      "logps/rejected": -432.3531188964844,
      "logps/weighted_chosen": -2.8445801734924316,
      "logps/weighted_rejected": -3.7008299827575684,
      "loss": 0.6044,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -132.17343139648438,
      "rewards/margins": 36.122459411621094,
      "rewards/rejected": -168.2585906982422,
      "rewards/weighted_accuracies": 0.690625011920929,
      "rewards/weighted_chosen": -0.573992908000946,
      "rewards/weighted_margins": 0.569226086139679,
      "rewards/weighted_rejected": -1.143530249595642,
      "step": 810
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 29.685457229614258,
      "learning_rate": 7.061193938131396e-07,
      "logits/chosen": -0.6138809323310852,
      "logits/rejected": -0.6571716070175171,
      "logps/chosen": -440.6265563964844,
      "logps/rejected": -433.95001220703125,
      "logps/weighted_chosen": -3.1656250953674316,
      "logps/weighted_rejected": -3.6409668922424316,
      "loss": 0.5893,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -140.87890625,
      "rewards/margins": 27.572656631469727,
      "rewards/rejected": -168.5109405517578,
      "rewards/weighted_accuracies": 0.684374988079071,
      "rewards/weighted_chosen": -0.5785247683525085,
      "rewards/weighted_margins": 0.638214111328125,
      "rewards/weighted_rejected": -1.216912865638733,
      "step": 820
    },
    {
      "epoch": 0.4344412457471866,
      "grad_norm": 28.759309768676758,
      "learning_rate": 6.977601622699789e-07,
      "logits/chosen": -0.6784942746162415,
      "logits/rejected": -0.7357269525527954,
      "logps/chosen": -413.9476623535156,
      "logps/rejected": -463.1390686035156,
      "logps/weighted_chosen": -3.047466993331909,
      "logps/weighted_rejected": -3.9415526390075684,
      "loss": 0.5322,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -129.1398468017578,
      "rewards/margins": 55.296485900878906,
      "rewards/rejected": -184.421875,
      "rewards/weighted_accuracies": 0.7250000238418579,
      "rewards/weighted_chosen": -0.4962509274482727,
      "rewards/weighted_margins": 0.755261242389679,
      "rewards/weighted_rejected": -1.2515869140625,
      "step": 830
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 41.56013107299805,
      "learning_rate": 6.893348804390882e-07,
      "logits/chosen": -0.809680163860321,
      "logits/rejected": -0.809558093547821,
      "logps/chosen": -445.3421936035156,
      "logps/rejected": -454.68438720703125,
      "logps/weighted_chosen": -3.277099609375,
      "logps/weighted_rejected": -3.73974609375,
      "loss": 0.5971,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -150.06405639648438,
      "rewards/margins": 44.780860900878906,
      "rewards/rejected": -194.8703155517578,
      "rewards/weighted_accuracies": 0.6968749761581421,
      "rewards/weighted_chosen": -0.6049469113349915,
      "rewards/weighted_margins": 0.59771728515625,
      "rewards/weighted_rejected": -1.2025878429412842,
      "step": 840
    },
    {
      "epoch": 0.44490970950013087,
      "grad_norm": 21.975399017333984,
      "learning_rate": 6.808463622961578e-07,
      "logits/chosen": -0.838226318359375,
      "logits/rejected": -0.878466784954071,
      "logps/chosen": -470.1796875,
      "logps/rejected": -514.8531494140625,
      "logps/weighted_chosen": -3.189526319503784,
      "logps/weighted_rejected": -4.007568359375,
      "loss": 0.5758,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -177.51953125,
      "rewards/margins": 55.28593826293945,
      "rewards/rejected": -232.81094360351562,
      "rewards/weighted_accuracies": 0.706250011920929,
      "rewards/weighted_chosen": -0.6749969720840454,
      "rewards/weighted_margins": 0.754681408405304,
      "rewards/weighted_rejected": -1.429284691810608,
      "step": 850
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 22.851951599121094,
      "learning_rate": 6.722974429372925e-07,
      "logits/chosen": -0.839556872844696,
      "logits/rejected": -0.871502697467804,
      "logps/chosen": -531.9484252929688,
      "logps/rejected": -532.2999877929688,
      "logps/weighted_chosen": -3.0276856422424316,
      "logps/weighted_rejected": -4.387353420257568,
      "loss": 0.5743,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -228.4031219482422,
      "rewards/margins": 41.707420349121094,
      "rewards/rejected": -270.15155029296875,
      "rewards/weighted_accuracies": 0.7406250238418579,
      "rewards/weighted_chosen": -0.8227783441543579,
      "rewards/weighted_margins": 0.8454529047012329,
      "rewards/weighted_rejected": -1.6683471202850342,
      "step": 860
    },
    {
      "epoch": 0.4553781732530751,
      "grad_norm": 34.66981887817383,
      "learning_rate": 6.636909776321128e-07,
      "logits/chosen": -0.8797591924667358,
      "logits/rejected": -0.864776611328125,
      "logps/chosen": -461.953125,
      "logps/rejected": -522.6062622070312,
      "logps/weighted_chosen": -3.339587450027466,
      "logps/weighted_rejected": -4.197558403015137,
      "loss": 0.5676,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -200.29843139648438,
      "rewards/margins": 57.83671951293945,
      "rewards/rejected": -258.1499938964844,
      "rewards/weighted_accuracies": 0.706250011920929,
      "rewards/weighted_chosen": -0.8672088384628296,
      "rewards/weighted_margins": 0.72900390625,
      "rewards/weighted_rejected": -1.596582055091858,
      "step": 870
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 402.62530517578125,
      "learning_rate": 6.550298408701174e-07,
      "logits/chosen": -0.860809326171875,
      "logits/rejected": -0.9049621820449829,
      "logps/chosen": -489.68438720703125,
      "logps/rejected": -540.2687377929688,
      "logps/weighted_chosen": -3.544872999191284,
      "logps/weighted_rejected": -4.589892387390137,
      "loss": 0.596,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -202.95565795898438,
      "rewards/margins": 54.89179611206055,
      "rewards/rejected": -257.78515625,
      "rewards/weighted_accuracies": 0.6968749761581421,
      "rewards/weighted_chosen": -0.791333019733429,
      "rewards/weighted_margins": 0.6679824590682983,
      "rewards/weighted_rejected": -1.4595428705215454,
      "step": 880
    },
    {
      "epoch": 0.46584663700601936,
      "grad_norm": 342.0785827636719,
      "learning_rate": 6.463169254006276e-07,
      "logits/chosen": -0.8872314691543579,
      "logits/rejected": -0.918865978717804,
      "logps/chosen": -482.1578063964844,
      "logps/rejected": -486.10467529296875,
      "logps/weighted_chosen": -3.362231492996216,
      "logps/weighted_rejected": -4.306298732757568,
      "loss": 0.5402,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -201.04843139648438,
      "rewards/margins": 38.162498474121094,
      "rewards/rejected": -239.21133422851562,
      "rewards/weighted_accuracies": 0.7281249761581421,
      "rewards/weighted_chosen": -0.7565551996231079,
      "rewards/weighted_margins": 0.8916381597518921,
      "rewards/weighted_rejected": -1.6484863758087158,
      "step": 890
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 84.898193359375,
      "learning_rate": 6.375551412666326e-07,
      "logits/chosen": -0.8932129144668579,
      "logits/rejected": -0.909472644329071,
      "logps/chosen": -485.47967529296875,
      "logps/rejected": -491.5093688964844,
      "logps/weighted_chosen": -3.119433641433716,
      "logps/weighted_rejected": -4.143603324890137,
      "loss": 0.6461,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -211.3484344482422,
      "rewards/margins": 23.850391387939453,
      "rewards/rejected": -235.22225952148438,
      "rewards/weighted_accuracies": 0.653124988079071,
      "rewards/weighted_chosen": -0.972399890422821,
      "rewards/weighted_margins": 0.627001941204071,
      "rewards/weighted_rejected": -1.599572777748108,
      "step": 900
    },
    {
      "epoch": 0.4763151007589636,
      "grad_norm": 242.4366455078125,
      "learning_rate": 6.287474148328583e-07,
      "logits/chosen": -0.855010986328125,
      "logits/rejected": -0.8482757806777954,
      "logps/chosen": -457.9312438964844,
      "logps/rejected": -459.96875,
      "logps/weighted_chosen": -3.5470948219299316,
      "logps/weighted_rejected": -4.787109375,
      "loss": 0.6583,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -187.0656280517578,
      "rewards/margins": 25.948827743530273,
      "rewards/rejected": -213.04296875,
      "rewards/weighted_accuracies": 0.628125011920929,
      "rewards/weighted_chosen": -0.9626404047012329,
      "rewards/weighted_margins": 0.5732177495956421,
      "rewards/weighted_rejected": -1.5358765125274658,
      "step": 910
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 163.63558959960938,
      "learning_rate": 6.198966878083857e-07,
      "logits/chosen": -0.8556182980537415,
      "logits/rejected": -0.8749450445175171,
      "logps/chosen": -450.89373779296875,
      "logps/rejected": -486.953125,
      "logps/weighted_chosen": -3.41851806640625,
      "logps/weighted_rejected": -4.127148628234863,
      "loss": 0.598,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -179.65234375,
      "rewards/margins": 40.17695236206055,
      "rewards/rejected": -219.8562469482422,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.7931762933731079,
      "rewards/weighted_margins": 0.6718719601631165,
      "rewards/weighted_rejected": -1.465295433998108,
      "step": 920
    },
    {
      "epoch": 0.48678356451190785,
      "grad_norm": 299.4981689453125,
      "learning_rate": 6.110059162641439e-07,
      "logits/chosen": -0.8365691900253296,
      "logits/rejected": -0.8519042730331421,
      "logps/chosen": -456.8046875,
      "logps/rejected": -481.1812438964844,
      "logps/weighted_chosen": -2.855395555496216,
      "logps/weighted_rejected": -3.79541015625,
      "loss": 0.5691,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -174.2941436767578,
      "rewards/margins": 39.63788986206055,
      "rewards/rejected": -213.9421844482422,
      "rewards/weighted_accuracies": 0.699999988079071,
      "rewards/weighted_chosen": -0.6965240240097046,
      "rewards/weighted_margins": 0.643646240234375,
      "rewards/weighted_rejected": -1.340509057044983,
      "step": 930
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 139.11936950683594,
      "learning_rate": 6.020780696456059e-07,
      "logits/chosen": -0.818524181842804,
      "logits/rejected": -0.8568817377090454,
      "logps/chosen": -432.6015625,
      "logps/rejected": -482.3765563964844,
      "logps/weighted_chosen": -2.738085985183716,
      "logps/weighted_rejected": -3.7889161109924316,
      "loss": 0.6145,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -161.802734375,
      "rewards/margins": 53.56367111206055,
      "rewards/rejected": -215.30624389648438,
      "rewards/weighted_accuracies": 0.6812499761581421,
      "rewards/weighted_chosen": -0.751434326171875,
      "rewards/weighted_margins": 0.5673767328262329,
      "rewards/weighted_rejected": -1.319006323814392,
      "step": 940
    },
    {
      "epoch": 0.49725202826485215,
      "grad_norm": 2830.01513671875,
      "learning_rate": 5.931161297810185e-07,
      "logits/chosen": -0.897666335105896,
      "logits/rejected": -0.912066638469696,
      "logps/chosen": -450.53125,
      "logps/rejected": -480.78436279296875,
      "logps/weighted_chosen": -3.4595704078674316,
      "logps/weighted_rejected": -4.21240234375,
      "loss": 0.6411,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -177.23983764648438,
      "rewards/margins": 42.097267150878906,
      "rewards/rejected": -219.32852172851562,
      "rewards/weighted_accuracies": 0.6468750238418579,
      "rewards/weighted_chosen": -0.7506958246231079,
      "rewards/weighted_margins": 0.633044421672821,
      "rewards/weighted_rejected": -1.3831908702850342,
      "step": 950
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 294.5556640625,
      "learning_rate": 5.841230898854959e-07,
      "logits/chosen": -0.8446868658065796,
      "logits/rejected": -0.860076904296875,
      "logps/chosen": -493.5140686035156,
      "logps/rejected": -503.9921875,
      "logps/weighted_chosen": -3.2405028343200684,
      "logps/weighted_rejected": -4.144775390625,
      "loss": 0.6412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -193.3210906982422,
      "rewards/margins": 49.00800704956055,
      "rewards/rejected": -242.33749389648438,
      "rewards/weighted_accuracies": 0.690625011920929,
      "rewards/weighted_chosen": -0.8943328857421875,
      "rewards/weighted_margins": 0.655200183391571,
      "rewards/weighted_rejected": -1.549963355064392,
      "step": 960
    },
    {
      "epoch": 0.5077204920177963,
      "grad_norm": 1039.4329833984375,
      "learning_rate": 5.751019535613102e-07,
      "logits/chosen": -0.816241443157196,
      "logits/rejected": -0.831072986125946,
      "logps/chosen": -434.7757873535156,
      "logps/rejected": -476.2437438964844,
      "logps/weighted_chosen": -3.277026414871216,
      "logps/weighted_rejected": -4.411572456359863,
      "loss": 0.6001,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -174.7824249267578,
      "rewards/margins": 54.372657775878906,
      "rewards/rejected": -229.171875,
      "rewards/weighted_accuracies": 0.699999988079071,
      "rewards/weighted_chosen": -0.8984619379043579,
      "rewards/weighted_margins": 0.785168468952179,
      "rewards/weighted_rejected": -1.68408203125,
      "step": 970
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 63.553932189941406,
      "learning_rate": 5.660557337947117e-07,
      "logits/chosen": -0.8274612426757812,
      "logits/rejected": -0.832653820514679,
      "logps/chosen": -484.9125061035156,
      "logps/rejected": -491.22344970703125,
      "logps/weighted_chosen": -2.8631834983825684,
      "logps/weighted_rejected": -3.8985838890075684,
      "loss": 0.5745,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -190.4035186767578,
      "rewards/margins": 43.454689025878906,
      "rewards/rejected": -233.85311889648438,
      "rewards/weighted_accuracies": 0.7093750238418579,
      "rewards/weighted_chosen": -0.7985321283340454,
      "rewards/weighted_margins": 0.6566101312637329,
      "rewards/weighted_rejected": -1.454565405845642,
      "step": 980
    },
    {
      "epoch": 0.5181889557707406,
      "grad_norm": 474.792724609375,
      "learning_rate": 5.569874519496174e-07,
      "logits/chosen": -0.8315078616142273,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -461.2124938964844,
      "logps/rejected": -500.48907470703125,
      "logps/weighted_chosen": -3.374279737472534,
      "logps/weighted_rejected": -4.527905464172363,
      "loss": 0.5744,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -186.0480499267578,
      "rewards/margins": 47.295310974121094,
      "rewards/rejected": -233.33749389648438,
      "rewards/weighted_accuracies": 0.721875011920929,
      "rewards/weighted_chosen": -0.7532867193222046,
      "rewards/weighted_margins": 0.7921081781387329,
      "rewards/weighted_rejected": -1.5448729991912842,
      "step": 990
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 46.039974212646484,
      "learning_rate": 5.47900136758499e-07,
      "logits/chosen": -0.7764679193496704,
      "logits/rejected": -0.8226165771484375,
      "logps/chosen": -472.5015563964844,
      "logps/rejected": -484.1937561035156,
      "logps/weighted_chosen": -3.4063720703125,
      "logps/weighted_rejected": -4.413744926452637,
      "loss": 0.5675,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": -204.89297485351562,
      "rewards/margins": 36.982810974121094,
      "rewards/rejected": -241.89883422851562,
      "rewards/weighted_accuracies": 0.703125,
      "rewards/weighted_chosen": -0.9316650629043579,
      "rewards/weighted_margins": 0.817004382610321,
      "rewards/weighted_rejected": -1.748803734779358,
      "step": 1000
    },
    {
      "epoch": 0.5234231876472127,
      "eval_logits/chosen": -0.8910121917724609,
      "eval_logits/rejected": -0.9021777510643005,
      "eval_logps/chosen": -512.3715209960938,
      "eval_logps/rejected": -536.406982421875,
      "eval_logps/weighted_chosen": -3.448028326034546,
      "eval_logps/weighted_rejected": -4.361933708190918,
      "eval_loss": 0.5965847373008728,
      "eval_rewards/accuracies": 0.5910000205039978,
      "eval_rewards/chosen": -223.7135009765625,
      "eval_rewards/margins": 45.03103256225586,
      "eval_rewards/rejected": -268.75,
      "eval_rewards/weighted_accuracies": 0.690500020980835,
      "eval_rewards/weighted_chosen": -0.9871044754981995,
      "eval_rewards/weighted_margins": 0.7057854533195496,
      "eval_rewards/weighted_rejected": -1.6928898096084595,
      "eval_runtime": 1512.2328,
      "eval_samples_per_second": 1.323,
      "eval_steps_per_second": 0.331,
      "step": 1000
    }
  ],
  "logging_steps": 10,
  "max_steps": 1911,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}