genv3pair1NoGT_1.5B_cdpo_ebs32_lr5e-06_beta0.1_epoch16.0_42/trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 16.0,
  "eval_steps": 20,
  "global_step": 2864,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00558659217877095,
      "grad_norm": 28.604416347843078,
      "learning_rate": 1.7421602787456446e-08,
      "logits/chosen": -2.4004337787628174,
      "logits/rejected": -2.5142080783843994,
      "logps/chosen": -60.584632873535156,
      "logps/rejected": -42.787071228027344,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0111731843575419,
      "grad_norm": 29.26993829736196,
      "learning_rate": 3.484320557491289e-08,
      "logits/chosen": -2.4949634075164795,
      "logits/rejected": -2.4507484436035156,
      "logps/chosen": -19.39887046813965,
      "logps/rejected": -25.19920539855957,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.01675977653631285,
      "grad_norm": 28.686495829946026,
      "learning_rate": 5.226480836236935e-08,
      "logits/chosen": -2.282752275466919,
      "logits/rejected": -2.2741856575012207,
      "logps/chosen": -27.87565803527832,
      "logps/rejected": -39.4168701171875,
      "loss": 0.688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014356613159179688,
      "rewards/margins": 0.014356613159179688,
      "rewards/rejected": 0.0,
      "step": 3
    },
    {
      "epoch": 0.0223463687150838,
      "grad_norm": 32.65066823843522,
      "learning_rate": 6.968641114982578e-08,
      "logits/chosen": -2.0593926906585693,
      "logits/rejected": -2.0425660610198975,
      "logps/chosen": -50.47325134277344,
      "logps/rejected": -44.69379425048828,
      "loss": 0.6942,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.008958149701356888,
      "rewards/margins": -0.008958149701356888,
      "rewards/rejected": 0.0,
      "step": 4
    },
    {
      "epoch": 0.027932960893854747,
      "grad_norm": 28.79963798905487,
      "learning_rate": 8.710801393728224e-08,
      "logits/chosen": -2.2757158279418945,
      "logits/rejected": -2.304229736328125,
      "logps/chosen": -38.040199279785156,
      "logps/rejected": -50.2257080078125,
      "loss": 0.6977,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0018065925687551498,
      "rewards/margins": 0.0018065925687551498,
      "rewards/rejected": 0.0,
      "step": 5
    },
    {
      "epoch": 0.0335195530726257,
      "grad_norm": 26.34828378708642,
      "learning_rate": 1.045296167247387e-07,
      "logits/chosen": -2.7875425815582275,
      "logits/rejected": -2.446885108947754,
      "logps/chosen": -17.942790985107422,
      "logps/rejected": -63.80572509765625,
      "loss": 0.6939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.031739283353090286,
      "rewards/margins": 0.031739283353090286,
      "rewards/rejected": 0.0,
      "step": 6
    },
    {
      "epoch": 0.03910614525139665,
      "grad_norm": 30.224428377991995,
      "learning_rate": 1.2195121951219514e-07,
      "logits/chosen": -2.6455585956573486,
      "logits/rejected": -2.5657124519348145,
      "logps/chosen": -18.65185546875,
      "logps/rejected": -20.791696548461914,
      "loss": 0.6856,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.017557192593812943,
      "rewards/margins": -0.017557192593812943,
      "rewards/rejected": 0.0,
      "step": 7
    },
    {
      "epoch": 0.0446927374301676,
      "grad_norm": 31.916338037035274,
      "learning_rate": 1.3937282229965157e-07,
      "logits/chosen": -2.543046474456787,
      "logits/rejected": -2.564054012298584,
      "logps/chosen": -20.067359924316406,
      "logps/rejected": -29.676353454589844,
      "loss": 0.6902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017066527158021927,
      "rewards/margins": 0.017066527158021927,
      "rewards/rejected": 0.0,
      "step": 8
    },
    {
      "epoch": 0.05027932960893855,
      "grad_norm": 26.85107277523636,
      "learning_rate": 1.5679442508710802e-07,
      "logits/chosen": -2.590404510498047,
      "logits/rejected": -2.5011065006256104,
      "logps/chosen": -27.8802433013916,
      "logps/rejected": -36.20390319824219,
      "loss": 0.6957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015048694796860218,
      "rewards/margins": 0.015048694796860218,
      "rewards/rejected": 0.0,
      "step": 9
    },
    {
      "epoch": 0.055865921787709494,
      "grad_norm": 27.31543664913514,
      "learning_rate": 1.7421602787456448e-07,
      "logits/chosen": -2.3443429470062256,
      "logits/rejected": -2.223762035369873,
      "logps/chosen": -53.466773986816406,
      "logps/rejected": -55.203060150146484,
      "loss": 0.6925,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.019601155072450638,
      "rewards/margins": -0.019601155072450638,
      "rewards/rejected": 0.0,
      "step": 10
    },
    {
      "epoch": 0.061452513966480445,
      "grad_norm": 28.295730637033238,
      "learning_rate": 1.916376306620209e-07,
      "logits/chosen": -2.4080286026000977,
      "logits/rejected": -2.274735927581787,
      "logps/chosen": -18.83185577392578,
      "logps/rejected": -27.68811798095703,
      "loss": 0.6955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02140343189239502,
      "rewards/margins": 0.02140343189239502,
      "rewards/rejected": 0.0,
      "step": 11
    },
    {
      "epoch": 0.0670391061452514,
      "grad_norm": 25.963424151512818,
      "learning_rate": 2.090592334494774e-07,
      "logits/chosen": -2.790097951889038,
      "logits/rejected": -2.4678592681884766,
      "logps/chosen": -15.408470153808594,
      "logps/rejected": -42.94584655761719,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002624010667204857,
      "rewards/margins": -0.002624010667204857,
      "rewards/rejected": 0.0,
      "step": 12
    },
    {
      "epoch": 0.07262569832402235,
      "grad_norm": 28.80222122605164,
      "learning_rate": 2.2648083623693382e-07,
      "logits/chosen": -2.3777613639831543,
      "logits/rejected": -2.094010829925537,
      "logps/chosen": -33.65294647216797,
      "logps/rejected": -53.04192352294922,
      "loss": 0.684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04453440010547638,
      "rewards/margins": 0.04453440010547638,
      "rewards/rejected": 0.0,
      "step": 13
    },
    {
      "epoch": 0.0782122905027933,
      "grad_norm": 28.96230892831842,
      "learning_rate": 2.439024390243903e-07,
      "logits/chosen": -2.5680267810821533,
      "logits/rejected": -2.397420883178711,
      "logps/chosen": -18.800567626953125,
      "logps/rejected": -48.686302185058594,
      "loss": 0.6863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.025628473609685898,
      "rewards/margins": 0.025628473609685898,
      "rewards/rejected": 0.0,
      "step": 14
    },
    {
      "epoch": 0.08379888268156424,
      "grad_norm": 29.652792746584346,
      "learning_rate": 2.6132404181184673e-07,
      "logits/chosen": -2.1543562412261963,
      "logits/rejected": -2.0685665607452393,
      "logps/chosen": -42.42938995361328,
      "logps/rejected": -50.57514953613281,
      "loss": 0.6883,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -4.527531564235687e-05,
      "rewards/margins": -4.527531564235687e-05,
      "rewards/rejected": 0.0,
      "step": 15
    },
    {
      "epoch": 0.0893854748603352,
      "grad_norm": 26.615257235619225,
      "learning_rate": 2.7874564459930313e-07,
      "logits/chosen": -2.0456478595733643,
      "logits/rejected": -2.078857660293579,
      "logps/chosen": -32.74793243408203,
      "logps/rejected": -39.494422912597656,
      "loss": 0.6917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011205173097550869,
      "rewards/margins": 0.011205173097550869,
      "rewards/rejected": 0.0,
      "step": 16
    },
    {
      "epoch": 0.09497206703910614,
      "grad_norm": 30.684521135094815,
      "learning_rate": 2.961672473867596e-07,
      "logits/chosen": -2.7974939346313477,
      "logits/rejected": -2.156824827194214,
      "logps/chosen": -24.319869995117188,
      "logps/rejected": -70.72748565673828,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00849900208413601,
      "rewards/margins": 0.00849900208413601,
      "rewards/rejected": 0.0,
      "step": 17
    },
    {
      "epoch": 0.1005586592178771,
      "grad_norm": 28.88086157850331,
      "learning_rate": 3.1358885017421604e-07,
      "logits/chosen": -2.507537603378296,
      "logits/rejected": -2.3943874835968018,
      "logps/chosen": -36.93671798706055,
      "logps/rejected": -38.78935241699219,
      "loss": 0.6773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.023644831031560898,
      "rewards/margins": 0.023644831031560898,
      "rewards/rejected": 0.0,
      "step": 18
    },
    {
      "epoch": 0.10614525139664804,
      "grad_norm": 27.872169165022918,
      "learning_rate": 3.310104529616725e-07,
      "logits/chosen": -2.5571093559265137,
      "logits/rejected": -2.620222330093384,
      "logps/chosen": -24.047679901123047,
      "logps/rejected": -25.181196212768555,
      "loss": 0.6731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0417286641895771,
      "rewards/margins": 0.0417286641895771,
      "rewards/rejected": 0.0,
      "step": 19
    },
    {
      "epoch": 0.11173184357541899,
      "grad_norm": 26.66489695953028,
      "learning_rate": 3.4843205574912896e-07,
      "logits/chosen": -2.504908561706543,
      "logits/rejected": -2.4020352363586426,
      "logps/chosen": -42.48687744140625,
      "logps/rejected": -45.32117462158203,
      "loss": 0.6754,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0348605141043663,
      "rewards/margins": 0.0348605141043663,
      "rewards/rejected": 0.0,
      "step": 20
    },
    {
      "epoch": 0.11173184357541899,
      "eval_logits/chosen": -2.402383327484131,
      "eval_logits/rejected": -2.2575807571411133,
      "eval_logps/chosen": -29.694604873657227,
      "eval_logps/rejected": -41.2544059753418,
      "eval_loss": 0.6737489104270935,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": 0.03754381462931633,
      "eval_rewards/margins": 0.03754381462931633,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 28.9376,
      "eval_samples_per_second": 10.713,
      "eval_steps_per_second": 0.346,
      "step": 20
    },
    {
      "epoch": 0.11731843575418995,
      "grad_norm": 27.376999316150574,
      "learning_rate": 3.6585365853658536e-07,
      "logits/chosen": -2.141979217529297,
      "logits/rejected": -1.9892356395721436,
      "logps/chosen": -39.02077865600586,
      "logps/rejected": -41.026512145996094,
      "loss": 0.6756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02041320875287056,
      "rewards/margins": 0.02041320875287056,
      "rewards/rejected": 0.0,
      "step": 21
    },
    {
      "epoch": 0.12290502793296089,
      "grad_norm": 27.30549387752372,
      "learning_rate": 3.832752613240418e-07,
      "logits/chosen": -2.482546806335449,
      "logits/rejected": -2.5740606784820557,
      "logps/chosen": -32.61042022705078,
      "logps/rejected": -38.36219024658203,
      "loss": 0.6745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03432812541723251,
      "rewards/margins": 0.03432812541723251,
      "rewards/rejected": 0.0,
      "step": 22
    },
    {
      "epoch": 0.12849162011173185,
      "grad_norm": 30.686914774531644,
      "learning_rate": 4.0069686411149827e-07,
      "logits/chosen": -2.0644421577453613,
      "logits/rejected": -2.1534571647644043,
      "logps/chosen": -97.73858642578125,
      "logps/rejected": -57.271820068359375,
      "loss": 0.6597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07766075432300568,
      "rewards/margins": 0.07766075432300568,
      "rewards/rejected": 0.0,
      "step": 23
    },
    {
      "epoch": 0.1340782122905028,
      "grad_norm": 23.949442224448944,
      "learning_rate": 4.181184668989548e-07,
      "logits/chosen": -2.277724027633667,
      "logits/rejected": -2.2604610919952393,
      "logps/chosen": -32.62269592285156,
      "logps/rejected": -33.62386703491211,
      "loss": 0.6566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06159992516040802,
      "rewards/margins": 0.06159992516040802,
      "rewards/rejected": 0.0,
      "step": 24
    },
    {
      "epoch": 0.13966480446927373,
      "grad_norm": 25.73531359720672,
      "learning_rate": 4.3554006968641123e-07,
      "logits/chosen": -2.501332998275757,
      "logits/rejected": -2.4362974166870117,
      "logps/chosen": -20.627525329589844,
      "logps/rejected": -39.836448669433594,
      "loss": 0.6434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1381058692932129,
      "rewards/margins": 0.1381058692932129,
      "rewards/rejected": 0.0,
      "step": 25
    },
    {
      "epoch": 0.1452513966480447,
      "grad_norm": 24.877687258581446,
      "learning_rate": 4.5296167247386764e-07,
      "logits/chosen": -2.6700615882873535,
      "logits/rejected": -2.542797327041626,
      "logps/chosen": -23.476177215576172,
      "logps/rejected": -43.861915588378906,
      "loss": 0.6411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04964141547679901,
      "rewards/margins": 0.04964141547679901,
      "rewards/rejected": 0.0,
      "step": 26
    },
    {
      "epoch": 0.15083798882681565,
      "grad_norm": 24.492161042604618,
      "learning_rate": 4.703832752613241e-07,
      "logits/chosen": -2.5896759033203125,
      "logits/rejected": -2.5783534049987793,
      "logps/chosen": -57.98297119140625,
      "logps/rejected": -55.31554412841797,
      "loss": 0.6341,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18279825150966644,
      "rewards/margins": 0.18279825150966644,
      "rewards/rejected": 0.0,
      "step": 27
    },
    {
      "epoch": 0.1564245810055866,
      "grad_norm": 23.586284010741924,
      "learning_rate": 4.878048780487805e-07,
      "logits/chosen": -2.6499085426330566,
      "logits/rejected": -2.530541181564331,
      "logps/chosen": -30.661283493041992,
      "logps/rejected": -35.1213493347168,
      "loss": 0.6345,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07806858420372009,
      "rewards/margins": 0.07806858420372009,
      "rewards/rejected": 0.0,
      "step": 28
    },
    {
      "epoch": 0.16201117318435754,
      "grad_norm": 25.03874607166256,
      "learning_rate": 5.05226480836237e-07,
      "logits/chosen": -2.365281581878662,
      "logits/rejected": -2.2794227600097656,
      "logps/chosen": -33.34615707397461,
      "logps/rejected": -36.05326843261719,
      "loss": 0.6293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13970595598220825,
      "rewards/margins": 0.13970595598220825,
      "rewards/rejected": 0.0,
      "step": 29
    },
    {
      "epoch": 0.16759776536312848,
      "grad_norm": 24.457205022067768,
      "learning_rate": 5.226480836236935e-07,
      "logits/chosen": -2.4882712364196777,
      "logits/rejected": -2.575115203857422,
      "logps/chosen": -33.33298110961914,
      "logps/rejected": -51.559608459472656,
      "loss": 0.6173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1426214873790741,
      "rewards/margins": 0.1426214873790741,
      "rewards/rejected": 0.0,
      "step": 30
    },
    {
      "epoch": 0.17318435754189945,
      "grad_norm": 24.256784811081157,
      "learning_rate": 5.400696864111499e-07,
      "logits/chosen": -2.4973974227905273,
      "logits/rejected": -2.2744290828704834,
      "logps/chosen": -18.786922454833984,
      "logps/rejected": -48.04046630859375,
      "loss": 0.6166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13153484463691711,
      "rewards/margins": 0.13153484463691711,
      "rewards/rejected": 0.0,
      "step": 31
    },
    {
      "epoch": 0.1787709497206704,
      "grad_norm": 22.676509189703683,
      "learning_rate": 5.574912891986063e-07,
      "logits/chosen": -2.4714839458465576,
      "logits/rejected": -2.326953649520874,
      "logps/chosen": -27.091869354248047,
      "logps/rejected": -41.458457946777344,
      "loss": 0.5999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2088303118944168,
      "rewards/margins": 0.2088303118944168,
      "rewards/rejected": 0.0,
      "step": 32
    },
    {
      "epoch": 0.18435754189944134,
      "grad_norm": 21.362709222136,
      "learning_rate": 5.749128919860628e-07,
      "logits/chosen": -2.6351981163024902,
      "logits/rejected": -2.7211241722106934,
      "logps/chosen": -26.578433990478516,
      "logps/rejected": -31.84450912475586,
      "loss": 0.5793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2085530161857605,
      "rewards/margins": 0.2085530161857605,
      "rewards/rejected": 0.0,
      "step": 33
    },
    {
      "epoch": 0.18994413407821228,
      "grad_norm": 18.94129067379663,
      "learning_rate": 5.923344947735192e-07,
      "logits/chosen": -2.727097272872925,
      "logits/rejected": -2.4062297344207764,
      "logps/chosen": -22.01129913330078,
      "logps/rejected": -64.09861755371094,
      "loss": 0.572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26633548736572266,
      "rewards/margins": 0.26633548736572266,
      "rewards/rejected": 0.0,
      "step": 34
    },
    {
      "epoch": 0.19553072625698323,
      "grad_norm": 19.529109144636863,
      "learning_rate": 6.097560975609757e-07,
      "logits/chosen": -2.65626859664917,
      "logits/rejected": -2.6179378032684326,
      "logps/chosen": -17.730281829833984,
      "logps/rejected": -28.115276336669922,
      "loss": 0.5533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29162508249282837,
      "rewards/margins": 0.29162508249282837,
      "rewards/rejected": 0.0,
      "step": 35
    },
    {
      "epoch": 0.2011173184357542,
      "grad_norm": 19.208146645847336,
      "learning_rate": 6.271777003484321e-07,
      "logits/chosen": -2.319551467895508,
      "logits/rejected": -2.2224769592285156,
      "logps/chosen": -48.81569290161133,
      "logps/rejected": -58.216224670410156,
      "loss": 0.55,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35527223348617554,
      "rewards/margins": 0.35527223348617554,
      "rewards/rejected": 0.0,
      "step": 36
    },
    {
      "epoch": 0.20670391061452514,
      "grad_norm": 18.649388531060243,
      "learning_rate": 6.445993031358885e-07,
      "logits/chosen": -2.6829380989074707,
      "logits/rejected": -2.7352614402770996,
      "logps/chosen": -27.37700653076172,
      "logps/rejected": -34.72381591796875,
      "loss": 0.5439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32753896713256836,
      "rewards/margins": 0.32753896713256836,
      "rewards/rejected": 0.0,
      "step": 37
    },
    {
      "epoch": 0.2122905027932961,
      "grad_norm": 16.593733000678494,
      "learning_rate": 6.62020905923345e-07,
      "logits/chosen": -2.701756715774536,
      "logits/rejected": -2.4107401371002197,
      "logps/chosen": -34.118202209472656,
      "logps/rejected": -51.019081115722656,
      "loss": 0.5484,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2833206057548523,
      "rewards/margins": 0.2833206057548523,
      "rewards/rejected": 0.0,
      "step": 38
    },
    {
      "epoch": 0.21787709497206703,
      "grad_norm": 18.791779509397703,
      "learning_rate": 6.794425087108014e-07,
      "logits/chosen": -2.367257833480835,
      "logits/rejected": -2.292243242263794,
      "logps/chosen": -46.675758361816406,
      "logps/rejected": -48.227848052978516,
      "loss": 0.5209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5180659890174866,
      "rewards/margins": 0.5180659890174866,
      "rewards/rejected": 0.0,
      "step": 39
    },
    {
      "epoch": 0.22346368715083798,
      "grad_norm": 16.979514429529072,
      "learning_rate": 6.968641114982579e-07,
      "logits/chosen": -2.430774211883545,
      "logits/rejected": -2.292480945587158,
      "logps/chosen": -33.29250717163086,
      "logps/rejected": -25.16018295288086,
      "loss": 0.524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40291017293930054,
      "rewards/margins": 0.40291017293930054,
      "rewards/rejected": 0.0,
      "step": 40
    },
    {
      "epoch": 0.22346368715083798,
      "eval_logits/chosen": -2.6366143226623535,
      "eval_logits/rejected": -2.533212661743164,
      "eval_logps/chosen": -26.260738372802734,
      "eval_logps/rejected": -37.393436431884766,
      "eval_loss": 0.5227345824241638,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.3809302747249603,
      "eval_rewards/margins": 0.3809302747249603,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5623,
      "eval_samples_per_second": 12.127,
      "eval_steps_per_second": 0.391,
      "step": 40
    },
    {
      "epoch": 0.22905027932960895,
      "grad_norm": 17.24591852602006,
      "learning_rate": 7.142857142857143e-07,
      "logits/chosen": -2.6994433403015137,
      "logits/rejected": -2.5430526733398438,
      "logps/chosen": -35.217132568359375,
      "logps/rejected": -39.94957733154297,
      "loss": 0.5176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3944741189479828,
      "rewards/margins": 0.3944741189479828,
      "rewards/rejected": 0.0,
      "step": 41
    },
    {
      "epoch": 0.2346368715083799,
      "grad_norm": 17.68950880344319,
      "learning_rate": 7.317073170731707e-07,
      "logits/chosen": -2.9518754482269287,
      "logits/rejected": -2.7146005630493164,
      "logps/chosen": -17.65860366821289,
      "logps/rejected": -41.11650085449219,
      "loss": 0.515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38902437686920166,
      "rewards/margins": 0.38902437686920166,
      "rewards/rejected": 0.0,
      "step": 42
    },
    {
      "epoch": 0.24022346368715083,
      "grad_norm": 17.15339031218512,
      "learning_rate": 7.491289198606272e-07,
      "logits/chosen": -2.8339080810546875,
      "logits/rejected": -2.794726610183716,
      "logps/chosen": -43.349151611328125,
      "logps/rejected": -27.547149658203125,
      "loss": 0.5225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37009480595588684,
      "rewards/margins": 0.37009480595588684,
      "rewards/rejected": 0.0,
      "step": 43
    },
    {
      "epoch": 0.24581005586592178,
      "grad_norm": 15.263344322100778,
      "learning_rate": 7.665505226480836e-07,
      "logits/chosen": -2.8955233097076416,
      "logits/rejected": -2.711134195327759,
      "logps/chosen": -25.269515991210938,
      "logps/rejected": -43.6644172668457,
      "loss": 0.4818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4662972390651703,
      "rewards/margins": 0.4662972390651703,
      "rewards/rejected": 0.0,
      "step": 44
    },
    {
      "epoch": 0.25139664804469275,
      "grad_norm": 14.277720023490666,
      "learning_rate": 7.8397212543554e-07,
      "logits/chosen": -3.1286208629608154,
      "logits/rejected": -2.773886203765869,
      "logps/chosen": -24.051340103149414,
      "logps/rejected": -46.798397064208984,
      "loss": 0.4892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5665565133094788,
      "rewards/margins": 0.5665565133094788,
      "rewards/rejected": 0.0,
      "step": 45
    },
    {
      "epoch": 0.2569832402234637,
      "grad_norm": 13.627267527150417,
      "learning_rate": 8.013937282229965e-07,
      "logits/chosen": -2.4379756450653076,
      "logits/rejected": -2.417691230773926,
      "logps/chosen": -62.11241149902344,
      "logps/rejected": -29.969600677490234,
      "loss": 0.4483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6498408317565918,
      "rewards/margins": 0.6498408317565918,
      "rewards/rejected": 0.0,
      "step": 46
    },
    {
      "epoch": 0.26256983240223464,
      "grad_norm": 13.488851353225957,
      "learning_rate": 8.188153310104529e-07,
      "logits/chosen": -3.079739570617676,
      "logits/rejected": -2.8087985515594482,
      "logps/chosen": -21.90243911743164,
      "logps/rejected": -35.09657669067383,
      "loss": 0.4584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4383593201637268,
      "rewards/margins": 0.4383593201637268,
      "rewards/rejected": 0.0,
      "step": 47
    },
    {
      "epoch": 0.2681564245810056,
      "grad_norm": 12.908481584134897,
      "learning_rate": 8.362369337979096e-07,
      "logits/chosen": -3.1024246215820312,
      "logits/rejected": -2.9160656929016113,
      "logps/chosen": -24.29102325439453,
      "logps/rejected": -30.861024856567383,
      "loss": 0.4343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.551195502281189,
      "rewards/margins": 0.551195502281189,
      "rewards/rejected": 0.0,
      "step": 48
    },
    {
      "epoch": 0.2737430167597765,
      "grad_norm": 14.675278410614423,
      "learning_rate": 8.53658536585366e-07,
      "logits/chosen": -2.733882427215576,
      "logits/rejected": -2.7973785400390625,
      "logps/chosen": -50.987579345703125,
      "logps/rejected": -34.03818893432617,
      "loss": 0.4452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6197081208229065,
      "rewards/margins": 0.6197081208229065,
      "rewards/rejected": 0.0,
      "step": 49
    },
    {
      "epoch": 0.27932960893854747,
      "grad_norm": 12.68438747100227,
      "learning_rate": 8.710801393728225e-07,
      "logits/chosen": -3.529649496078491,
      "logits/rejected": -3.2613019943237305,
      "logps/chosen": -17.576309204101562,
      "logps/rejected": -28.031644821166992,
      "loss": 0.4691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5316892862319946,
      "rewards/margins": 0.5316892862319946,
      "rewards/rejected": 0.0,
      "step": 50
    },
    {
      "epoch": 0.2849162011173184,
      "grad_norm": 13.941937568805447,
      "learning_rate": 8.885017421602789e-07,
      "logits/chosen": -3.0492312908172607,
      "logits/rejected": -2.601745843887329,
      "logps/chosen": -19.333290100097656,
      "logps/rejected": -69.2382583618164,
      "loss": 0.4209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6651899218559265,
      "rewards/margins": 0.6651899218559265,
      "rewards/rejected": 0.0,
      "step": 51
    },
    {
      "epoch": 0.2905027932960894,
      "grad_norm": 13.28915376707466,
      "learning_rate": 9.059233449477353e-07,
      "logits/chosen": -3.1206836700439453,
      "logits/rejected": -3.108964681625366,
      "logps/chosen": -18.50385093688965,
      "logps/rejected": -23.461885452270508,
      "loss": 0.432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5996769070625305,
      "rewards/margins": 0.5996769070625305,
      "rewards/rejected": 0.0,
      "step": 52
    },
    {
      "epoch": 0.29608938547486036,
      "grad_norm": 12.620299449747305,
      "learning_rate": 9.233449477351918e-07,
      "logits/chosen": -3.427971839904785,
      "logits/rejected": -3.152095317840576,
      "logps/chosen": -22.81187629699707,
      "logps/rejected": -48.480472564697266,
      "loss": 0.4245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7129798531532288,
      "rewards/margins": 0.7129798531532288,
      "rewards/rejected": 0.0,
      "step": 53
    },
    {
      "epoch": 0.3016759776536313,
      "grad_norm": 13.008337256185284,
      "learning_rate": 9.407665505226482e-07,
      "logits/chosen": -3.519685745239258,
      "logits/rejected": -3.1098947525024414,
      "logps/chosen": -17.997909545898438,
      "logps/rejected": -48.807899475097656,
      "loss": 0.3855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6252263188362122,
      "rewards/margins": 0.6252263188362122,
      "rewards/rejected": 0.0,
      "step": 54
    },
    {
      "epoch": 0.30726256983240224,
      "grad_norm": 12.223584376212242,
      "learning_rate": 9.581881533101046e-07,
      "logits/chosen": -3.0058469772338867,
      "logits/rejected": -3.197657585144043,
      "logps/chosen": -45.985504150390625,
      "logps/rejected": -28.70703887939453,
      "loss": 0.3852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1006172895431519,
      "rewards/margins": 1.1006172895431519,
      "rewards/rejected": 0.0,
      "step": 55
    },
    {
      "epoch": 0.3128491620111732,
      "grad_norm": 12.981006037758728,
      "learning_rate": 9.75609756097561e-07,
      "logits/chosen": -2.8816592693328857,
      "logits/rejected": -2.709604263305664,
      "logps/chosen": -39.191261291503906,
      "logps/rejected": -39.2909049987793,
      "loss": 0.3863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.07462739944458,
      "rewards/margins": 1.07462739944458,
      "rewards/rejected": 0.0,
      "step": 56
    },
    {
      "epoch": 0.31843575418994413,
      "grad_norm": 11.925818280133381,
      "learning_rate": 9.930313588850176e-07,
      "logits/chosen": -3.183061361312866,
      "logits/rejected": -2.886544704437256,
      "logps/chosen": -17.684162139892578,
      "logps/rejected": -45.475154876708984,
      "loss": 0.382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7845501899719238,
      "rewards/margins": 0.7845501899719238,
      "rewards/rejected": 0.0,
      "step": 57
    },
    {
      "epoch": 0.3240223463687151,
      "grad_norm": 11.396840205282404,
      "learning_rate": 1.010452961672474e-06,
      "logits/chosen": -3.226580858230591,
      "logits/rejected": -2.9627156257629395,
      "logps/chosen": -28.460540771484375,
      "logps/rejected": -41.57842254638672,
      "loss": 0.3857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8089898228645325,
      "rewards/margins": 0.8089898228645325,
      "rewards/rejected": 0.0,
      "step": 58
    },
    {
      "epoch": 0.329608938547486,
      "grad_norm": 13.175688146071156,
      "learning_rate": 1.0278745644599304e-06,
      "logits/chosen": -3.048670768737793,
      "logits/rejected": -3.1021995544433594,
      "logps/chosen": -51.718421936035156,
      "logps/rejected": -56.63726806640625,
      "loss": 0.3717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7834699153900146,
      "rewards/margins": 0.7834699153900146,
      "rewards/rejected": 0.0,
      "step": 59
    },
    {
      "epoch": 0.33519553072625696,
      "grad_norm": 13.3809136230785,
      "learning_rate": 1.045296167247387e-06,
      "logits/chosen": -3.2117362022399902,
      "logits/rejected": -3.277792453765869,
      "logps/chosen": -20.510501861572266,
      "logps/rejected": -26.85704803466797,
      "loss": 0.4126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.818440318107605,
      "rewards/margins": 0.818440318107605,
      "rewards/rejected": 0.0,
      "step": 60
    },
    {
      "epoch": 0.33519553072625696,
      "eval_logits/chosen": -3.080505847930908,
      "eval_logits/rejected": -3.048234462738037,
      "eval_logps/chosen": -22.22114372253418,
      "eval_logps/rejected": -33.492042541503906,
      "eval_loss": 0.40122273564338684,
      "eval_rewards/accuracies": 0.949999988079071,
      "eval_rewards/chosen": 0.7848896384239197,
      "eval_rewards/margins": 0.7848896384239197,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.9069,
      "eval_samples_per_second": 11.966,
      "eval_steps_per_second": 0.386,
      "step": 60
    },
    {
      "epoch": 0.3407821229050279,
      "grad_norm": 14.78352307187615,
      "learning_rate": 1.0627177700348432e-06,
      "logits/chosen": -3.6447770595550537,
      "logits/rejected": -3.242866277694702,
      "logps/chosen": -17.142704010009766,
      "logps/rejected": -31.112533569335938,
      "loss": 0.4187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6670732498168945,
      "rewards/margins": 0.6670732498168945,
      "rewards/rejected": 0.0,
      "step": 61
    },
    {
      "epoch": 0.3463687150837989,
      "grad_norm": 13.1206787465395,
      "learning_rate": 1.0801393728222997e-06,
      "logits/chosen": -3.386688470840454,
      "logits/rejected": -3.3752613067626953,
      "logps/chosen": -17.9847469329834,
      "logps/rejected": -44.67578125,
      "loss": 0.3757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8634730577468872,
      "rewards/margins": 0.8634730577468872,
      "rewards/rejected": 0.0,
      "step": 62
    },
    {
      "epoch": 0.35195530726256985,
      "grad_norm": 12.333364023942405,
      "learning_rate": 1.0975609756097562e-06,
      "logits/chosen": -3.25191068649292,
      "logits/rejected": -3.433079719543457,
      "logps/chosen": -14.44582748413086,
      "logps/rejected": -25.727596282958984,
      "loss": 0.3996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6673988103866577,
      "rewards/margins": 0.6673988103866577,
      "rewards/rejected": 0.0,
      "step": 63
    },
    {
      "epoch": 0.3575418994413408,
      "grad_norm": 11.355366297304933,
      "learning_rate": 1.1149825783972125e-06,
      "logits/chosen": -3.105869770050049,
      "logits/rejected": -3.0901987552642822,
      "logps/chosen": -14.946333885192871,
      "logps/rejected": -28.918251037597656,
      "loss": 0.3733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7256795763969421,
      "rewards/margins": 0.7256795763969421,
      "rewards/rejected": 0.0,
      "step": 64
    },
    {
      "epoch": 0.36312849162011174,
      "grad_norm": 12.150073364117983,
      "learning_rate": 1.132404181184669e-06,
      "logits/chosen": -3.259650707244873,
      "logits/rejected": -3.214348554611206,
      "logps/chosen": -18.420530319213867,
      "logps/rejected": -36.986759185791016,
      "loss": 0.3919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.66029292345047,
      "rewards/margins": 0.66029292345047,
      "rewards/rejected": 0.0,
      "step": 65
    },
    {
      "epoch": 0.3687150837988827,
      "grad_norm": 15.14394087908454,
      "learning_rate": 1.1498257839721255e-06,
      "logits/chosen": -3.1852316856384277,
      "logits/rejected": -2.1967060565948486,
      "logps/chosen": -37.41112518310547,
      "logps/rejected": -55.96690368652344,
      "loss": 0.3958,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.602116048336029,
      "rewards/margins": 0.602116048336029,
      "rewards/rejected": 0.0,
      "step": 66
    },
    {
      "epoch": 0.3743016759776536,
      "grad_norm": 12.904688481182468,
      "learning_rate": 1.167247386759582e-06,
      "logits/chosen": -2.8984220027923584,
      "logits/rejected": -3.1042795181274414,
      "logps/chosen": -28.82742691040039,
      "logps/rejected": -26.898651123046875,
      "loss": 0.3708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8648268580436707,
      "rewards/margins": 0.8648268580436707,
      "rewards/rejected": 0.0,
      "step": 67
    },
    {
      "epoch": 0.37988826815642457,
      "grad_norm": 16.18401932290786,
      "learning_rate": 1.1846689895470384e-06,
      "logits/chosen": -3.0397374629974365,
      "logits/rejected": -2.817532539367676,
      "logps/chosen": -42.113380432128906,
      "logps/rejected": -36.67097854614258,
      "loss": 0.3882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1938257217407227,
      "rewards/margins": 1.1938257217407227,
      "rewards/rejected": 0.0,
      "step": 68
    },
    {
      "epoch": 0.3854748603351955,
      "grad_norm": 13.70995085528444,
      "learning_rate": 1.2020905923344949e-06,
      "logits/chosen": -3.2553746700286865,
      "logits/rejected": -3.4143080711364746,
      "logps/chosen": -17.168376922607422,
      "logps/rejected": -39.18716049194336,
      "loss": 0.3633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7523192167282104,
      "rewards/margins": 0.7523192167282104,
      "rewards/rejected": 0.0,
      "step": 69
    },
    {
      "epoch": 0.39106145251396646,
      "grad_norm": 12.095767642095886,
      "learning_rate": 1.2195121951219514e-06,
      "logits/chosen": -3.533640146255493,
      "logits/rejected": -3.4749014377593994,
      "logps/chosen": -18.32729148864746,
      "logps/rejected": -35.298885345458984,
      "loss": 0.3415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9114996194839478,
      "rewards/margins": 0.9114996194839478,
      "rewards/rejected": 0.0,
      "step": 70
    },
    {
      "epoch": 0.39664804469273746,
      "grad_norm": 13.512654585283979,
      "learning_rate": 1.2369337979094077e-06,
      "logits/chosen": -3.4323909282684326,
      "logits/rejected": -3.2560689449310303,
      "logps/chosen": -20.945514678955078,
      "logps/rejected": -38.79281997680664,
      "loss": 0.3676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5347429513931274,
      "rewards/margins": 0.5347429513931274,
      "rewards/rejected": 0.0,
      "step": 71
    },
    {
      "epoch": 0.4022346368715084,
      "grad_norm": 21.313250480939924,
      "learning_rate": 1.2543554006968642e-06,
      "logits/chosen": -3.7857320308685303,
      "logits/rejected": -3.7294421195983887,
      "logps/chosen": -13.244667053222656,
      "logps/rejected": -38.8890495300293,
      "loss": 0.4039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8888472318649292,
      "rewards/margins": 0.8888472318649292,
      "rewards/rejected": 0.0,
      "step": 72
    },
    {
      "epoch": 0.40782122905027934,
      "grad_norm": 17.659199468305705,
      "learning_rate": 1.2717770034843207e-06,
      "logits/chosen": -3.362807273864746,
      "logits/rejected": -3.4535279273986816,
      "logps/chosen": -9.89373779296875,
      "logps/rejected": -35.40988540649414,
      "loss": 0.4112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47540393471717834,
      "rewards/margins": 0.47540393471717834,
      "rewards/rejected": 0.0,
      "step": 73
    },
    {
      "epoch": 0.4134078212290503,
      "grad_norm": 14.851393294870789,
      "learning_rate": 1.289198606271777e-06,
      "logits/chosen": -3.188673257827759,
      "logits/rejected": -3.254382610321045,
      "logps/chosen": -30.632671356201172,
      "logps/rejected": -38.90617370605469,
      "loss": 0.4097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5215028524398804,
      "rewards/margins": 0.5215028524398804,
      "rewards/rejected": 0.0,
      "step": 74
    },
    {
      "epoch": 0.41899441340782123,
      "grad_norm": 13.504000100636278,
      "learning_rate": 1.3066202090592337e-06,
      "logits/chosen": -3.1063969135284424,
      "logits/rejected": -3.068325996398926,
      "logps/chosen": -27.31705665588379,
      "logps/rejected": -35.688777923583984,
      "loss": 0.3945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7157747745513916,
      "rewards/margins": 0.7157747745513916,
      "rewards/rejected": 0.0,
      "step": 75
    },
    {
      "epoch": 0.4245810055865922,
      "grad_norm": 14.362047396500865,
      "learning_rate": 1.32404181184669e-06,
      "logits/chosen": -3.871370792388916,
      "logits/rejected": -3.6852171421051025,
      "logps/chosen": -15.522777557373047,
      "logps/rejected": -32.04133605957031,
      "loss": 0.3997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9099875688552856,
      "rewards/margins": 0.9099875688552856,
      "rewards/rejected": 0.0,
      "step": 76
    },
    {
      "epoch": 0.4301675977653631,
      "grad_norm": 13.490883883013238,
      "learning_rate": 1.3414634146341465e-06,
      "logits/chosen": -3.380272388458252,
      "logits/rejected": -3.355849266052246,
      "logps/chosen": -30.052743911743164,
      "logps/rejected": -32.10454559326172,
      "loss": 0.3599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5621362924575806,
      "rewards/margins": 1.5621362924575806,
      "rewards/rejected": 0.0,
      "step": 77
    },
    {
      "epoch": 0.43575418994413406,
      "grad_norm": 18.563075274014928,
      "learning_rate": 1.3588850174216028e-06,
      "logits/chosen": -3.294529438018799,
      "logits/rejected": -3.4775681495666504,
      "logps/chosen": -21.397064208984375,
      "logps/rejected": -36.405723571777344,
      "loss": 0.4045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1538485288619995,
      "rewards/margins": 1.1538485288619995,
      "rewards/rejected": 0.0,
      "step": 78
    },
    {
      "epoch": 0.441340782122905,
      "grad_norm": 11.129326612024675,
      "learning_rate": 1.3763066202090593e-06,
      "logits/chosen": -3.2412707805633545,
      "logits/rejected": -3.4183619022369385,
      "logps/chosen": -18.27306365966797,
      "logps/rejected": -29.19367218017578,
      "loss": 0.3788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.844042181968689,
      "rewards/margins": 0.844042181968689,
      "rewards/rejected": 0.0,
      "step": 79
    },
    {
      "epoch": 0.44692737430167595,
      "grad_norm": 11.197509482096788,
      "learning_rate": 1.3937282229965158e-06,
      "logits/chosen": -3.317974090576172,
      "logits/rejected": -3.313884973526001,
      "logps/chosen": -25.37278938293457,
      "logps/rejected": -34.24852752685547,
      "loss": 0.3615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7071568965911865,
      "rewards/margins": 0.7071568965911865,
      "rewards/rejected": 0.0,
      "step": 80
    },
    {
      "epoch": 0.44692737430167595,
      "eval_logits/chosen": -3.2501137256622314,
      "eval_logits/rejected": -3.262934446334839,
      "eval_logps/chosen": -20.515239715576172,
      "eval_logps/rejected": -32.11551284790039,
      "eval_loss": 0.36819127202033997,
      "eval_rewards/accuracies": 0.9750000238418579,
      "eval_rewards/chosen": 0.9554802775382996,
      "eval_rewards/margins": 0.9554802775382996,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6071,
      "eval_samples_per_second": 12.106,
      "eval_steps_per_second": 0.391,
      "step": 80
    },
    {
      "epoch": 0.45251396648044695,
      "grad_norm": 16.969667499744748,
      "learning_rate": 1.4111498257839723e-06,
      "logits/chosen": -3.320915699005127,
      "logits/rejected": -3.4016387462615967,
      "logps/chosen": -29.42806625366211,
      "logps/rejected": -43.149349212646484,
      "loss": 0.4378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9389697909355164,
      "rewards/margins": 0.9389697909355164,
      "rewards/rejected": 0.0,
      "step": 81
    },
    {
      "epoch": 0.4581005586592179,
      "grad_norm": 9.325355001263958,
      "learning_rate": 1.4285714285714286e-06,
      "logits/chosen": -3.267411708831787,
      "logits/rejected": -3.152595043182373,
      "logps/chosen": -26.968229293823242,
      "logps/rejected": -42.821258544921875,
      "loss": 0.3265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.774876594543457,
      "rewards/margins": 1.774876594543457,
      "rewards/rejected": 0.0,
      "step": 82
    },
    {
      "epoch": 0.46368715083798884,
      "grad_norm": 17.48022071292398,
      "learning_rate": 1.4459930313588851e-06,
      "logits/chosen": -3.909846544265747,
      "logits/rejected": -3.6435470581054688,
      "logps/chosen": -16.59403419494629,
      "logps/rejected": -19.272979736328125,
      "loss": 0.393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.133768081665039,
      "rewards/margins": 1.133768081665039,
      "rewards/rejected": 0.0,
      "step": 83
    },
    {
      "epoch": 0.4692737430167598,
      "grad_norm": 15.236018076263754,
      "learning_rate": 1.4634146341463414e-06,
      "logits/chosen": -3.5131428241729736,
      "logits/rejected": -3.5897560119628906,
      "logps/chosen": -15.98919677734375,
      "logps/rejected": -83.14163208007812,
      "loss": 0.3735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0626312494277954,
      "rewards/margins": 1.0626312494277954,
      "rewards/rejected": 0.0,
      "step": 84
    },
    {
      "epoch": 0.4748603351955307,
      "grad_norm": 15.298139420413333,
      "learning_rate": 1.4808362369337982e-06,
      "logits/chosen": -3.164956569671631,
      "logits/rejected": -3.247239589691162,
      "logps/chosen": -34.28429412841797,
      "logps/rejected": -34.51239013671875,
      "loss": 0.3415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.936699628829956,
      "rewards/margins": 0.936699628829956,
      "rewards/rejected": 0.0,
      "step": 85
    },
    {
      "epoch": 0.48044692737430167,
      "grad_norm": 12.884810189337255,
      "learning_rate": 1.4982578397212545e-06,
      "logits/chosen": -3.3326456546783447,
      "logits/rejected": -3.0906426906585693,
      "logps/chosen": -11.453349113464355,
      "logps/rejected": -58.37992858886719,
      "loss": 0.3636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7145702838897705,
      "rewards/margins": 0.7145702838897705,
      "rewards/rejected": 0.0,
      "step": 86
    },
    {
      "epoch": 0.4860335195530726,
      "grad_norm": 10.373441260096184,
      "learning_rate": 1.515679442508711e-06,
      "logits/chosen": -3.2780582904815674,
      "logits/rejected": -3.3151652812957764,
      "logps/chosen": -24.00762176513672,
      "logps/rejected": -19.62729835510254,
      "loss": 0.3258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.46215975284576416,
      "rewards/margins": 0.46215975284576416,
      "rewards/rejected": 0.0,
      "step": 87
    },
    {
      "epoch": 0.49162011173184356,
      "grad_norm": 12.998056126660705,
      "learning_rate": 1.5331010452961673e-06,
      "logits/chosen": -3.4969863891601562,
      "logits/rejected": -3.5198898315429688,
      "logps/chosen": -35.2387580871582,
      "logps/rejected": -31.844161987304688,
      "loss": 0.3501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.828506350517273,
      "rewards/margins": 0.828506350517273,
      "rewards/rejected": 0.0,
      "step": 88
    },
    {
      "epoch": 0.4972067039106145,
      "grad_norm": 10.461319613216489,
      "learning_rate": 1.5505226480836238e-06,
      "logits/chosen": -3.3303346633911133,
      "logits/rejected": -3.484466791152954,
      "logps/chosen": -15.128799438476562,
      "logps/rejected": -34.323944091796875,
      "loss": 0.3415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9731521606445312,
      "rewards/margins": 0.9731521606445312,
      "rewards/rejected": 0.0,
      "step": 89
    },
    {
      "epoch": 0.5027932960893855,
      "grad_norm": 12.25933617704301,
      "learning_rate": 1.56794425087108e-06,
      "logits/chosen": -3.303130865097046,
      "logits/rejected": -3.2368879318237305,
      "logps/chosen": -23.713232040405273,
      "logps/rejected": -23.562557220458984,
      "loss": 0.3733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7316214442253113,
      "rewards/margins": 0.7316214442253113,
      "rewards/rejected": 0.0,
      "step": 90
    },
    {
      "epoch": 0.5083798882681564,
      "grad_norm": 17.555997001892898,
      "learning_rate": 1.5853658536585368e-06,
      "logits/chosen": -3.480325698852539,
      "logits/rejected": -3.489372968673706,
      "logps/chosen": -22.084779739379883,
      "logps/rejected": -45.449859619140625,
      "loss": 0.4394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8015055060386658,
      "rewards/margins": 0.8015055060386658,
      "rewards/rejected": 0.0,
      "step": 91
    },
    {
      "epoch": 0.5139664804469274,
      "grad_norm": 11.035063558732562,
      "learning_rate": 1.602787456445993e-06,
      "logits/chosen": -3.297541856765747,
      "logits/rejected": -3.258798360824585,
      "logps/chosen": -21.342756271362305,
      "logps/rejected": -18.519176483154297,
      "loss": 0.3528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0773072242736816,
      "rewards/margins": 1.0773072242736816,
      "rewards/rejected": 0.0,
      "step": 92
    },
    {
      "epoch": 0.5195530726256983,
      "grad_norm": 10.805068088024747,
      "learning_rate": 1.6202090592334496e-06,
      "logits/chosen": -3.3417906761169434,
      "logits/rejected": -3.2954111099243164,
      "logps/chosen": -20.167362213134766,
      "logps/rejected": -26.74268341064453,
      "loss": 0.35,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0528857707977295,
      "rewards/margins": 1.0528857707977295,
      "rewards/rejected": 0.0,
      "step": 93
    },
    {
      "epoch": 0.5251396648044693,
      "grad_norm": 11.475826677669762,
      "learning_rate": 1.6376306620209059e-06,
      "logits/chosen": -3.1418368816375732,
      "logits/rejected": -3.1323318481445312,
      "logps/chosen": -20.764015197753906,
      "logps/rejected": -23.263172149658203,
      "loss": 0.3669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9657772183418274,
      "rewards/margins": 0.9657772183418274,
      "rewards/rejected": 0.0,
      "step": 94
    },
    {
      "epoch": 0.5307262569832403,
      "grad_norm": 13.109975701130185,
      "learning_rate": 1.6550522648083626e-06,
      "logits/chosen": -3.705745220184326,
      "logits/rejected": -3.0725183486938477,
      "logps/chosen": -10.723477363586426,
      "logps/rejected": -52.0366096496582,
      "loss": 0.388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9355510473251343,
      "rewards/margins": 0.9355510473251343,
      "rewards/rejected": 0.0,
      "step": 95
    },
    {
      "epoch": 0.5363128491620112,
      "grad_norm": 12.00631427881905,
      "learning_rate": 1.6724738675958191e-06,
      "logits/chosen": -3.7319202423095703,
      "logits/rejected": -3.6239516735076904,
      "logps/chosen": -12.31922435760498,
      "logps/rejected": -18.5076904296875,
      "loss": 0.3401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6795104742050171,
      "rewards/margins": 0.6795104742050171,
      "rewards/rejected": 0.0,
      "step": 96
    },
    {
      "epoch": 0.5418994413407822,
      "grad_norm": 12.768695612575527,
      "learning_rate": 1.6898954703832754e-06,
      "logits/chosen": -3.292173385620117,
      "logits/rejected": -3.3058021068573,
      "logps/chosen": -22.064472198486328,
      "logps/rejected": -63.23373794555664,
      "loss": 0.367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8686464428901672,
      "rewards/margins": 0.8686464428901672,
      "rewards/rejected": 0.0,
      "step": 97
    },
    {
      "epoch": 0.547486033519553,
      "grad_norm": 11.473297727733847,
      "learning_rate": 1.707317073170732e-06,
      "logits/chosen": -3.5606942176818848,
      "logits/rejected": -3.54246187210083,
      "logps/chosen": -12.611563682556152,
      "logps/rejected": -22.92498016357422,
      "loss": 0.3495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6282155513763428,
      "rewards/margins": 0.6282155513763428,
      "rewards/rejected": 0.0,
      "step": 98
    },
    {
      "epoch": 0.553072625698324,
      "grad_norm": 11.714018090177658,
      "learning_rate": 1.7247386759581882e-06,
      "logits/chosen": -3.622633695602417,
      "logits/rejected": -3.530763864517212,
      "logps/chosen": -13.129388809204102,
      "logps/rejected": -24.20990562438965,
      "loss": 0.3514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9047285914421082,
      "rewards/margins": 0.9047285914421082,
      "rewards/rejected": 0.0,
      "step": 99
    },
    {
      "epoch": 0.5586592178770949,
      "grad_norm": 12.552337818290061,
      "learning_rate": 1.742160278745645e-06,
      "logits/chosen": -3.374194860458374,
      "logits/rejected": -3.2695066928863525,
      "logps/chosen": -34.33903503417969,
      "logps/rejected": -33.92147445678711,
      "loss": 0.2937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3563446998596191,
      "rewards/margins": 1.3563446998596191,
      "rewards/rejected": 0.0,
      "step": 100
    },
    {
      "epoch": 0.5586592178770949,
      "eval_logits/chosen": -3.2703537940979004,
      "eval_logits/rejected": -3.2905681133270264,
      "eval_logps/chosen": -20.075654983520508,
      "eval_logps/rejected": -31.596282958984375,
      "eval_loss": 0.35813865065574646,
      "eval_rewards/accuracies": 0.9750000238418579,
      "eval_rewards/chosen": 0.9994388818740845,
      "eval_rewards/margins": 0.9994388818740845,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.7011,
      "eval_samples_per_second": 12.062,
      "eval_steps_per_second": 0.389,
      "step": 100
    },
    {
      "epoch": 0.5642458100558659,
      "grad_norm": 11.583096994737856,
      "learning_rate": 1.7595818815331012e-06,
      "logits/chosen": -3.191580057144165,
      "logits/rejected": -3.2694149017333984,
      "logps/chosen": -15.161776542663574,
      "logps/rejected": -13.718299865722656,
      "loss": 0.37,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5109202265739441,
      "rewards/margins": 0.5109202265739441,
      "rewards/rejected": 0.0,
      "step": 101
    },
    {
      "epoch": 0.5698324022346368,
      "grad_norm": 11.237757765739552,
      "learning_rate": 1.7770034843205577e-06,
      "logits/chosen": -3.413356065750122,
      "logits/rejected": -3.379192352294922,
      "logps/chosen": -29.95255470275879,
      "logps/rejected": -31.522491455078125,
      "loss": 0.3306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5131101608276367,
      "rewards/margins": 1.5131101608276367,
      "rewards/rejected": 0.0,
      "step": 102
    },
    {
      "epoch": 0.5754189944134078,
      "grad_norm": 16.047541128700225,
      "learning_rate": 1.794425087108014e-06,
      "logits/chosen": -3.561173677444458,
      "logits/rejected": -3.591588258743286,
      "logps/chosen": -11.23000717163086,
      "logps/rejected": -32.61612319946289,
      "loss": 0.3944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.792344331741333,
      "rewards/margins": 0.792344331741333,
      "rewards/rejected": 0.0,
      "step": 103
    },
    {
      "epoch": 0.5810055865921788,
      "grad_norm": 10.98441086882471,
      "learning_rate": 1.8118466898954705e-06,
      "logits/chosen": -3.2839152812957764,
      "logits/rejected": -3.2498652935028076,
      "logps/chosen": -21.42696762084961,
      "logps/rejected": -23.835294723510742,
      "loss": 0.3457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1637508869171143,
      "rewards/margins": 1.1637508869171143,
      "rewards/rejected": 0.0,
      "step": 104
    },
    {
      "epoch": 0.5865921787709497,
      "grad_norm": 13.302926646606267,
      "learning_rate": 1.8292682926829268e-06,
      "logits/chosen": -3.1135566234588623,
      "logits/rejected": -3.1127512454986572,
      "logps/chosen": -20.421844482421875,
      "logps/rejected": -20.50809097290039,
      "loss": 0.3634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9201916456222534,
      "rewards/margins": 0.9201916456222534,
      "rewards/rejected": 0.0,
      "step": 105
    },
    {
      "epoch": 0.5921787709497207,
      "grad_norm": 12.623713996887721,
      "learning_rate": 1.8466898954703836e-06,
      "logits/chosen": -3.675037384033203,
      "logits/rejected": -3.4529781341552734,
      "logps/chosen": -13.883888244628906,
      "logps/rejected": -31.733394622802734,
      "loss": 0.3425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1756565570831299,
      "rewards/margins": 1.1756565570831299,
      "rewards/rejected": 0.0,
      "step": 106
    },
    {
      "epoch": 0.5977653631284916,
      "grad_norm": 20.529556076362905,
      "learning_rate": 1.8641114982578399e-06,
      "logits/chosen": -3.676447868347168,
      "logits/rejected": -3.6586716175079346,
      "logps/chosen": -15.871049880981445,
      "logps/rejected": -34.52062225341797,
      "loss": 0.3824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1021968126296997,
      "rewards/margins": 1.1021968126296997,
      "rewards/rejected": 0.0,
      "step": 107
    },
    {
      "epoch": 0.6033519553072626,
      "grad_norm": 11.300590071629848,
      "learning_rate": 1.8815331010452964e-06,
      "logits/chosen": -3.2093544006347656,
      "logits/rejected": -3.14147686958313,
      "logps/chosen": -33.618751525878906,
      "logps/rejected": -29.43307876586914,
      "loss": 0.3302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.75767982006073,
      "rewards/margins": 1.75767982006073,
      "rewards/rejected": 0.0,
      "step": 108
    },
    {
      "epoch": 0.6089385474860335,
      "grad_norm": 12.631111675186169,
      "learning_rate": 1.8989547038327527e-06,
      "logits/chosen": -3.234271764755249,
      "logits/rejected": -3.099966287612915,
      "logps/chosen": -25.658048629760742,
      "logps/rejected": -49.90553665161133,
      "loss": 0.3259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8175694942474365,
      "rewards/margins": 0.8175694942474365,
      "rewards/rejected": 0.0,
      "step": 109
    },
    {
      "epoch": 0.6145251396648045,
      "grad_norm": 12.904021873545778,
      "learning_rate": 1.916376306620209e-06,
      "logits/chosen": -3.2384815216064453,
      "logits/rejected": -3.3648018836975098,
      "logps/chosen": -17.749008178710938,
      "logps/rejected": -77.1066665649414,
      "loss": 0.341,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8747891187667847,
      "rewards/margins": 0.8747891187667847,
      "rewards/rejected": 0.0,
      "step": 110
    },
    {
      "epoch": 0.6201117318435754,
      "grad_norm": 14.789052937517669,
      "learning_rate": 1.9337979094076655e-06,
      "logits/chosen": -3.3222389221191406,
      "logits/rejected": -2.6916258335113525,
      "logps/chosen": -21.706462860107422,
      "logps/rejected": -57.381370544433594,
      "loss": 0.3579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0143804550170898,
      "rewards/margins": 1.0143804550170898,
      "rewards/rejected": 0.0,
      "step": 111
    },
    {
      "epoch": 0.6256983240223464,
      "grad_norm": 16.021634804418767,
      "learning_rate": 1.951219512195122e-06,
      "logits/chosen": -3.5547173023223877,
      "logits/rejected": -3.6582894325256348,
      "logps/chosen": -21.062744140625,
      "logps/rejected": -41.700416564941406,
      "loss": 0.3807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3826320171356201,
      "rewards/margins": 0.3826320171356201,
      "rewards/rejected": 0.0,
      "step": 112
    },
    {
      "epoch": 0.6312849162011173,
      "grad_norm": 14.938577020495154,
      "learning_rate": 1.9686411149825785e-06,
      "logits/chosen": -3.5193817615509033,
      "logits/rejected": -3.394737958908081,
      "logps/chosen": -34.965030670166016,
      "logps/rejected": -49.21733856201172,
      "loss": 0.3818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.846462607383728,
      "rewards/margins": 0.846462607383728,
      "rewards/rejected": 0.0,
      "step": 113
    },
    {
      "epoch": 0.6368715083798883,
      "grad_norm": 11.44686036212196,
      "learning_rate": 1.986062717770035e-06,
      "logits/chosen": -3.4230716228485107,
      "logits/rejected": -3.5235867500305176,
      "logps/chosen": -17.509788513183594,
      "logps/rejected": -27.067529678344727,
      "loss": 0.3383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3331539630889893,
      "rewards/margins": 1.3331539630889893,
      "rewards/rejected": 0.0,
      "step": 114
    },
    {
      "epoch": 0.6424581005586593,
      "grad_norm": 14.294747637646047,
      "learning_rate": 2.0034843205574915e-06,
      "logits/chosen": -3.1321969032287598,
      "logits/rejected": -3.1000611782073975,
      "logps/chosen": -38.75387191772461,
      "logps/rejected": -26.065773010253906,
      "loss": 0.3803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6701682209968567,
      "rewards/margins": 0.6701682209968567,
      "rewards/rejected": 0.0,
      "step": 115
    },
    {
      "epoch": 0.6480446927374302,
      "grad_norm": 11.14128840530159,
      "learning_rate": 2.020905923344948e-06,
      "logits/chosen": -3.6006064414978027,
      "logits/rejected": -3.4040277004241943,
      "logps/chosen": -27.2326602935791,
      "logps/rejected": -30.274707794189453,
      "loss": 0.3205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.462801218032837,
      "rewards/margins": 1.462801218032837,
      "rewards/rejected": 0.0,
      "step": 116
    },
    {
      "epoch": 0.6536312849162011,
      "grad_norm": 16.049486048537503,
      "learning_rate": 2.038327526132404e-06,
      "logits/chosen": -3.434110164642334,
      "logits/rejected": -3.358154296875,
      "logps/chosen": -22.82818603515625,
      "logps/rejected": -47.18830108642578,
      "loss": 0.3233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4857500791549683,
      "rewards/margins": 1.4857500791549683,
      "rewards/rejected": 0.0,
      "step": 117
    },
    {
      "epoch": 0.659217877094972,
      "grad_norm": 14.30943925407536,
      "learning_rate": 2.055749128919861e-06,
      "logits/chosen": -3.521904468536377,
      "logits/rejected": -3.4602296352386475,
      "logps/chosen": -26.022472381591797,
      "logps/rejected": -50.527244567871094,
      "loss": 0.3206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.309912919998169,
      "rewards/margins": 1.309912919998169,
      "rewards/rejected": 0.0,
      "step": 118
    },
    {
      "epoch": 0.664804469273743,
      "grad_norm": 13.8469845455708,
      "learning_rate": 2.073170731707317e-06,
      "logits/chosen": -3.6135969161987305,
      "logits/rejected": -3.545849561691284,
      "logps/chosen": -34.74000549316406,
      "logps/rejected": -23.716455459594727,
      "loss": 0.3584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6294252276420593,
      "rewards/margins": 0.6294252276420593,
      "rewards/rejected": 0.0,
      "step": 119
    },
    {
      "epoch": 0.6703910614525139,
      "grad_norm": 11.494476801859298,
      "learning_rate": 2.090592334494774e-06,
      "logits/chosen": -3.9000139236450195,
      "logits/rejected": -3.928020715713501,
      "logps/chosen": -23.197811126708984,
      "logps/rejected": -28.104434967041016,
      "loss": 0.3308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1710760593414307,
      "rewards/margins": 1.1710760593414307,
      "rewards/rejected": 0.0,
      "step": 120
    },
    {
      "epoch": 0.6703910614525139,
      "eval_logits/chosen": -3.290062665939331,
      "eval_logits/rejected": -3.318080186843872,
      "eval_logps/chosen": -19.811399459838867,
      "eval_logps/rejected": -31.391864776611328,
      "eval_loss": 0.35239025950431824,
      "eval_rewards/accuracies": 0.9750000238418579,
      "eval_rewards/chosen": 1.0258643627166748,
      "eval_rewards/margins": 1.0258643627166748,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5451,
      "eval_samples_per_second": 12.135,
      "eval_steps_per_second": 0.391,
      "step": 120
    },
    {
      "epoch": 0.6759776536312849,
      "grad_norm": 14.408055646985774,
      "learning_rate": 2.10801393728223e-06,
      "logits/chosen": -3.605227470397949,
      "logits/rejected": -3.7759881019592285,
      "logps/chosen": -20.896387100219727,
      "logps/rejected": -35.246768951416016,
      "loss": 0.3872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.113234043121338,
      "rewards/margins": 1.113234043121338,
      "rewards/rejected": 0.0,
      "step": 121
    },
    {
      "epoch": 0.6815642458100558,
      "grad_norm": 13.53838402945996,
      "learning_rate": 2.1254355400696864e-06,
      "logits/chosen": -3.2952921390533447,
      "logits/rejected": -3.3568828105926514,
      "logps/chosen": -31.607402801513672,
      "logps/rejected": -16.904987335205078,
      "loss": 0.4012,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15581277012825012,
      "rewards/margins": 0.15581277012825012,
      "rewards/rejected": 0.0,
      "step": 122
    },
    {
      "epoch": 0.6871508379888268,
      "grad_norm": 14.29167184436156,
      "learning_rate": 2.1428571428571427e-06,
      "logits/chosen": -3.1098594665527344,
      "logits/rejected": -3.153810739517212,
      "logps/chosen": -34.63990020751953,
      "logps/rejected": -33.18708038330078,
      "loss": 0.3811,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4939751625061035,
      "rewards/margins": 1.4939751625061035,
      "rewards/rejected": 0.0,
      "step": 123
    },
    {
      "epoch": 0.6927374301675978,
      "grad_norm": 12.009047015762913,
      "learning_rate": 2.1602787456445995e-06,
      "logits/chosen": -3.470487594604492,
      "logits/rejected": -3.438277006149292,
      "logps/chosen": -15.662784576416016,
      "logps/rejected": -37.81660842895508,
      "loss": 0.3127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9637354016304016,
      "rewards/margins": 0.9637354016304016,
      "rewards/rejected": 0.0,
      "step": 124
    },
    {
      "epoch": 0.6983240223463687,
      "grad_norm": 13.03058983647168,
      "learning_rate": 2.1777003484320557e-06,
      "logits/chosen": -3.5475521087646484,
      "logits/rejected": -3.5056161880493164,
      "logps/chosen": -14.38143539428711,
      "logps/rejected": -43.139495849609375,
      "loss": 0.354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7005370259284973,
      "rewards/margins": 0.7005370259284973,
      "rewards/rejected": 0.0,
      "step": 125
    },
    {
      "epoch": 0.7039106145251397,
      "grad_norm": 13.619814099957717,
      "learning_rate": 2.1951219512195125e-06,
      "logits/chosen": -3.2412755489349365,
      "logits/rejected": -3.233492136001587,
      "logps/chosen": -14.173643112182617,
      "logps/rejected": -45.96443557739258,
      "loss": 0.3441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9142405986785889,
      "rewards/margins": 0.9142405986785889,
      "rewards/rejected": 0.0,
      "step": 126
    },
    {
      "epoch": 0.7094972067039106,
      "grad_norm": 9.74261691974096,
      "learning_rate": 2.2125435540069688e-06,
      "logits/chosen": -3.5439116954803467,
      "logits/rejected": -3.426776885986328,
      "logps/chosen": -31.885967254638672,
      "logps/rejected": -30.898303985595703,
      "loss": 0.3079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4220666885375977,
      "rewards/margins": 1.4220666885375977,
      "rewards/rejected": 0.0,
      "step": 127
    },
    {
      "epoch": 0.7150837988826816,
      "grad_norm": 15.315593917491686,
      "learning_rate": 2.229965156794425e-06,
      "logits/chosen": -3.586400270462036,
      "logits/rejected": -3.457953929901123,
      "logps/chosen": -15.121441841125488,
      "logps/rejected": -47.6505126953125,
      "loss": 0.3549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8860883712768555,
      "rewards/margins": 0.8860883712768555,
      "rewards/rejected": 0.0,
      "step": 128
    },
    {
      "epoch": 0.7206703910614525,
      "grad_norm": 12.373400171401133,
      "learning_rate": 2.2473867595818818e-06,
      "logits/chosen": -3.5708751678466797,
      "logits/rejected": -3.6276514530181885,
      "logps/chosen": -18.086673736572266,
      "logps/rejected": -25.492902755737305,
      "loss": 0.3613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1381096839904785,
      "rewards/margins": 1.1381096839904785,
      "rewards/rejected": 0.0,
      "step": 129
    },
    {
      "epoch": 0.7262569832402235,
      "grad_norm": 10.799503985367195,
      "learning_rate": 2.264808362369338e-06,
      "logits/chosen": -3.834345817565918,
      "logits/rejected": -3.8486640453338623,
      "logps/chosen": -28.06903076171875,
      "logps/rejected": -27.766605377197266,
      "loss": 0.323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9422239065170288,
      "rewards/margins": 0.9422239065170288,
      "rewards/rejected": 0.0,
      "step": 130
    },
    {
      "epoch": 0.7318435754189944,
      "grad_norm": 21.511039952080687,
      "learning_rate": 2.282229965156795e-06,
      "logits/chosen": -3.1798486709594727,
      "logits/rejected": -3.278769016265869,
      "logps/chosen": -15.774320602416992,
      "logps/rejected": -29.89617919921875,
      "loss": 0.4224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0550003051757812,
      "rewards/margins": 1.0550003051757812,
      "rewards/rejected": 0.0,
      "step": 131
    },
    {
      "epoch": 0.7374301675977654,
      "grad_norm": 12.327840789942552,
      "learning_rate": 2.299651567944251e-06,
      "logits/chosen": -3.3334593772888184,
      "logits/rejected": -3.064690589904785,
      "logps/chosen": -19.42068862915039,
      "logps/rejected": -37.30345153808594,
      "loss": 0.3617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8687980771064758,
      "rewards/margins": 0.8687980771064758,
      "rewards/rejected": 0.0,
      "step": 132
    },
    {
      "epoch": 0.7430167597765364,
      "grad_norm": 15.301315920632492,
      "learning_rate": 2.317073170731708e-06,
      "logits/chosen": -3.1581974029541016,
      "logits/rejected": -3.073570728302002,
      "logps/chosen": -27.522403717041016,
      "logps/rejected": -36.12481689453125,
      "loss": 0.3758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2702422142028809,
      "rewards/margins": 1.2702422142028809,
      "rewards/rejected": 0.0,
      "step": 133
    },
    {
      "epoch": 0.7486033519553073,
      "grad_norm": 10.627243319889736,
      "learning_rate": 2.334494773519164e-06,
      "logits/chosen": -3.6665456295013428,
      "logits/rejected": -3.6920619010925293,
      "logps/chosen": -20.004789352416992,
      "logps/rejected": -25.155750274658203,
      "loss": 0.3031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0919253826141357,
      "rewards/margins": 1.0919253826141357,
      "rewards/rejected": 0.0,
      "step": 134
    },
    {
      "epoch": 0.7541899441340782,
      "grad_norm": 13.80377434809262,
      "learning_rate": 2.3519163763066204e-06,
      "logits/chosen": -3.4695613384246826,
      "logits/rejected": -3.378028154373169,
      "logps/chosen": -11.901710510253906,
      "logps/rejected": -25.48476791381836,
      "loss": 0.3336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6722303032875061,
      "rewards/margins": 0.6722303032875061,
      "rewards/rejected": 0.0,
      "step": 135
    },
    {
      "epoch": 0.7597765363128491,
      "grad_norm": 16.08513570051224,
      "learning_rate": 2.3693379790940767e-06,
      "logits/chosen": -3.3368277549743652,
      "logits/rejected": -3.3044915199279785,
      "logps/chosen": -63.31962203979492,
      "logps/rejected": -41.86566162109375,
      "loss": 0.3711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6263803243637085,
      "rewards/margins": 0.6263803243637085,
      "rewards/rejected": 0.0,
      "step": 136
    },
    {
      "epoch": 0.7653631284916201,
      "grad_norm": 12.755202877733156,
      "learning_rate": 2.3867595818815334e-06,
      "logits/chosen": -3.554523468017578,
      "logits/rejected": -3.4455997943878174,
      "logps/chosen": -24.041217803955078,
      "logps/rejected": -29.966903686523438,
      "loss": 0.3829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5250310897827148,
      "rewards/margins": 0.5250310897827148,
      "rewards/rejected": 0.0,
      "step": 137
    },
    {
      "epoch": 0.770949720670391,
      "grad_norm": 15.35871125649775,
      "learning_rate": 2.4041811846689897e-06,
      "logits/chosen": -3.4806480407714844,
      "logits/rejected": -3.304854154586792,
      "logps/chosen": -50.9167594909668,
      "logps/rejected": -38.277015686035156,
      "loss": 0.3495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0982774496078491,
      "rewards/margins": 1.0982774496078491,
      "rewards/rejected": 0.0,
      "step": 138
    },
    {
      "epoch": 0.776536312849162,
      "grad_norm": 15.512280398865256,
      "learning_rate": 2.4216027874564464e-06,
      "logits/chosen": -3.270118236541748,
      "logits/rejected": -3.20051908493042,
      "logps/chosen": -27.433284759521484,
      "logps/rejected": -26.233314514160156,
      "loss": 0.3607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4795646667480469,
      "rewards/margins": 1.4795646667480469,
      "rewards/rejected": 0.0,
      "step": 139
    },
    {
      "epoch": 0.7821229050279329,
      "grad_norm": 15.313269435276363,
      "learning_rate": 2.4390243902439027e-06,
      "logits/chosen": -3.4366750717163086,
      "logits/rejected": -3.0814208984375,
      "logps/chosen": -34.931583404541016,
      "logps/rejected": -20.90804672241211,
      "loss": 0.3877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1181458234786987,
      "rewards/margins": 1.1181458234786987,
      "rewards/rejected": 0.0,
      "step": 140
    },
    {
      "epoch": 0.7821229050279329,
      "eval_logits/chosen": -3.2833049297332764,
      "eval_logits/rejected": -3.314002513885498,
      "eval_logps/chosen": -19.659954071044922,
      "eval_logps/rejected": -31.290943145751953,
      "eval_loss": 0.35020968317985535,
      "eval_rewards/accuracies": 0.9750000238418579,
      "eval_rewards/chosen": 1.0410090684890747,
      "eval_rewards/margins": 1.0410090684890747,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.529,
      "eval_samples_per_second": 12.143,
      "eval_steps_per_second": 0.392,
      "step": 140
    },
    {
      "epoch": 0.7877094972067039,
      "grad_norm": 14.329364915099111,
      "learning_rate": 2.456445993031359e-06,
      "logits/chosen": -3.5259008407592773,
      "logits/rejected": -3.6189160346984863,
      "logps/chosen": -16.59333610534668,
      "logps/rejected": -45.04124450683594,
      "loss": 0.4149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6079448461532593,
      "rewards/margins": 0.6079448461532593,
      "rewards/rejected": 0.0,
      "step": 141
    },
    {
      "epoch": 0.7932960893854749,
      "grad_norm": 12.649300295796525,
      "learning_rate": 2.4738675958188153e-06,
      "logits/chosen": -3.644083023071289,
      "logits/rejected": -3.524257183074951,
      "logps/chosen": -10.90259075164795,
      "logps/rejected": -45.416648864746094,
      "loss": 0.3387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9181069731712341,
      "rewards/margins": 0.9181069731712341,
      "rewards/rejected": 0.0,
      "step": 142
    },
    {
      "epoch": 0.7988826815642458,
      "grad_norm": 12.528556816255303,
      "learning_rate": 2.491289198606272e-06,
      "logits/chosen": -3.251450538635254,
      "logits/rejected": -3.2836382389068604,
      "logps/chosen": -23.983842849731445,
      "logps/rejected": -27.958499908447266,
      "loss": 0.3207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5557739734649658,
      "rewards/margins": 1.5557739734649658,
      "rewards/rejected": 0.0,
      "step": 143
    },
    {
      "epoch": 0.8044692737430168,
      "grad_norm": 14.744928499973746,
      "learning_rate": 2.5087108013937284e-06,
      "logits/chosen": -3.5456674098968506,
      "logits/rejected": -3.4777238368988037,
      "logps/chosen": -18.8297119140625,
      "logps/rejected": -25.727781295776367,
      "loss": 0.3764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8823195099830627,
      "rewards/margins": 0.8823195099830627,
      "rewards/rejected": 0.0,
      "step": 144
    },
    {
      "epoch": 0.8100558659217877,
      "grad_norm": 16.321224812430483,
      "learning_rate": 2.5261324041811846e-06,
      "logits/chosen": -3.3257229328155518,
      "logits/rejected": -3.480558156967163,
      "logps/chosen": -24.921436309814453,
      "logps/rejected": -49.64356231689453,
      "loss": 0.3366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.450674057006836,
      "rewards/margins": 1.450674057006836,
      "rewards/rejected": 0.0,
      "step": 145
    },
    {
      "epoch": 0.8156424581005587,
      "grad_norm": 12.312668522681452,
      "learning_rate": 2.5435540069686414e-06,
      "logits/chosen": -3.4369521141052246,
      "logits/rejected": -3.3019845485687256,
      "logps/chosen": -20.841720581054688,
      "logps/rejected": -29.958717346191406,
      "loss": 0.3705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6902042031288147,
      "rewards/margins": 0.6902042031288147,
      "rewards/rejected": 0.0,
      "step": 146
    },
    {
      "epoch": 0.8212290502793296,
      "grad_norm": 14.115382442735905,
      "learning_rate": 2.5609756097560977e-06,
      "logits/chosen": -3.3569397926330566,
      "logits/rejected": -3.2133901119232178,
      "logps/chosen": -24.021366119384766,
      "logps/rejected": -83.68838500976562,
      "loss": 0.3394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8873471021652222,
      "rewards/margins": 0.8873471021652222,
      "rewards/rejected": 0.0,
      "step": 147
    },
    {
      "epoch": 0.8268156424581006,
      "grad_norm": 16.048350199431102,
      "learning_rate": 2.578397212543554e-06,
      "logits/chosen": -3.3912353515625,
      "logits/rejected": -3.0636935234069824,
      "logps/chosen": -17.168373107910156,
      "logps/rejected": -21.969812393188477,
      "loss": 0.346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8750801682472229,
      "rewards/margins": 0.8750801682472229,
      "rewards/rejected": 0.0,
      "step": 148
    },
    {
      "epoch": 0.8324022346368715,
      "grad_norm": 10.522377587622268,
      "learning_rate": 2.5958188153310103e-06,
      "logits/chosen": -3.52348256111145,
      "logits/rejected": -3.6835505962371826,
      "logps/chosen": -17.63974380493164,
      "logps/rejected": -52.213722229003906,
      "loss": 0.3276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8563964366912842,
      "rewards/margins": 0.8563964366912842,
      "rewards/rejected": 0.0,
      "step": 149
    },
    {
      "epoch": 0.8379888268156425,
      "grad_norm": 14.615246839779736,
      "learning_rate": 2.6132404181184674e-06,
      "logits/chosen": -3.406809091567993,
      "logits/rejected": -3.6157028675079346,
      "logps/chosen": -29.557723999023438,
      "logps/rejected": -25.100234985351562,
      "loss": 0.3716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1877628564834595,
      "rewards/margins": 1.1877628564834595,
      "rewards/rejected": 0.0,
      "step": 150
    },
    {
      "epoch": 0.8435754189944135,
      "grad_norm": 11.397516844963512,
      "learning_rate": 2.6306620209059237e-06,
      "logits/chosen": -3.5597522258758545,
      "logits/rejected": -3.4989771842956543,
      "logps/chosen": -18.32758140563965,
      "logps/rejected": -42.90640640258789,
      "loss": 0.2898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.620005488395691,
      "rewards/margins": 1.620005488395691,
      "rewards/rejected": 0.0,
      "step": 151
    },
    {
      "epoch": 0.8491620111731844,
      "grad_norm": 9.317958293548163,
      "learning_rate": 2.64808362369338e-06,
      "logits/chosen": -3.4241790771484375,
      "logits/rejected": -3.135045051574707,
      "logps/chosen": -20.312877655029297,
      "logps/rejected": -27.797212600708008,
      "loss": 0.2743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5626355409622192,
      "rewards/margins": 1.5626355409622192,
      "rewards/rejected": 0.0,
      "step": 152
    },
    {
      "epoch": 0.8547486033519553,
      "grad_norm": 15.67902562189744,
      "learning_rate": 2.6655052264808363e-06,
      "logits/chosen": -3.655200958251953,
      "logits/rejected": -3.6608643531799316,
      "logps/chosen": -28.622434616088867,
      "logps/rejected": -14.362475395202637,
      "loss": 0.4001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.647813081741333,
      "rewards/margins": 0.647813081741333,
      "rewards/rejected": 0.0,
      "step": 153
    },
    {
      "epoch": 0.8603351955307262,
      "grad_norm": 17.0141808950758,
      "learning_rate": 2.682926829268293e-06,
      "logits/chosen": -3.3439056873321533,
      "logits/rejected": -3.3395209312438965,
      "logps/chosen": -9.615730285644531,
      "logps/rejected": -18.34330177307129,
      "loss": 0.3515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8768234848976135,
      "rewards/margins": 0.8768234848976135,
      "rewards/rejected": 0.0,
      "step": 154
    },
    {
      "epoch": 0.8659217877094972,
      "grad_norm": 11.303779573493944,
      "learning_rate": 2.7003484320557493e-06,
      "logits/chosen": -3.533757209777832,
      "logits/rejected": -3.4577863216400146,
      "logps/chosen": -17.952903747558594,
      "logps/rejected": -26.117233276367188,
      "loss": 0.3227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.094635009765625,
      "rewards/margins": 1.094635009765625,
      "rewards/rejected": 0.0,
      "step": 155
    },
    {
      "epoch": 0.8715083798882681,
      "grad_norm": 14.265254244830738,
      "learning_rate": 2.7177700348432056e-06,
      "logits/chosen": -3.6299633979797363,
      "logits/rejected": -3.439765691757202,
      "logps/chosen": -14.325593948364258,
      "logps/rejected": -37.461524963378906,
      "loss": 0.3926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8906704187393188,
      "rewards/margins": 0.8906704187393188,
      "rewards/rejected": 0.0,
      "step": 156
    },
    {
      "epoch": 0.8770949720670391,
      "grad_norm": 15.961266011967748,
      "learning_rate": 2.735191637630662e-06,
      "logits/chosen": -3.159883499145508,
      "logits/rejected": -2.885693073272705,
      "logps/chosen": -24.985214233398438,
      "logps/rejected": -48.805702209472656,
      "loss": 0.3519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3967676162719727,
      "rewards/margins": 1.3967676162719727,
      "rewards/rejected": 0.0,
      "step": 157
    },
    {
      "epoch": 0.88268156424581,
      "grad_norm": 20.5585716748845,
      "learning_rate": 2.7526132404181186e-06,
      "logits/chosen": -3.5617966651916504,
      "logits/rejected": -3.4111664295196533,
      "logps/chosen": -13.357250213623047,
      "logps/rejected": -24.29848289489746,
      "loss": 0.3853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5184606313705444,
      "rewards/margins": 0.5184606313705444,
      "rewards/rejected": 0.0,
      "step": 158
    },
    {
      "epoch": 0.888268156424581,
      "grad_norm": 11.253988747563776,
      "learning_rate": 2.7700348432055753e-06,
      "logits/chosen": -3.6450881958007812,
      "logits/rejected": -3.531221389770508,
      "logps/chosen": -11.096773147583008,
      "logps/rejected": -22.444561004638672,
      "loss": 0.3079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8602174520492554,
      "rewards/margins": 0.8602174520492554,
      "rewards/rejected": 0.0,
      "step": 159
    },
    {
      "epoch": 0.8938547486033519,
      "grad_norm": 17.443055638365454,
      "learning_rate": 2.7874564459930316e-06,
      "logits/chosen": -3.582772970199585,
      "logits/rejected": -3.148221015930176,
      "logps/chosen": -29.191118240356445,
      "logps/rejected": -49.01987075805664,
      "loss": 0.349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2191967964172363,
      "rewards/margins": 1.2191967964172363,
      "rewards/rejected": 0.0,
      "step": 160
    },
    {
      "epoch": 0.8938547486033519,
      "eval_logits/chosen": -3.3124027252197266,
      "eval_logits/rejected": -3.345750093460083,
      "eval_logps/chosen": -19.546247482299805,
      "eval_logps/rejected": -31.4556941986084,
      "eval_loss": 0.3500177264213562,
      "eval_rewards/accuracies": 0.9750000238418579,
      "eval_rewards/chosen": 1.0523794889450073,
      "eval_rewards/margins": 1.0523794889450073,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5486,
      "eval_samples_per_second": 12.134,
      "eval_steps_per_second": 0.391,
      "step": 160
    },
    {
      "epoch": 0.8994413407821229,
      "grad_norm": 11.182589448932966,
      "learning_rate": 2.8048780487804884e-06,
      "logits/chosen": -3.564889907836914,
      "logits/rejected": -3.32627272605896,
      "logps/chosen": -17.505149841308594,
      "logps/rejected": -41.035858154296875,
      "loss": 0.3416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1506426334381104,
      "rewards/margins": 1.1506426334381104,
      "rewards/rejected": 0.0,
      "step": 161
    },
    {
      "epoch": 0.9050279329608939,
      "grad_norm": 12.329102900360867,
      "learning_rate": 2.8222996515679447e-06,
      "logits/chosen": -3.5099709033966064,
      "logits/rejected": -3.072869062423706,
      "logps/chosen": -16.431236267089844,
      "logps/rejected": -17.999515533447266,
      "loss": 0.3191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0277546644210815,
      "rewards/margins": 1.0277546644210815,
      "rewards/rejected": 0.0,
      "step": 162
    },
    {
      "epoch": 0.9106145251396648,
      "grad_norm": 13.988121887230498,
      "learning_rate": 2.839721254355401e-06,
      "logits/chosen": -3.4606502056121826,
      "logits/rejected": -3.430018901824951,
      "logps/chosen": -22.302532196044922,
      "logps/rejected": -24.89926528930664,
      "loss": 0.3581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8143572807312012,
      "rewards/margins": 0.8143572807312012,
      "rewards/rejected": 0.0,
      "step": 163
    },
    {
      "epoch": 0.9162011173184358,
      "grad_norm": 12.237490054137051,
      "learning_rate": 2.8571428571428573e-06,
      "logits/chosen": -3.3651864528656006,
      "logits/rejected": -3.2225098609924316,
      "logps/chosen": -20.013275146484375,
      "logps/rejected": -20.411333084106445,
      "loss": 0.3319,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3669614791870117,
      "rewards/margins": 1.3669614791870117,
      "rewards/rejected": 0.0,
      "step": 164
    },
    {
      "epoch": 0.9217877094972067,
      "grad_norm": 12.168295916199606,
      "learning_rate": 2.874564459930314e-06,
      "logits/chosen": -3.547543525695801,
      "logits/rejected": -3.534101963043213,
      "logps/chosen": -38.00954055786133,
      "logps/rejected": -30.079078674316406,
      "loss": 0.3206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2733391523361206,
      "rewards/margins": 1.2733391523361206,
      "rewards/rejected": 0.0,
      "step": 165
    },
    {
      "epoch": 0.9273743016759777,
      "grad_norm": 12.81258421303075,
      "learning_rate": 2.8919860627177703e-06,
      "logits/chosen": -3.4986557960510254,
      "logits/rejected": -3.389159917831421,
      "logps/chosen": -8.50405502319336,
      "logps/rejected": -26.14635467529297,
      "loss": 0.363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4830951690673828,
      "rewards/margins": 0.4830951690673828,
      "rewards/rejected": 0.0,
      "step": 166
    },
    {
      "epoch": 0.9329608938547486,
      "grad_norm": 12.873328484366281,
      "learning_rate": 2.9094076655052266e-06,
      "logits/chosen": -3.5657150745391846,
      "logits/rejected": -3.3100697994232178,
      "logps/chosen": -20.26061248779297,
      "logps/rejected": -27.789817810058594,
      "loss": 0.3123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.962171196937561,
      "rewards/margins": 0.962171196937561,
      "rewards/rejected": 0.0,
      "step": 167
    },
    {
      "epoch": 0.9385474860335196,
      "grad_norm": 26.567617048457095,
      "learning_rate": 2.926829268292683e-06,
      "logits/chosen": -3.237934112548828,
      "logits/rejected": -3.3871254920959473,
      "logps/chosen": -13.236063957214355,
      "logps/rejected": -22.47895050048828,
      "loss": 0.5017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.778165340423584,
      "rewards/margins": 0.778165340423584,
      "rewards/rejected": 0.0,
      "step": 168
    },
    {
      "epoch": 0.9441340782122905,
      "grad_norm": 25.846257024168484,
      "learning_rate": 2.94425087108014e-06,
      "logits/chosen": -3.6739680767059326,
      "logits/rejected": -3.7278950214385986,
      "logps/chosen": -18.5478458404541,
      "logps/rejected": -28.209840774536133,
      "loss": 0.4307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9200146198272705,
      "rewards/margins": 0.9200146198272705,
      "rewards/rejected": 0.0,
      "step": 169
    },
    {
      "epoch": 0.9497206703910615,
      "grad_norm": 13.70645837092811,
      "learning_rate": 2.9616724738675963e-06,
      "logits/chosen": -3.370866060256958,
      "logits/rejected": -3.528033494949341,
      "logps/chosen": -27.201026916503906,
      "logps/rejected": -31.993549346923828,
      "loss": 0.3392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9744494557380676,
      "rewards/margins": 0.9744494557380676,
      "rewards/rejected": 0.0,
      "step": 170
    },
    {
      "epoch": 0.9553072625698324,
      "grad_norm": 19.19166327027628,
      "learning_rate": 2.9790940766550526e-06,
      "logits/chosen": -3.811903715133667,
      "logits/rejected": -3.689284563064575,
      "logps/chosen": -19.421972274780273,
      "logps/rejected": -34.969451904296875,
      "loss": 0.3743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0990993976593018,
      "rewards/margins": 1.0990993976593018,
      "rewards/rejected": 0.0,
      "step": 171
    },
    {
      "epoch": 0.9608938547486033,
      "grad_norm": 13.504256131976662,
      "learning_rate": 2.996515679442509e-06,
      "logits/chosen": -3.523135185241699,
      "logits/rejected": -3.4348065853118896,
      "logps/chosen": -18.443361282348633,
      "logps/rejected": -35.0711555480957,
      "loss": 0.3671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8971686363220215,
      "rewards/margins": 0.8971686363220215,
      "rewards/rejected": 0.0,
      "step": 172
    },
    {
      "epoch": 0.9664804469273743,
      "grad_norm": 13.794696000269452,
      "learning_rate": 3.0139372822299656e-06,
      "logits/chosen": -3.6839420795440674,
      "logits/rejected": -3.519639253616333,
      "logps/chosen": -19.81829833984375,
      "logps/rejected": -30.47394371032715,
      "loss": 0.3447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8508032560348511,
      "rewards/margins": 0.8508032560348511,
      "rewards/rejected": 0.0,
      "step": 173
    },
    {
      "epoch": 0.9720670391061452,
      "grad_norm": 11.403527276660718,
      "learning_rate": 3.031358885017422e-06,
      "logits/chosen": -3.3853251934051514,
      "logits/rejected": -3.530020236968994,
      "logps/chosen": -22.992746353149414,
      "logps/rejected": -53.81678771972656,
      "loss": 0.3253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8674898147583008,
      "rewards/margins": 0.8674898147583008,
      "rewards/rejected": 0.0,
      "step": 174
    },
    {
      "epoch": 0.9776536312849162,
      "grad_norm": 13.908950876033321,
      "learning_rate": 3.0487804878048782e-06,
      "logits/chosen": -3.235455274581909,
      "logits/rejected": -3.450087547302246,
      "logps/chosen": -55.55506134033203,
      "logps/rejected": -27.579978942871094,
      "loss": 0.3384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4162485599517822,
      "rewards/margins": 1.4162485599517822,
      "rewards/rejected": 0.0,
      "step": 175
    },
    {
      "epoch": 0.9832402234636871,
      "grad_norm": 13.352641881030522,
      "learning_rate": 3.0662020905923345e-06,
      "logits/chosen": -3.6966638565063477,
      "logits/rejected": -3.772146224975586,
      "logps/chosen": -10.773515701293945,
      "logps/rejected": -31.16201400756836,
      "loss": 0.3394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6668235063552856,
      "rewards/margins": 0.6668235063552856,
      "rewards/rejected": 0.0,
      "step": 176
    },
    {
      "epoch": 0.9888268156424581,
      "grad_norm": 24.240102874370475,
      "learning_rate": 3.0836236933797912e-06,
      "logits/chosen": -3.6584088802337646,
      "logits/rejected": -3.671759843826294,
      "logps/chosen": -16.63373565673828,
      "logps/rejected": -19.929149627685547,
      "loss": 0.4662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8085893392562866,
      "rewards/margins": 0.8085893392562866,
      "rewards/rejected": 0.0,
      "step": 177
    },
    {
      "epoch": 0.994413407821229,
      "grad_norm": 14.782169386749416,
      "learning_rate": 3.1010452961672475e-06,
      "logits/chosen": -3.4400901794433594,
      "logits/rejected": -3.476458787918091,
      "logps/chosen": -15.018159866333008,
      "logps/rejected": -50.66072463989258,
      "loss": 0.3814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.019031286239624,
      "rewards/margins": 1.019031286239624,
      "rewards/rejected": 0.0,
      "step": 178
    },
    {
      "epoch": 1.0,
      "grad_norm": 15.862340953715776,
      "learning_rate": 3.118466898954704e-06,
      "logits/chosen": -3.5413336753845215,
      "logits/rejected": -3.354410171508789,
      "logps/chosen": -27.737991333007812,
      "logps/rejected": -39.09562301635742,
      "loss": 0.3497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1306571960449219,
      "rewards/margins": 1.1306571960449219,
      "rewards/rejected": 0.0,
      "step": 179
    },
    {
      "epoch": 1.005586592178771,
      "grad_norm": 7.022287057732487,
      "learning_rate": 3.13588850174216e-06,
      "logits/chosen": -3.703089714050293,
      "logits/rejected": -3.657090902328491,
      "logps/chosen": -10.80535888671875,
      "logps/rejected": -18.158931732177734,
      "loss": 0.2556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.23873770236969,
      "rewards/margins": 1.23873770236969,
      "rewards/rejected": 0.0,
      "step": 180
    },
    {
      "epoch": 1.005586592178771,
      "eval_logits/chosen": -3.290764570236206,
      "eval_logits/rejected": -3.3175876140594482,
      "eval_logps/chosen": -20.149810791015625,
      "eval_logps/rejected": -31.72638511657715,
      "eval_loss": 0.3621155321598053,
      "eval_rewards/accuracies": 0.9750000238418579,
      "eval_rewards/chosen": 0.9920231699943542,
      "eval_rewards/margins": 0.9920231699943542,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5545,
      "eval_samples_per_second": 12.131,
      "eval_steps_per_second": 0.391,
      "step": 180
    },
    {
      "epoch": 1.011173184357542,
      "grad_norm": 8.376186496658782,
      "learning_rate": 3.1533101045296173e-06,
      "logits/chosen": -3.379521608352661,
      "logits/rejected": -3.4831197261810303,
      "logps/chosen": -11.817754745483398,
      "logps/rejected": -17.60910415649414,
      "loss": 0.2708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7589106559753418,
      "rewards/margins": 0.7589106559753418,
      "rewards/rejected": 0.0,
      "step": 181
    },
    {
      "epoch": 1.0167597765363128,
      "grad_norm": 7.941191248609296,
      "learning_rate": 3.1707317073170736e-06,
      "logits/chosen": -3.28135347366333,
      "logits/rejected": -3.387300968170166,
      "logps/chosen": -17.23596954345703,
      "logps/rejected": -19.249799728393555,
      "loss": 0.2249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.332868218421936,
      "rewards/margins": 1.332868218421936,
      "rewards/rejected": 0.0,
      "step": 182
    },
    {
      "epoch": 1.0223463687150838,
      "grad_norm": 6.567908698551698,
      "learning_rate": 3.18815331010453e-06,
      "logits/chosen": -3.225551128387451,
      "logits/rejected": -3.139052391052246,
      "logps/chosen": -14.68458366394043,
      "logps/rejected": -17.34343910217285,
      "loss": 0.2084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2798361778259277,
      "rewards/margins": 1.2798361778259277,
      "rewards/rejected": 0.0,
      "step": 183
    },
    {
      "epoch": 1.0279329608938548,
      "grad_norm": 9.478133942208041,
      "learning_rate": 3.205574912891986e-06,
      "logits/chosen": -3.5002570152282715,
      "logits/rejected": -3.1645779609680176,
      "logps/chosen": -13.268685340881348,
      "logps/rejected": -41.177059173583984,
      "loss": 0.266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5364269018173218,
      "rewards/margins": 1.5364269018173218,
      "rewards/rejected": 0.0,
      "step": 184
    },
    {
      "epoch": 1.0335195530726258,
      "grad_norm": 8.45318500266249,
      "learning_rate": 3.222996515679443e-06,
      "logits/chosen": -3.9391069412231445,
      "logits/rejected": -3.665884494781494,
      "logps/chosen": -13.378816604614258,
      "logps/rejected": -21.519012451171875,
      "loss": 0.2623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3343417644500732,
      "rewards/margins": 1.3343417644500732,
      "rewards/rejected": 0.0,
      "step": 185
    },
    {
      "epoch": 1.0391061452513966,
      "grad_norm": 9.586630039698045,
      "learning_rate": 3.240418118466899e-06,
      "logits/chosen": -3.8299098014831543,
      "logits/rejected": -3.6465885639190674,
      "logps/chosen": -18.336143493652344,
      "logps/rejected": -41.8558235168457,
      "loss": 0.251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8175854682922363,
      "rewards/margins": 1.8175854682922363,
      "rewards/rejected": 0.0,
      "step": 186
    },
    {
      "epoch": 1.0446927374301676,
      "grad_norm": 9.823401572888672,
      "learning_rate": 3.2578397212543555e-06,
      "logits/chosen": -3.58217453956604,
      "logits/rejected": -3.56648325920105,
      "logps/chosen": -17.7921199798584,
      "logps/rejected": -21.959434509277344,
      "loss": 0.2414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7480595111846924,
      "rewards/margins": 1.7480595111846924,
      "rewards/rejected": 0.0,
      "step": 187
    },
    {
      "epoch": 1.0502793296089385,
      "grad_norm": 8.563790217907744,
      "learning_rate": 3.2752613240418118e-06,
      "logits/chosen": -3.6265671253204346,
      "logits/rejected": -3.7216241359710693,
      "logps/chosen": -18.125682830810547,
      "logps/rejected": -45.22937774658203,
      "loss": 0.2408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1215565204620361,
      "rewards/margins": 1.1215565204620361,
      "rewards/rejected": 0.0,
      "step": 188
    },
    {
      "epoch": 1.0558659217877095,
      "grad_norm": 7.878644904436243,
      "learning_rate": 3.292682926829269e-06,
      "logits/chosen": -3.5503900051116943,
      "logits/rejected": -3.741912364959717,
      "logps/chosen": -11.355476379394531,
      "logps/rejected": -18.585906982421875,
      "loss": 0.2242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.082261323928833,
      "rewards/margins": 1.082261323928833,
      "rewards/rejected": 0.0,
      "step": 189
    },
    {
      "epoch": 1.0614525139664805,
      "grad_norm": 12.69134898491356,
      "learning_rate": 3.310104529616725e-06,
      "logits/chosen": -3.254089593887329,
      "logits/rejected": -3.4980621337890625,
      "logps/chosen": -24.628639221191406,
      "logps/rejected": -34.67395782470703,
      "loss": 0.2697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4680547714233398,
      "rewards/margins": 1.4680547714233398,
      "rewards/rejected": 0.0,
      "step": 190
    },
    {
      "epoch": 1.0670391061452513,
      "grad_norm": 9.163458725644189,
      "learning_rate": 3.3275261324041815e-06,
      "logits/chosen": -3.4832603931427,
      "logits/rejected": -3.421940803527832,
      "logps/chosen": -7.0905680656433105,
      "logps/rejected": -36.14787673950195,
      "loss": 0.2479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0431461334228516,
      "rewards/margins": 1.0431461334228516,
      "rewards/rejected": 0.0,
      "step": 191
    },
    {
      "epoch": 1.0726256983240223,
      "grad_norm": 12.297971677598818,
      "learning_rate": 3.3449477351916382e-06,
      "logits/chosen": -3.7893319129943848,
      "logits/rejected": -3.6246352195739746,
      "logps/chosen": -15.060861587524414,
      "logps/rejected": -42.71746063232422,
      "loss": 0.2417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.399600625038147,
      "rewards/margins": 1.399600625038147,
      "rewards/rejected": 0.0,
      "step": 192
    },
    {
      "epoch": 1.0782122905027933,
      "grad_norm": 15.363607187682483,
      "learning_rate": 3.3623693379790945e-06,
      "logits/chosen": -3.402564525604248,
      "logits/rejected": -3.4380149841308594,
      "logps/chosen": -12.444375038146973,
      "logps/rejected": -37.55222702026367,
      "loss": 0.2673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4193130731582642,
      "rewards/margins": 1.4193130731582642,
      "rewards/rejected": 0.0,
      "step": 193
    },
    {
      "epoch": 1.0837988826815643,
      "grad_norm": 10.675409717176816,
      "learning_rate": 3.379790940766551e-06,
      "logits/chosen": -2.934166193008423,
      "logits/rejected": -3.027604579925537,
      "logps/chosen": -12.435279846191406,
      "logps/rejected": -36.643310546875,
      "loss": 0.2604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4027774333953857,
      "rewards/margins": 1.4027774333953857,
      "rewards/rejected": 0.0,
      "step": 194
    },
    {
      "epoch": 1.089385474860335,
      "grad_norm": 17.31884756281499,
      "learning_rate": 3.397212543554007e-06,
      "logits/chosen": -3.3987059593200684,
      "logits/rejected": -3.37333083152771,
      "logps/chosen": -47.33638000488281,
      "logps/rejected": -27.212692260742188,
      "loss": 0.2864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1229281425476074,
      "rewards/margins": 1.1229281425476074,
      "rewards/rejected": 0.0,
      "step": 195
    },
    {
      "epoch": 1.094972067039106,
      "grad_norm": 9.33718072897061,
      "learning_rate": 3.414634146341464e-06,
      "logits/chosen": -3.759153127670288,
      "logits/rejected": -3.6373844146728516,
      "logps/chosen": -10.45933723449707,
      "logps/rejected": -55.07743835449219,
      "loss": 0.239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3542287349700928,
      "rewards/margins": 1.3542287349700928,
      "rewards/rejected": 0.0,
      "step": 196
    },
    {
      "epoch": 1.100558659217877,
      "grad_norm": 10.03949205130813,
      "learning_rate": 3.43205574912892e-06,
      "logits/chosen": -3.4442873001098633,
      "logits/rejected": -3.462092638015747,
      "logps/chosen": -10.042339324951172,
      "logps/rejected": -57.771240234375,
      "loss": 0.2283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.329883098602295,
      "rewards/margins": 1.329883098602295,
      "rewards/rejected": 0.0,
      "step": 197
    },
    {
      "epoch": 1.106145251396648,
      "grad_norm": 9.645649453611389,
      "learning_rate": 3.4494773519163764e-06,
      "logits/chosen": -3.740504026412964,
      "logits/rejected": -3.326383113861084,
      "logps/chosen": -3.412041187286377,
      "logps/rejected": -34.53120422363281,
      "loss": 0.2775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8617033958435059,
      "rewards/margins": 0.8617033958435059,
      "rewards/rejected": 0.0,
      "step": 198
    },
    {
      "epoch": 1.111731843575419,
      "grad_norm": 13.547013792946222,
      "learning_rate": 3.4668989547038327e-06,
      "logits/chosen": -3.569826602935791,
      "logits/rejected": -3.3062267303466797,
      "logps/chosen": -48.62763214111328,
      "logps/rejected": -39.51261520385742,
      "loss": 0.244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.3126307725906372,
      "rewards/margins": 1.3126307725906372,
      "rewards/rejected": 0.0,
      "step": 199
    },
    {
      "epoch": 1.1173184357541899,
      "grad_norm": 10.274813206716017,
      "learning_rate": 3.48432055749129e-06,
      "logits/chosen": -2.158135414123535,
      "logits/rejected": -2.2126452922821045,
      "logps/chosen": -43.74163818359375,
      "logps/rejected": -74.06904602050781,
      "loss": 0.2414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6305572986602783,
      "rewards/margins": 1.6305572986602783,
      "rewards/rejected": 0.0,
      "step": 200
    },
    {
      "epoch": 1.1173184357541899,
      "eval_logits/chosen": -3.4032585620880127,
      "eval_logits/rejected": -3.462423801422119,
      "eval_logps/chosen": -20.62289810180664,
      "eval_logps/rejected": -33.361961364746094,
      "eval_loss": 0.4005957841873169,
      "eval_rewards/accuracies": 0.925000011920929,
      "eval_rewards/chosen": 0.9447145462036133,
      "eval_rewards/margins": 0.9447145462036133,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6758,
      "eval_samples_per_second": 12.074,
      "eval_steps_per_second": 0.389,
      "step": 200
    },
    {
      "epoch": 1.1229050279329609,
      "grad_norm": 11.256368484298264,
      "learning_rate": 3.501742160278746e-06,
      "logits/chosen": -3.6768410205841064,
      "logits/rejected": -3.3176660537719727,
      "logps/chosen": -7.796246528625488,
      "logps/rejected": -56.44394302368164,
      "loss": 0.2859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9981168508529663,
      "rewards/margins": 0.9981168508529663,
      "rewards/rejected": 0.0,
      "step": 201
    },
    {
      "epoch": 1.1284916201117319,
      "grad_norm": 14.644107148926217,
      "learning_rate": 3.5191637630662025e-06,
      "logits/chosen": -3.4845829010009766,
      "logits/rejected": -3.550323724746704,
      "logps/chosen": -17.96717071533203,
      "logps/rejected": -16.248380661010742,
      "loss": 0.2927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0157550573349,
      "rewards/margins": 1.0157550573349,
      "rewards/rejected": 0.0,
      "step": 202
    },
    {
      "epoch": 1.1340782122905029,
      "grad_norm": 9.489455644743195,
      "learning_rate": 3.5365853658536588e-06,
      "logits/chosen": -3.530244827270508,
      "logits/rejected": -3.441255569458008,
      "logps/chosen": -24.940214157104492,
      "logps/rejected": -32.34701156616211,
      "loss": 0.2245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.352152705192566,
      "rewards/margins": 1.352152705192566,
      "rewards/rejected": 0.0,
      "step": 203
    },
    {
      "epoch": 1.1396648044692737,
      "grad_norm": 8.888360940188676,
      "learning_rate": 3.5540069686411155e-06,
      "logits/chosen": -3.528174877166748,
      "logits/rejected": -3.621192693710327,
      "logps/chosen": -13.649995803833008,
      "logps/rejected": -31.500852584838867,
      "loss": 0.2153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6193513870239258,
      "rewards/margins": 1.6193513870239258,
      "rewards/rejected": 0.0,
      "step": 204
    },
    {
      "epoch": 1.1452513966480447,
      "grad_norm": 10.71831442685278,
      "learning_rate": 3.5714285714285718e-06,
      "logits/chosen": -3.36834716796875,
      "logits/rejected": -3.424790143966675,
      "logps/chosen": -40.53661346435547,
      "logps/rejected": -30.97374153137207,
      "loss": 0.2579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8003473281860352,
      "rewards/margins": 1.8003473281860352,
      "rewards/rejected": 0.0,
      "step": 205
    },
    {
      "epoch": 1.1508379888268156,
      "grad_norm": 10.944981920099934,
      "learning_rate": 3.588850174216028e-06,
      "logits/chosen": -3.7728233337402344,
      "logits/rejected": -3.773298501968384,
      "logps/chosen": -16.62015151977539,
      "logps/rejected": -25.555927276611328,
      "loss": 0.2509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1026296615600586,
      "rewards/margins": 1.1026296615600586,
      "rewards/rejected": 0.0,
      "step": 206
    },
    {
      "epoch": 1.1564245810055866,
      "grad_norm": 9.855938529055885,
      "learning_rate": 3.6062717770034844e-06,
      "logits/chosen": -3.6583240032196045,
      "logits/rejected": -3.5246965885162354,
      "logps/chosen": -18.89337921142578,
      "logps/rejected": -31.112926483154297,
      "loss": 0.253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5077738761901855,
      "rewards/margins": 1.5077738761901855,
      "rewards/rejected": 0.0,
      "step": 207
    },
    {
      "epoch": 1.1620111731843576,
      "grad_norm": 9.69379427622202,
      "learning_rate": 3.623693379790941e-06,
      "logits/chosen": -3.5392768383026123,
      "logits/rejected": -3.695221185684204,
      "logps/chosen": -13.727815628051758,
      "logps/rejected": -57.575584411621094,
      "loss": 0.2426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.669374942779541,
      "rewards/margins": 0.669374942779541,
      "rewards/rejected": 0.0,
      "step": 208
    },
    {
      "epoch": 1.1675977653631284,
      "grad_norm": 18.694097708456376,
      "learning_rate": 3.6411149825783974e-06,
      "logits/chosen": -3.6019654273986816,
      "logits/rejected": -3.6211609840393066,
      "logps/chosen": -11.520591735839844,
      "logps/rejected": -27.420429229736328,
      "loss": 0.3202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0585640668869019,
      "rewards/margins": 1.0585640668869019,
      "rewards/rejected": 0.0,
      "step": 209
    },
    {
      "epoch": 1.1731843575418994,
      "grad_norm": 10.873250351471892,
      "learning_rate": 3.6585365853658537e-06,
      "logits/chosen": -3.5948538780212402,
      "logits/rejected": -3.5472543239593506,
      "logps/chosen": -14.013772964477539,
      "logps/rejected": -23.8314151763916,
      "loss": 0.2544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9067032933235168,
      "rewards/margins": 0.9067032933235168,
      "rewards/rejected": 0.0,
      "step": 210
    },
    {
      "epoch": 1.1787709497206704,
      "grad_norm": 15.485859755927704,
      "learning_rate": 3.67595818815331e-06,
      "logits/chosen": -3.3313405513763428,
      "logits/rejected": -3.072429895401001,
      "logps/chosen": -13.470560073852539,
      "logps/rejected": -33.34019088745117,
      "loss": 0.3016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6502934694290161,
      "rewards/margins": 1.6502934694290161,
      "rewards/rejected": 0.0,
      "step": 211
    },
    {
      "epoch": 1.1843575418994414,
      "grad_norm": 9.403591047190222,
      "learning_rate": 3.693379790940767e-06,
      "logits/chosen": -3.345604419708252,
      "logits/rejected": -3.2238712310791016,
      "logps/chosen": -14.749405860900879,
      "logps/rejected": -16.980634689331055,
      "loss": 0.2312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2261402606964111,
      "rewards/margins": 1.2261402606964111,
      "rewards/rejected": 0.0,
      "step": 212
    },
    {
      "epoch": 1.1899441340782122,
      "grad_norm": 13.090941110686423,
      "learning_rate": 3.7108013937282234e-06,
      "logits/chosen": -3.809760332107544,
      "logits/rejected": -3.793926954269409,
      "logps/chosen": -17.937414169311523,
      "logps/rejected": -52.34578323364258,
      "loss": 0.2777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6192893981933594,
      "rewards/margins": 1.6192893981933594,
      "rewards/rejected": 0.0,
      "step": 213
    },
    {
      "epoch": 1.1955307262569832,
      "grad_norm": 9.655773798660277,
      "learning_rate": 3.7282229965156797e-06,
      "logits/chosen": -3.536470890045166,
      "logits/rejected": -3.509004831314087,
      "logps/chosen": -23.860004425048828,
      "logps/rejected": -31.276809692382812,
      "loss": 0.2771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6330792903900146,
      "rewards/margins": 1.6330792903900146,
      "rewards/rejected": 0.0,
      "step": 214
    },
    {
      "epoch": 1.2011173184357542,
      "grad_norm": 13.6414616868088,
      "learning_rate": 3.745644599303136e-06,
      "logits/chosen": -3.776118516921997,
      "logits/rejected": -3.607973337173462,
      "logps/chosen": -26.299179077148438,
      "logps/rejected": -23.433828353881836,
      "loss": 0.24,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8330148458480835,
      "rewards/margins": 1.8330148458480835,
      "rewards/rejected": 0.0,
      "step": 215
    },
    {
      "epoch": 1.2067039106145252,
      "grad_norm": 14.847381009688226,
      "learning_rate": 3.7630662020905927e-06,
      "logits/chosen": -3.7925591468811035,
      "logits/rejected": -3.5007779598236084,
      "logps/chosen": -15.910371780395508,
      "logps/rejected": -29.95450210571289,
      "loss": 0.2587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.066568374633789,
      "rewards/margins": 2.066568374633789,
      "rewards/rejected": 0.0,
      "step": 216
    },
    {
      "epoch": 1.2122905027932962,
      "grad_norm": 8.422201756486825,
      "learning_rate": 3.780487804878049e-06,
      "logits/chosen": -3.720338821411133,
      "logits/rejected": -3.918419122695923,
      "logps/chosen": -14.455623626708984,
      "logps/rejected": -29.2593994140625,
      "loss": 0.2387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2446038722991943,
      "rewards/margins": 1.2446038722991943,
      "rewards/rejected": 0.0,
      "step": 217
    },
    {
      "epoch": 1.217877094972067,
      "grad_norm": 11.951505397007134,
      "learning_rate": 3.7979094076655053e-06,
      "logits/chosen": -3.3473784923553467,
      "logits/rejected": -3.243225336074829,
      "logps/chosen": -19.990514755249023,
      "logps/rejected": -28.22415542602539,
      "loss": 0.2866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5900161266326904,
      "rewards/margins": 1.5900161266326904,
      "rewards/rejected": 0.0,
      "step": 218
    },
    {
      "epoch": 1.223463687150838,
      "grad_norm": 7.086311280798665,
      "learning_rate": 3.815331010452962e-06,
      "logits/chosen": -3.3548178672790527,
      "logits/rejected": -3.277843952178955,
      "logps/chosen": -20.96944808959961,
      "logps/rejected": -37.220584869384766,
      "loss": 0.2006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.78322172164917,
      "rewards/margins": 1.78322172164917,
      "rewards/rejected": 0.0,
      "step": 219
    },
    {
      "epoch": 1.229050279329609,
      "grad_norm": 15.176074623583974,
      "learning_rate": 3.832752613240418e-06,
      "logits/chosen": -3.5489754676818848,
      "logits/rejected": -3.7463839054107666,
      "logps/chosen": -41.65248107910156,
      "logps/rejected": -41.696922302246094,
      "loss": 0.2905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7217035293579102,
      "rewards/margins": 1.7217035293579102,
      "rewards/rejected": 0.0,
      "step": 220
    },
    {
      "epoch": 1.229050279329609,
      "eval_logits/chosen": -3.409276247024536,
      "eval_logits/rejected": -3.4644932746887207,
      "eval_logps/chosen": -20.847408294677734,
      "eval_logps/rejected": -32.939964294433594,
      "eval_loss": 0.40232640504837036,
      "eval_rewards/accuracies": 0.8999999761581421,
      "eval_rewards/chosen": 0.9222632646560669,
      "eval_rewards/margins": 0.9222632646560669,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5218,
      "eval_samples_per_second": 12.146,
      "eval_steps_per_second": 0.392,
      "step": 220
    },
    {
      "epoch": 1.23463687150838,
      "grad_norm": 8.549311810824502,
      "learning_rate": 3.850174216027875e-06,
      "logits/chosen": -3.668447256088257,
      "logits/rejected": -3.6480414867401123,
      "logps/chosen": -11.687830924987793,
      "logps/rejected": -33.238399505615234,
      "loss": 0.2675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6335495710372925,
      "rewards/margins": 1.6335495710372925,
      "rewards/rejected": 0.0,
      "step": 221
    },
    {
      "epoch": 1.2402234636871508,
      "grad_norm": 10.173448731836723,
      "learning_rate": 3.867595818815331e-06,
      "logits/chosen": -3.624209403991699,
      "logits/rejected": -3.596116542816162,
      "logps/chosen": -19.490001678466797,
      "logps/rejected": -17.541736602783203,
      "loss": 0.2825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1532347202301025,
      "rewards/margins": 1.1532347202301025,
      "rewards/rejected": 0.0,
      "step": 222
    },
    {
      "epoch": 1.2458100558659218,
      "grad_norm": 18.531547016133164,
      "learning_rate": 3.885017421602787e-06,
      "logits/chosen": -3.4726502895355225,
      "logits/rejected": -3.640723943710327,
      "logps/chosen": -6.3072428703308105,
      "logps/rejected": -38.85729217529297,
      "loss": 0.2956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.063659906387329,
      "rewards/margins": 1.063659906387329,
      "rewards/rejected": 0.0,
      "step": 223
    },
    {
      "epoch": 1.2513966480446927,
      "grad_norm": 14.040179487109642,
      "learning_rate": 3.902439024390244e-06,
      "logits/chosen": -3.71232533454895,
      "logits/rejected": -3.872772455215454,
      "logps/chosen": -15.276671409606934,
      "logps/rejected": -24.9880428314209,
      "loss": 0.3332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3843878507614136,
      "rewards/margins": 1.3843878507614136,
      "rewards/rejected": 0.0,
      "step": 224
    },
    {
      "epoch": 1.2569832402234637,
      "grad_norm": 7.529043278767307,
      "learning_rate": 3.919860627177701e-06,
      "logits/chosen": -3.1976165771484375,
      "logits/rejected": -3.219167470932007,
      "logps/chosen": -44.19377136230469,
      "logps/rejected": -30.909975051879883,
      "loss": 0.2162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0954546928405762,
      "rewards/margins": 1.0954546928405762,
      "rewards/rejected": 0.0,
      "step": 225
    },
    {
      "epoch": 1.2625698324022347,
      "grad_norm": 12.3077678796358,
      "learning_rate": 3.937282229965157e-06,
      "logits/chosen": -3.689211130142212,
      "logits/rejected": -3.639636754989624,
      "logps/chosen": -28.675701141357422,
      "logps/rejected": -28.942533493041992,
      "loss": 0.2884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.943276047706604,
      "rewards/margins": 0.943276047706604,
      "rewards/rejected": 0.0,
      "step": 226
    },
    {
      "epoch": 1.2681564245810055,
      "grad_norm": 10.876253077519584,
      "learning_rate": 3.954703832752614e-06,
      "logits/chosen": -3.535916328430176,
      "logits/rejected": -3.509960412979126,
      "logps/chosen": -21.791345596313477,
      "logps/rejected": -37.74628448486328,
      "loss": 0.2555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2623603343963623,
      "rewards/margins": 2.2623603343963623,
      "rewards/rejected": 0.0,
      "step": 227
    },
    {
      "epoch": 1.2737430167597765,
      "grad_norm": 11.571854539456316,
      "learning_rate": 3.97212543554007e-06,
      "logits/chosen": -3.1920554637908936,
      "logits/rejected": -3.263157844543457,
      "logps/chosen": -24.391693115234375,
      "logps/rejected": -31.14853286743164,
      "loss": 0.2636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9780235290527344,
      "rewards/margins": 0.9780235290527344,
      "rewards/rejected": 0.0,
      "step": 228
    },
    {
      "epoch": 1.2793296089385475,
      "grad_norm": 10.439575604120233,
      "learning_rate": 3.989547038327527e-06,
      "logits/chosen": -3.4396400451660156,
      "logits/rejected": -3.5043418407440186,
      "logps/chosen": -32.503536224365234,
      "logps/rejected": -55.587913513183594,
      "loss": 0.2194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3546091318130493,
      "rewards/margins": 1.3546091318130493,
      "rewards/rejected": 0.0,
      "step": 229
    },
    {
      "epoch": 1.2849162011173183,
      "grad_norm": 13.104955978524666,
      "learning_rate": 4.006968641114983e-06,
      "logits/chosen": -3.388406753540039,
      "logits/rejected": -3.2240676879882812,
      "logps/chosen": -10.35389518737793,
      "logps/rejected": -38.71877670288086,
      "loss": 0.2765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5072083473205566,
      "rewards/margins": 1.5072083473205566,
      "rewards/rejected": 0.0,
      "step": 230
    },
    {
      "epoch": 1.2905027932960893,
      "grad_norm": 16.402721599941056,
      "learning_rate": 4.024390243902439e-06,
      "logits/chosen": -3.7177958488464355,
      "logits/rejected": -3.721942663192749,
      "logps/chosen": -13.645185470581055,
      "logps/rejected": -28.600276947021484,
      "loss": 0.3124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9151885509490967,
      "rewards/margins": 0.9151885509490967,
      "rewards/rejected": 0.0,
      "step": 231
    },
    {
      "epoch": 1.2960893854748603,
      "grad_norm": 17.055853069033,
      "learning_rate": 4.041811846689896e-06,
      "logits/chosen": -3.4767673015594482,
      "logits/rejected": -3.198622226715088,
      "logps/chosen": -26.67482566833496,
      "logps/rejected": -113.12110900878906,
      "loss": 0.3238,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3831629157066345,
      "rewards/margins": 0.3831629157066345,
      "rewards/rejected": 0.0,
      "step": 232
    },
    {
      "epoch": 1.3016759776536313,
      "grad_norm": 11.78420665078755,
      "learning_rate": 4.059233449477352e-06,
      "logits/chosen": -2.6962203979492188,
      "logits/rejected": -2.8129491806030273,
      "logps/chosen": -37.849361419677734,
      "logps/rejected": -35.45374298095703,
      "loss": 0.2614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.339527130126953,
      "rewards/margins": 2.339527130126953,
      "rewards/rejected": 0.0,
      "step": 233
    },
    {
      "epoch": 1.3072625698324023,
      "grad_norm": 9.254106592562872,
      "learning_rate": 4.076655052264808e-06,
      "logits/chosen": -3.930264949798584,
      "logits/rejected": -3.920576572418213,
      "logps/chosen": -14.508851051330566,
      "logps/rejected": -33.40388488769531,
      "loss": 0.2434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5266382694244385,
      "rewards/margins": 1.5266382694244385,
      "rewards/rejected": 0.0,
      "step": 234
    },
    {
      "epoch": 1.3128491620111733,
      "grad_norm": 8.251092703736992,
      "learning_rate": 4.094076655052265e-06,
      "logits/chosen": -2.9808385372161865,
      "logits/rejected": -3.293715715408325,
      "logps/chosen": -38.886138916015625,
      "logps/rejected": -21.410964965820312,
      "loss": 0.2559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.231886386871338,
      "rewards/margins": 2.231886386871338,
      "rewards/rejected": 0.0,
      "step": 235
    },
    {
      "epoch": 1.318435754189944,
      "grad_norm": 8.654567517554257,
      "learning_rate": 4.111498257839722e-06,
      "logits/chosen": -3.789379119873047,
      "logits/rejected": -3.8351480960845947,
      "logps/chosen": -27.43928337097168,
      "logps/rejected": -27.69175148010254,
      "loss": 0.2138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0133039951324463,
      "rewards/margins": 2.0133039951324463,
      "rewards/rejected": 0.0,
      "step": 236
    },
    {
      "epoch": 1.324022346368715,
      "grad_norm": 9.787787230915328,
      "learning_rate": 4.128919860627178e-06,
      "logits/chosen": -3.447953224182129,
      "logits/rejected": -3.667243003845215,
      "logps/chosen": -5.6970014572143555,
      "logps/rejected": -54.013694763183594,
      "loss": 0.2525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2237179279327393,
      "rewards/margins": 1.2237179279327393,
      "rewards/rejected": 0.0,
      "step": 237
    },
    {
      "epoch": 1.329608938547486,
      "grad_norm": 24.37951134979593,
      "learning_rate": 4.146341463414634e-06,
      "logits/chosen": -3.7857072353363037,
      "logits/rejected": -3.565685510635376,
      "logps/chosen": -23.67583465576172,
      "logps/rejected": -28.298423767089844,
      "loss": 0.2599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0864548683166504,
      "rewards/margins": 2.0864548683166504,
      "rewards/rejected": 0.0,
      "step": 238
    },
    {
      "epoch": 1.3351955307262569,
      "grad_norm": 10.95225223984137,
      "learning_rate": 4.163763066202091e-06,
      "logits/chosen": -3.657374382019043,
      "logits/rejected": -3.669152021408081,
      "logps/chosen": -11.589214324951172,
      "logps/rejected": -56.517494201660156,
      "loss": 0.2696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4046443700790405,
      "rewards/margins": 1.4046443700790405,
      "rewards/rejected": 0.0,
      "step": 239
    },
    {
      "epoch": 1.3407821229050279,
      "grad_norm": 9.546420230951075,
      "learning_rate": 4.181184668989548e-06,
      "logits/chosen": -3.5277018547058105,
      "logits/rejected": -3.36171555519104,
      "logps/chosen": -15.049688339233398,
      "logps/rejected": -40.23430633544922,
      "loss": 0.2398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2863930463790894,
      "rewards/margins": 1.2863930463790894,
      "rewards/rejected": 0.0,
      "step": 240
    },
    {
      "epoch": 1.3407821229050279,
      "eval_logits/chosen": -3.4341378211975098,
      "eval_logits/rejected": -3.5001742839813232,
      "eval_logps/chosen": -21.054502487182617,
      "eval_logps/rejected": -34.73943328857422,
      "eval_loss": 0.43234288692474365,
      "eval_rewards/accuracies": 0.925000011920929,
      "eval_rewards/chosen": 0.9015538096427917,
      "eval_rewards/margins": 0.9015538096427917,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5934,
      "eval_samples_per_second": 12.112,
      "eval_steps_per_second": 0.391,
      "step": 240
    },
    {
      "epoch": 1.3463687150837989,
      "grad_norm": 10.18117094327002,
      "learning_rate": 4.198606271777004e-06,
      "logits/chosen": -3.7662901878356934,
      "logits/rejected": -3.751497268676758,
      "logps/chosen": -15.95394515991211,
      "logps/rejected": -35.43588638305664,
      "loss": 0.2631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3605461120605469,
      "rewards/margins": 1.3605461120605469,
      "rewards/rejected": 0.0,
      "step": 241
    },
    {
      "epoch": 1.3519553072625698,
      "grad_norm": 15.659006008787635,
      "learning_rate": 4.21602787456446e-06,
      "logits/chosen": -2.7622427940368652,
      "logits/rejected": -2.796217679977417,
      "logps/chosen": -18.15198516845703,
      "logps/rejected": -25.631563186645508,
      "loss": 0.3009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4997844696044922,
      "rewards/margins": 1.4997844696044922,
      "rewards/rejected": 0.0,
      "step": 242
    },
    {
      "epoch": 1.3575418994413408,
      "grad_norm": 14.782655608525339,
      "learning_rate": 4.2334494773519166e-06,
      "logits/chosen": -3.780031681060791,
      "logits/rejected": -3.943005084991455,
      "logps/chosen": -13.740594863891602,
      "logps/rejected": -31.765470504760742,
      "loss": 0.2975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6437757015228271,
      "rewards/margins": 0.6437757015228271,
      "rewards/rejected": 0.0,
      "step": 243
    },
    {
      "epoch": 1.3631284916201118,
      "grad_norm": 10.322854683976361,
      "learning_rate": 4.250871080139373e-06,
      "logits/chosen": -3.785707473754883,
      "logits/rejected": -3.7427587509155273,
      "logps/chosen": -14.411982536315918,
      "logps/rejected": -28.815702438354492,
      "loss": 0.2207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4109615087509155,
      "rewards/margins": 1.4109615087509155,
      "rewards/rejected": 0.0,
      "step": 244
    },
    {
      "epoch": 1.3687150837988826,
      "grad_norm": 11.308029251568392,
      "learning_rate": 4.268292682926829e-06,
      "logits/chosen": -3.5322625637054443,
      "logits/rejected": -3.316295623779297,
      "logps/chosen": -17.916444778442383,
      "logps/rejected": -29.387725830078125,
      "loss": 0.2376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0318412780761719,
      "rewards/margins": 1.0318412780761719,
      "rewards/rejected": 0.0,
      "step": 245
    },
    {
      "epoch": 1.3743016759776536,
      "grad_norm": 15.557221130741413,
      "learning_rate": 4.2857142857142855e-06,
      "logits/chosen": -3.8445544242858887,
      "logits/rejected": -3.6276297569274902,
      "logps/chosen": -21.44609832763672,
      "logps/rejected": -29.25690460205078,
      "loss": 0.3119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8193881511688232,
      "rewards/margins": 1.8193881511688232,
      "rewards/rejected": 0.0,
      "step": 246
    },
    {
      "epoch": 1.3798882681564246,
      "grad_norm": 10.719091820757608,
      "learning_rate": 4.303135888501743e-06,
      "logits/chosen": -3.245727062225342,
      "logits/rejected": -3.333449125289917,
      "logps/chosen": -11.272308349609375,
      "logps/rejected": -35.163536071777344,
      "loss": 0.2824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6696861982345581,
      "rewards/margins": 0.6696861982345581,
      "rewards/rejected": 0.0,
      "step": 247
    },
    {
      "epoch": 1.3854748603351954,
      "grad_norm": 11.165854061300672,
      "learning_rate": 4.320557491289199e-06,
      "logits/chosen": -3.422884941101074,
      "logits/rejected": -3.4054672718048096,
      "logps/chosen": -14.373163223266602,
      "logps/rejected": -44.08214569091797,
      "loss": 0.239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6517525911331177,
      "rewards/margins": 1.6517525911331177,
      "rewards/rejected": 0.0,
      "step": 248
    },
    {
      "epoch": 1.3910614525139664,
      "grad_norm": 12.998533409884022,
      "learning_rate": 4.337979094076655e-06,
      "logits/chosen": -3.78513765335083,
      "logits/rejected": -3.665369749069214,
      "logps/chosen": -14.236440658569336,
      "logps/rejected": -24.677932739257812,
      "loss": 0.2411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5018088817596436,
      "rewards/margins": 1.5018088817596436,
      "rewards/rejected": 0.0,
      "step": 249
    },
    {
      "epoch": 1.3966480446927374,
      "grad_norm": 12.126632985885687,
      "learning_rate": 4.3554006968641115e-06,
      "logits/chosen": -3.837740659713745,
      "logits/rejected": -3.8506643772125244,
      "logps/chosen": -13.64283561706543,
      "logps/rejected": -73.83622741699219,
      "loss": 0.2592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6358001232147217,
      "rewards/margins": 0.6358001232147217,
      "rewards/rejected": 0.0,
      "step": 250
    },
    {
      "epoch": 1.4022346368715084,
      "grad_norm": 8.290283817480663,
      "learning_rate": 4.372822299651569e-06,
      "logits/chosen": -3.3841419219970703,
      "logits/rejected": -3.4281210899353027,
      "logps/chosen": -41.78927993774414,
      "logps/rejected": -25.866710662841797,
      "loss": 0.2194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.71621572971344,
      "rewards/margins": 1.71621572971344,
      "rewards/rejected": 0.0,
      "step": 251
    },
    {
      "epoch": 1.4078212290502794,
      "grad_norm": 8.562897032839903,
      "learning_rate": 4.390243902439025e-06,
      "logits/chosen": -3.664454460144043,
      "logits/rejected": -3.3467624187469482,
      "logps/chosen": -27.251789093017578,
      "logps/rejected": -27.237363815307617,
      "loss": 0.2348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9396095275878906,
      "rewards/margins": 1.9396095275878906,
      "rewards/rejected": 0.0,
      "step": 252
    },
    {
      "epoch": 1.4134078212290504,
      "grad_norm": 14.14222789536397,
      "learning_rate": 4.407665505226481e-06,
      "logits/chosen": -4.1109724044799805,
      "logits/rejected": -3.9541358947753906,
      "logps/chosen": -16.12481117248535,
      "logps/rejected": -41.76468276977539,
      "loss": 0.3046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4873075485229492,
      "rewards/margins": 1.4873075485229492,
      "rewards/rejected": 0.0,
      "step": 253
    },
    {
      "epoch": 1.4189944134078212,
      "grad_norm": 13.304438416457653,
      "learning_rate": 4.4250871080139375e-06,
      "logits/chosen": -3.450211763381958,
      "logits/rejected": -3.4964663982391357,
      "logps/chosen": -12.438718795776367,
      "logps/rejected": -32.481536865234375,
      "loss": 0.292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0299915075302124,
      "rewards/margins": 1.0299915075302124,
      "rewards/rejected": 0.0,
      "step": 254
    },
    {
      "epoch": 1.4245810055865922,
      "grad_norm": 9.216769556926586,
      "learning_rate": 4.442508710801394e-06,
      "logits/chosen": -3.819256544113159,
      "logits/rejected": -3.7547171115875244,
      "logps/chosen": -13.066776275634766,
      "logps/rejected": -27.592281341552734,
      "loss": 0.2325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.442773461341858,
      "rewards/margins": 1.442773461341858,
      "rewards/rejected": 0.0,
      "step": 255
    },
    {
      "epoch": 1.4301675977653632,
      "grad_norm": 18.108599213593603,
      "learning_rate": 4.45993031358885e-06,
      "logits/chosen": -3.6777217388153076,
      "logits/rejected": -3.7299575805664062,
      "logps/chosen": -9.910593032836914,
      "logps/rejected": -23.844945907592773,
      "loss": 0.3482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7967113852500916,
      "rewards/margins": 0.7967113852500916,
      "rewards/rejected": 0.0,
      "step": 256
    },
    {
      "epoch": 1.435754189944134,
      "grad_norm": 14.709735546372627,
      "learning_rate": 4.477351916376306e-06,
      "logits/chosen": -3.548936605453491,
      "logits/rejected": -3.436323881149292,
      "logps/chosen": -21.526458740234375,
      "logps/rejected": -35.82958984375,
      "loss": 0.271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.515000343322754,
      "rewards/margins": 1.515000343322754,
      "rewards/rejected": 0.0,
      "step": 257
    },
    {
      "epoch": 1.441340782122905,
      "grad_norm": 12.073100455204953,
      "learning_rate": 4.4947735191637636e-06,
      "logits/chosen": -3.5760366916656494,
      "logits/rejected": -3.5511622428894043,
      "logps/chosen": -23.415634155273438,
      "logps/rejected": -32.68937683105469,
      "loss": 0.2687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8378002643585205,
      "rewards/margins": 1.8378002643585205,
      "rewards/rejected": 0.0,
      "step": 258
    },
    {
      "epoch": 1.446927374301676,
      "grad_norm": 12.150869924622258,
      "learning_rate": 4.51219512195122e-06,
      "logits/chosen": -3.3420419692993164,
      "logits/rejected": -3.200608015060425,
      "logps/chosen": -16.347652435302734,
      "logps/rejected": -41.95685577392578,
      "loss": 0.2791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0902082920074463,
      "rewards/margins": 1.0902082920074463,
      "rewards/rejected": 0.0,
      "step": 259
    },
    {
      "epoch": 1.452513966480447,
      "grad_norm": 13.532979261737815,
      "learning_rate": 4.529616724738676e-06,
      "logits/chosen": -3.8272554874420166,
      "logits/rejected": -3.915024518966675,
      "logps/chosen": -10.427151679992676,
      "logps/rejected": -26.84328842163086,
      "loss": 0.3179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9687432050704956,
      "rewards/margins": 0.9687432050704956,
      "rewards/rejected": 0.0,
      "step": 260
    },
    {
      "epoch": 1.452513966480447,
      "eval_logits/chosen": -3.4396655559539795,
      "eval_logits/rejected": -3.4989960193634033,
      "eval_logps/chosen": -20.880056381225586,
      "eval_logps/rejected": -34.23080062866211,
      "eval_loss": 0.44072094559669495,
      "eval_rewards/accuracies": 0.925000011920929,
      "eval_rewards/chosen": 0.9189985990524292,
      "eval_rewards/margins": 0.9189985990524292,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5968,
      "eval_samples_per_second": 12.111,
      "eval_steps_per_second": 0.391,
      "step": 260
    },
    {
      "epoch": 1.458100558659218,
      "grad_norm": 12.454325797249647,
      "learning_rate": 4.5470383275261325e-06,
      "logits/chosen": -3.5631842613220215,
      "logits/rejected": -3.706714391708374,
      "logps/chosen": -23.585498809814453,
      "logps/rejected": -23.53061294555664,
      "loss": 0.2706,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1001571416854858,
      "rewards/margins": 1.1001571416854858,
      "rewards/rejected": 0.0,
      "step": 261
    },
    {
      "epoch": 1.463687150837989,
      "grad_norm": 10.260827180873068,
      "learning_rate": 4.56445993031359e-06,
      "logits/chosen": -3.364802837371826,
      "logits/rejected": -3.2886526584625244,
      "logps/chosen": -33.09785079956055,
      "logps/rejected": -46.10432434082031,
      "loss": 0.2605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2655953168869019,
      "rewards/margins": 1.2655953168869019,
      "rewards/rejected": 0.0,
      "step": 262
    },
    {
      "epoch": 1.4692737430167597,
      "grad_norm": 11.086034776671553,
      "learning_rate": 4.581881533101046e-06,
      "logits/chosen": -3.1375889778137207,
      "logits/rejected": -3.227323293685913,
      "logps/chosen": -9.931549072265625,
      "logps/rejected": -16.99380111694336,
      "loss": 0.2875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2103887796401978,
      "rewards/margins": 1.2103887796401978,
      "rewards/rejected": 0.0,
      "step": 263
    },
    {
      "epoch": 1.4748603351955307,
      "grad_norm": 12.319335479739904,
      "learning_rate": 4.599303135888502e-06,
      "logits/chosen": -3.8864805698394775,
      "logits/rejected": -3.7573463916778564,
      "logps/chosen": -21.3018798828125,
      "logps/rejected": -30.8250789642334,
      "loss": 0.2803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.065790057182312,
      "rewards/margins": 1.065790057182312,
      "rewards/rejected": 0.0,
      "step": 264
    },
    {
      "epoch": 1.4804469273743017,
      "grad_norm": 11.198844057520986,
      "learning_rate": 4.6167247386759585e-06,
      "logits/chosen": -3.5912914276123047,
      "logits/rejected": -3.3209681510925293,
      "logps/chosen": -12.173465728759766,
      "logps/rejected": -72.7391128540039,
      "loss": 0.2666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4054875373840332,
      "rewards/margins": 1.4054875373840332,
      "rewards/rejected": 0.0,
      "step": 265
    },
    {
      "epoch": 1.4860335195530725,
      "grad_norm": 11.35798086592169,
      "learning_rate": 4.634146341463416e-06,
      "logits/chosen": -3.599052667617798,
      "logits/rejected": -3.5266199111938477,
      "logps/chosen": -15.70073127746582,
      "logps/rejected": -32.182167053222656,
      "loss": 0.2632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.104032039642334,
      "rewards/margins": 1.104032039642334,
      "rewards/rejected": 0.0,
      "step": 266
    },
    {
      "epoch": 1.4916201117318435,
      "grad_norm": 10.850695282794945,
      "learning_rate": 4.651567944250872e-06,
      "logits/chosen": -3.4162943363189697,
      "logits/rejected": -3.7148170471191406,
      "logps/chosen": -31.101173400878906,
      "logps/rejected": -46.19312286376953,
      "loss": 0.2624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.704024076461792,
      "rewards/margins": 1.704024076461792,
      "rewards/rejected": 0.0,
      "step": 267
    },
    {
      "epoch": 1.4972067039106145,
      "grad_norm": 8.716719693433088,
      "learning_rate": 4.668989547038328e-06,
      "logits/chosen": -3.347233533859253,
      "logits/rejected": -3.0602099895477295,
      "logps/chosen": -28.352867126464844,
      "logps/rejected": -36.049285888671875,
      "loss": 0.2383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2565803527832031,
      "rewards/margins": 1.2565803527832031,
      "rewards/rejected": 0.0,
      "step": 268
    },
    {
      "epoch": 1.5027932960893855,
      "grad_norm": 9.138503870880099,
      "learning_rate": 4.6864111498257845e-06,
      "logits/chosen": -3.4606969356536865,
      "logits/rejected": -3.3620502948760986,
      "logps/chosen": -13.7554292678833,
      "logps/rejected": -55.7286262512207,
      "loss": 0.2779,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6413142681121826,
      "rewards/margins": 1.6413142681121826,
      "rewards/rejected": 0.0,
      "step": 269
    },
    {
      "epoch": 1.5083798882681565,
      "grad_norm": 14.03272055424108,
      "learning_rate": 4.703832752613241e-06,
      "logits/chosen": -3.344608783721924,
      "logits/rejected": -3.5553927421569824,
      "logps/chosen": -37.084930419921875,
      "logps/rejected": -28.88469886779785,
      "loss": 0.292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7738668322563171,
      "rewards/margins": 0.7738668322563171,
      "rewards/rejected": 0.0,
      "step": 270
    },
    {
      "epoch": 1.5139664804469275,
      "grad_norm": 8.858514031872641,
      "learning_rate": 4.721254355400697e-06,
      "logits/chosen": -3.866206407546997,
      "logits/rejected": -3.762759208679199,
      "logps/chosen": -19.737865447998047,
      "logps/rejected": -27.114578247070312,
      "loss": 0.2471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1749141216278076,
      "rewards/margins": 1.1749141216278076,
      "rewards/rejected": 0.0,
      "step": 271
    },
    {
      "epoch": 1.5195530726256983,
      "grad_norm": 10.953993752944418,
      "learning_rate": 4.738675958188153e-06,
      "logits/chosen": -3.638108968734741,
      "logits/rejected": -2.957117795944214,
      "logps/chosen": -25.93838882446289,
      "logps/rejected": -52.8266487121582,
      "loss": 0.3056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4903568029403687,
      "rewards/margins": 1.4903568029403687,
      "rewards/rejected": 0.0,
      "step": 272
    },
    {
      "epoch": 1.5251396648044693,
      "grad_norm": 18.292502809816778,
      "learning_rate": 4.75609756097561e-06,
      "logits/chosen": -3.684561014175415,
      "logits/rejected": -3.712930917739868,
      "logps/chosen": -16.731224060058594,
      "logps/rejected": -24.5842342376709,
      "loss": 0.2772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0367354154586792,
      "rewards/margins": 1.0367354154586792,
      "rewards/rejected": 0.0,
      "step": 273
    },
    {
      "epoch": 1.5307262569832403,
      "grad_norm": 15.878775609604954,
      "learning_rate": 4.773519163763067e-06,
      "logits/chosen": -3.6799376010894775,
      "logits/rejected": -3.143343687057495,
      "logps/chosen": -19.85594367980957,
      "logps/rejected": -60.51586151123047,
      "loss": 0.3199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9860464334487915,
      "rewards/margins": 0.9860464334487915,
      "rewards/rejected": 0.0,
      "step": 274
    },
    {
      "epoch": 1.536312849162011,
      "grad_norm": 18.692430252368645,
      "learning_rate": 4.790940766550523e-06,
      "logits/chosen": -3.183251142501831,
      "logits/rejected": -3.367392063140869,
      "logps/chosen": -32.86224365234375,
      "logps/rejected": -32.807777404785156,
      "loss": 0.3295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.422969102859497,
      "rewards/margins": 1.422969102859497,
      "rewards/rejected": 0.0,
      "step": 275
    },
    {
      "epoch": 1.541899441340782,
      "grad_norm": 16.29266302069184,
      "learning_rate": 4.8083623693379794e-06,
      "logits/chosen": -3.476862668991089,
      "logits/rejected": -3.5308008193969727,
      "logps/chosen": -27.919349670410156,
      "logps/rejected": -34.2692756652832,
      "loss": 0.3256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1970975399017334,
      "rewards/margins": 1.1970975399017334,
      "rewards/rejected": 0.0,
      "step": 276
    },
    {
      "epoch": 1.547486033519553,
      "grad_norm": 15.887898533601508,
      "learning_rate": 4.825783972125436e-06,
      "logits/chosen": -3.48037052154541,
      "logits/rejected": -3.6866586208343506,
      "logps/chosen": -38.57249450683594,
      "logps/rejected": -33.640724182128906,
      "loss": 0.2916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0205070972442627,
      "rewards/margins": 2.0205070972442627,
      "rewards/rejected": 0.0,
      "step": 277
    },
    {
      "epoch": 1.553072625698324,
      "grad_norm": 15.383221123486878,
      "learning_rate": 4.843205574912893e-06,
      "logits/chosen": -3.718332529067993,
      "logits/rejected": -3.5420737266540527,
      "logps/chosen": -15.735451698303223,
      "logps/rejected": -30.01406478881836,
      "loss": 0.319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0062814950942993,
      "rewards/margins": 1.0062814950942993,
      "rewards/rejected": 0.0,
      "step": 278
    },
    {
      "epoch": 1.558659217877095,
      "grad_norm": 21.00686389346566,
      "learning_rate": 4.860627177700349e-06,
      "logits/chosen": -3.5888757705688477,
      "logits/rejected": -3.2339837551116943,
      "logps/chosen": -20.475627899169922,
      "logps/rejected": -25.02223014831543,
      "loss": 0.4999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6502015590667725,
      "rewards/margins": 1.6502015590667725,
      "rewards/rejected": 0.0,
      "step": 279
    },
    {
      "epoch": 1.564245810055866,
      "grad_norm": 15.633757473781014,
      "learning_rate": 4.8780487804878055e-06,
      "logits/chosen": -3.4400904178619385,
      "logits/rejected": -3.5991039276123047,
      "logps/chosen": -40.779930114746094,
      "logps/rejected": -33.4875373840332,
      "loss": 0.3244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6489032506942749,
      "rewards/margins": 0.6489032506942749,
      "rewards/rejected": 0.0,
      "step": 280
    },
    {
      "epoch": 1.564245810055866,
      "eval_logits/chosen": -3.3496432304382324,
      "eval_logits/rejected": -3.393968105316162,
      "eval_logps/chosen": -22.369251251220703,
      "eval_logps/rejected": -34.92453384399414,
      "eval_loss": 0.46735382080078125,
      "eval_rewards/accuracies": 0.8999999761581421,
      "eval_rewards/chosen": 0.7700790166854858,
      "eval_rewards/margins": 0.7700790166854858,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6515,
      "eval_samples_per_second": 12.085,
      "eval_steps_per_second": 0.39,
      "step": 280
    },
    {
      "epoch": 1.5698324022346368,
      "grad_norm": 11.837750083976902,
      "learning_rate": 4.895470383275262e-06,
      "logits/chosen": -3.5244596004486084,
      "logits/rejected": -3.477862596511841,
      "logps/chosen": -20.005226135253906,
      "logps/rejected": -19.05292510986328,
      "loss": 0.303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2793457508087158,
      "rewards/margins": 1.2793457508087158,
      "rewards/rejected": 0.0,
      "step": 281
    },
    {
      "epoch": 1.5754189944134078,
      "grad_norm": 13.114096125445906,
      "learning_rate": 4.912891986062718e-06,
      "logits/chosen": -3.549536943435669,
      "logits/rejected": -3.5553932189941406,
      "logps/chosen": -11.152833938598633,
      "logps/rejected": -29.83534812927246,
      "loss": 0.3212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9834164381027222,
      "rewards/margins": 0.9834164381027222,
      "rewards/rejected": 0.0,
      "step": 282
    },
    {
      "epoch": 1.5810055865921788,
      "grad_norm": 10.423684220332746,
      "learning_rate": 4.930313588850174e-06,
      "logits/chosen": -3.5083072185516357,
      "logits/rejected": -3.39090895652771,
      "logps/chosen": -24.920913696289062,
      "logps/rejected": -25.61043930053711,
      "loss": 0.2571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4839280843734741,
      "rewards/margins": 1.4839280843734741,
      "rewards/rejected": 0.0,
      "step": 283
    },
    {
      "epoch": 1.5865921787709496,
      "grad_norm": 9.904945327904072,
      "learning_rate": 4.947735191637631e-06,
      "logits/chosen": -3.6139285564422607,
      "logits/rejected": -3.510378122329712,
      "logps/chosen": -11.68631362915039,
      "logps/rejected": -30.090097427368164,
      "loss": 0.2807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.144413709640503,
      "rewards/margins": 1.144413709640503,
      "rewards/rejected": 0.0,
      "step": 284
    },
    {
      "epoch": 1.5921787709497206,
      "grad_norm": 11.730555272146443,
      "learning_rate": 4.965156794425087e-06,
      "logits/chosen": -3.561675548553467,
      "logits/rejected": -3.50357985496521,
      "logps/chosen": -15.266813278198242,
      "logps/rejected": -24.020959854125977,
      "loss": 0.2941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0296943187713623,
      "rewards/margins": 1.0296943187713623,
      "rewards/rejected": 0.0,
      "step": 285
    },
    {
      "epoch": 1.5977653631284916,
      "grad_norm": 14.246647541520591,
      "learning_rate": 4.982578397212544e-06,
      "logits/chosen": -3.1360175609588623,
      "logits/rejected": -3.4105215072631836,
      "logps/chosen": -15.020916938781738,
      "logps/rejected": -35.978118896484375,
      "loss": 0.2805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5202000141143799,
      "rewards/margins": 1.5202000141143799,
      "rewards/rejected": 0.0,
      "step": 286
    },
    {
      "epoch": 1.6033519553072626,
      "grad_norm": 13.475611959105608,
      "learning_rate": 5e-06,
      "logits/chosen": -3.442643404006958,
      "logits/rejected": -3.2227795124053955,
      "logps/chosen": -11.743917465209961,
      "logps/rejected": -23.279390335083008,
      "loss": 0.3031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0162521600723267,
      "rewards/margins": 1.0162521600723267,
      "rewards/rejected": 0.0,
      "step": 287
    },
    {
      "epoch": 1.6089385474860336,
      "grad_norm": 13.850908978868818,
      "learning_rate": 4.999998142277388e-06,
      "logits/chosen": -3.3041787147521973,
      "logits/rejected": -3.3850367069244385,
      "logps/chosen": -15.760190963745117,
      "logps/rejected": -26.645065307617188,
      "loss": 0.3054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0894606113433838,
      "rewards/margins": 1.0894606113433838,
      "rewards/rejected": 0.0,
      "step": 288
    },
    {
      "epoch": 1.6145251396648046,
      "grad_norm": 15.004016837309557,
      "learning_rate": 4.99999256911231e-06,
      "logits/chosen": -3.4978020191192627,
      "logits/rejected": -3.3281736373901367,
      "logps/chosen": -17.68164825439453,
      "logps/rejected": -50.31319046020508,
      "loss": 0.3527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5064957141876221,
      "rewards/margins": 0.5064957141876221,
      "rewards/rejected": 0.0,
      "step": 289
    },
    {
      "epoch": 1.6201117318435754,
      "grad_norm": 11.972847047642931,
      "learning_rate": 4.99998328051305e-06,
      "logits/chosen": -3.2840428352355957,
      "logits/rejected": -3.075277090072632,
      "logps/chosen": -10.45104694366455,
      "logps/rejected": -43.799537658691406,
      "loss": 0.3099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8480896353721619,
      "rewards/margins": 0.8480896353721619,
      "rewards/rejected": 0.0,
      "step": 290
    },
    {
      "epoch": 1.6256983240223464,
      "grad_norm": 11.882270314147794,
      "learning_rate": 4.999970276493413e-06,
      "logits/chosen": -3.473766803741455,
      "logits/rejected": -3.2515597343444824,
      "logps/chosen": -19.28267478942871,
      "logps/rejected": -25.323917388916016,
      "loss": 0.2556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0844205617904663,
      "rewards/margins": 1.0844205617904663,
      "rewards/rejected": 0.0,
      "step": 291
    },
    {
      "epoch": 1.6312849162011172,
      "grad_norm": 15.932480217061926,
      "learning_rate": 4.999953557072725e-06,
      "logits/chosen": -3.7766754627227783,
      "logits/rejected": -3.628626585006714,
      "logps/chosen": -23.625944137573242,
      "logps/rejected": -35.20496368408203,
      "loss": 0.3439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6357585191726685,
      "rewards/margins": 1.6357585191726685,
      "rewards/rejected": 0.0,
      "step": 292
    },
    {
      "epoch": 1.6368715083798882,
      "grad_norm": 9.512853209615994,
      "learning_rate": 4.999933122275834e-06,
      "logits/chosen": -3.438551425933838,
      "logits/rejected": -3.2634828090667725,
      "logps/chosen": -9.443525314331055,
      "logps/rejected": -47.26200866699219,
      "loss": 0.2962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1862258911132812,
      "rewards/margins": 1.1862258911132812,
      "rewards/rejected": 0.0,
      "step": 293
    },
    {
      "epoch": 1.6424581005586592,
      "grad_norm": 12.100081690364695,
      "learning_rate": 4.999908972133109e-06,
      "logits/chosen": -3.395019769668579,
      "logits/rejected": -3.520336866378784,
      "logps/chosen": -5.480558395385742,
      "logps/rejected": -23.635469436645508,
      "loss": 0.3411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6740145087242126,
      "rewards/margins": 0.6740145087242126,
      "rewards/rejected": 0.0,
      "step": 294
    },
    {
      "epoch": 1.6480446927374302,
      "grad_norm": 12.278858430550196,
      "learning_rate": 4.999881106680442e-06,
      "logits/chosen": -3.5028188228607178,
      "logits/rejected": -3.634075403213501,
      "logps/chosen": -24.905838012695312,
      "logps/rejected": -20.41421890258789,
      "loss": 0.3235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3901655673980713,
      "rewards/margins": 1.3901655673980713,
      "rewards/rejected": 0.0,
      "step": 295
    },
    {
      "epoch": 1.6536312849162011,
      "grad_norm": 11.458774210499104,
      "learning_rate": 4.999849525959246e-06,
      "logits/chosen": -3.5890326499938965,
      "logits/rejected": -3.630063772201538,
      "logps/chosen": -16.270549774169922,
      "logps/rejected": -39.96139907836914,
      "loss": 0.2956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5788393020629883,
      "rewards/margins": 1.5788393020629883,
      "rewards/rejected": 0.0,
      "step": 296
    },
    {
      "epoch": 1.6592178770949721,
      "grad_norm": 15.161675282681013,
      "learning_rate": 4.9998142300164554e-06,
      "logits/chosen": -3.735058307647705,
      "logits/rejected": -3.760857105255127,
      "logps/chosen": -16.359577178955078,
      "logps/rejected": -25.271255493164062,
      "loss": 0.3302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8601205348968506,
      "rewards/margins": 0.8601205348968506,
      "rewards/rejected": 0.0,
      "step": 297
    },
    {
      "epoch": 1.6648044692737431,
      "grad_norm": 10.987756534212636,
      "learning_rate": 4.999775218904527e-06,
      "logits/chosen": -3.370944023132324,
      "logits/rejected": -3.456624746322632,
      "logps/chosen": -22.148635864257812,
      "logps/rejected": -43.08995819091797,
      "loss": 0.2755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8073457479476929,
      "rewards/margins": 1.8073457479476929,
      "rewards/rejected": 0.0,
      "step": 298
    },
    {
      "epoch": 1.670391061452514,
      "grad_norm": 17.252663895978994,
      "learning_rate": 4.9997324926814375e-06,
      "logits/chosen": -3.516693115234375,
      "logits/rejected": -3.6426684856414795,
      "logps/chosen": -49.149288177490234,
      "logps/rejected": -35.43327331542969,
      "loss": 0.3349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.06516170501709,
      "rewards/margins": 2.06516170501709,
      "rewards/rejected": 0.0,
      "step": 299
    },
    {
      "epoch": 1.675977653631285,
      "grad_norm": 16.0659757908231,
      "learning_rate": 4.999686051410685e-06,
      "logits/chosen": -3.5796830654144287,
      "logits/rejected": -3.5483884811401367,
      "logps/chosen": -12.121188163757324,
      "logps/rejected": -32.60401916503906,
      "loss": 0.3518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9835079908370972,
      "rewards/margins": 0.9835079908370972,
      "rewards/rejected": 0.0,
      "step": 300
    },
    {
      "epoch": 1.675977653631285,
      "eval_logits/chosen": -3.3894457817077637,
      "eval_logits/rejected": -3.439662456512451,
      "eval_logps/chosen": -22.98815155029297,
      "eval_logps/rejected": -35.93120193481445,
      "eval_loss": 0.4766485393047333,
      "eval_rewards/accuracies": 0.824999988079071,
      "eval_rewards/chosen": 0.7081891298294067,
      "eval_rewards/margins": 0.7081891298294067,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.564,
      "eval_samples_per_second": 12.126,
      "eval_steps_per_second": 0.391,
      "step": 300
    },
    {
      "epoch": 1.6815642458100557,
      "grad_norm": 14.822260115288357,
      "learning_rate": 4.999635895161291e-06,
      "logits/chosen": -3.5524628162384033,
      "logits/rejected": -3.5624430179595947,
      "logps/chosen": -14.109922409057617,
      "logps/rejected": -39.42070007324219,
      "loss": 0.3302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5496107935905457,
      "rewards/margins": 0.5496107935905457,
      "rewards/rejected": 0.0,
      "step": 301
    },
    {
      "epoch": 1.6871508379888267,
      "grad_norm": 13.775546281142542,
      "learning_rate": 4.9995820240077965e-06,
      "logits/chosen": -3.7093091011047363,
      "logits/rejected": -3.6113970279693604,
      "logps/chosen": -26.860158920288086,
      "logps/rejected": -44.514095306396484,
      "loss": 0.2878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3746150732040405,
      "rewards/margins": 1.3746150732040405,
      "rewards/rejected": 0.0,
      "step": 302
    },
    {
      "epoch": 1.6927374301675977,
      "grad_norm": 11.436067633445484,
      "learning_rate": 4.999524438030263e-06,
      "logits/chosen": -3.9714839458465576,
      "logits/rejected": -3.9220614433288574,
      "logps/chosen": -12.678996086120605,
      "logps/rejected": -33.54206848144531,
      "loss": 0.3142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7705928087234497,
      "rewards/margins": 0.7705928087234497,
      "rewards/rejected": 0.0,
      "step": 303
    },
    {
      "epoch": 1.6983240223463687,
      "grad_norm": 14.519093186397964,
      "learning_rate": 4.9994631373142735e-06,
      "logits/chosen": -3.3212006092071533,
      "logits/rejected": -3.1078639030456543,
      "logps/chosen": -35.14177703857422,
      "logps/rejected": -23.85761260986328,
      "loss": 0.2816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8307337164878845,
      "rewards/margins": 0.8307337164878845,
      "rewards/rejected": 0.0,
      "step": 304
    },
    {
      "epoch": 1.7039106145251397,
      "grad_norm": 13.912911036392611,
      "learning_rate": 4.999398121950931e-06,
      "logits/chosen": -3.619938373565674,
      "logits/rejected": -3.703584671020508,
      "logps/chosen": -20.0784854888916,
      "logps/rejected": -30.164087295532227,
      "loss": 0.2937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1850967407226562,
      "rewards/margins": 1.1850967407226562,
      "rewards/rejected": 0.0,
      "step": 305
    },
    {
      "epoch": 1.7094972067039107,
      "grad_norm": 15.948713221302564,
      "learning_rate": 4.999329392036862e-06,
      "logits/chosen": -3.258681297302246,
      "logits/rejected": -3.2982540130615234,
      "logps/chosen": -16.34724998474121,
      "logps/rejected": -21.408132553100586,
      "loss": 0.3206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6659841537475586,
      "rewards/margins": 1.6659841537475586,
      "rewards/rejected": 0.0,
      "step": 306
    },
    {
      "epoch": 1.7150837988826817,
      "grad_norm": 10.897392527161072,
      "learning_rate": 4.999256947674209e-06,
      "logits/chosen": -3.2319517135620117,
      "logits/rejected": -3.4667553901672363,
      "logps/chosen": -15.389171600341797,
      "logps/rejected": -18.9658145904541,
      "loss": 0.2994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4186292886734009,
      "rewards/margins": 1.4186292886734009,
      "rewards/rejected": 0.0,
      "step": 307
    },
    {
      "epoch": 1.7206703910614525,
      "grad_norm": 14.348464772090393,
      "learning_rate": 4.99918078897064e-06,
      "logits/chosen": -3.5301859378814697,
      "logits/rejected": -3.410062789916992,
      "logps/chosen": -16.71352767944336,
      "logps/rejected": -31.04961395263672,
      "loss": 0.4067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9533332586288452,
      "rewards/margins": 0.9533332586288452,
      "rewards/rejected": 0.0,
      "step": 308
    },
    {
      "epoch": 1.7262569832402235,
      "grad_norm": 14.61701145887242,
      "learning_rate": 4.9991009160393375e-06,
      "logits/chosen": -3.4313948154449463,
      "logits/rejected": -3.391411542892456,
      "logps/chosen": -25.133384704589844,
      "logps/rejected": -21.48000717163086,
      "loss": 0.3645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3086190223693848,
      "rewards/margins": 1.3086190223693848,
      "rewards/rejected": 0.0,
      "step": 309
    },
    {
      "epoch": 1.7318435754189943,
      "grad_norm": 13.252149310094342,
      "learning_rate": 4.999017328999009e-06,
      "logits/chosen": -3.6059181690216064,
      "logits/rejected": -3.1875791549682617,
      "logps/chosen": -15.433399200439453,
      "logps/rejected": -48.29927062988281,
      "loss": 0.312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6770381927490234,
      "rewards/margins": 0.6770381927490234,
      "rewards/rejected": 0.0,
      "step": 310
    },
    {
      "epoch": 1.7374301675977653,
      "grad_norm": 14.88881719849951,
      "learning_rate": 4.998930027973879e-06,
      "logits/chosen": -3.6587672233581543,
      "logits/rejected": -3.664543867111206,
      "logps/chosen": -6.532458305358887,
      "logps/rejected": -37.94033432006836,
      "loss": 0.3755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6313368678092957,
      "rewards/margins": 0.6313368678092957,
      "rewards/rejected": 0.0,
      "step": 311
    },
    {
      "epoch": 1.7430167597765363,
      "grad_norm": 14.193899203401795,
      "learning_rate": 4.998839013093692e-06,
      "logits/chosen": -3.6035656929016113,
      "logits/rejected": -3.6511073112487793,
      "logps/chosen": -11.155267715454102,
      "logps/rejected": -36.070098876953125,
      "loss": 0.3405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.94596266746521,
      "rewards/margins": 0.94596266746521,
      "rewards/rejected": 0.0,
      "step": 312
    },
    {
      "epoch": 1.7486033519553073,
      "grad_norm": 15.14591477503134,
      "learning_rate": 4.998744284493713e-06,
      "logits/chosen": -3.5023045539855957,
      "logits/rejected": -3.56253719329834,
      "logps/chosen": -12.477398872375488,
      "logps/rejected": -35.304603576660156,
      "loss": 0.3688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7695649266242981,
      "rewards/margins": 0.7695649266242981,
      "rewards/rejected": 0.0,
      "step": 313
    },
    {
      "epoch": 1.7541899441340782,
      "grad_norm": 19.96989207350826,
      "learning_rate": 4.998645842314725e-06,
      "logits/chosen": -3.3509910106658936,
      "logits/rejected": -3.4612343311309814,
      "logps/chosen": -19.143600463867188,
      "logps/rejected": -23.390043258666992,
      "loss": 0.3203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3178532123565674,
      "rewards/margins": 1.3178532123565674,
      "rewards/rejected": 0.0,
      "step": 314
    },
    {
      "epoch": 1.7597765363128492,
      "grad_norm": 9.61324215499714,
      "learning_rate": 4.998543686703031e-06,
      "logits/chosen": -3.2884953022003174,
      "logits/rejected": -3.1774449348449707,
      "logps/chosen": -26.337665557861328,
      "logps/rejected": -31.024473190307617,
      "loss": 0.2813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0511679649353027,
      "rewards/margins": 1.0511679649353027,
      "rewards/rejected": 0.0,
      "step": 315
    },
    {
      "epoch": 1.7653631284916202,
      "grad_norm": 12.994576047718118,
      "learning_rate": 4.998437817810452e-06,
      "logits/chosen": -3.3661205768585205,
      "logits/rejected": -3.200685977935791,
      "logps/chosen": -19.763824462890625,
      "logps/rejected": -60.52202606201172,
      "loss": 0.3139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5586993098258972,
      "rewards/margins": 0.5586993098258972,
      "rewards/rejected": 0.0,
      "step": 316
    },
    {
      "epoch": 1.770949720670391,
      "grad_norm": 20.496776733984852,
      "learning_rate": 4.998328235794329e-06,
      "logits/chosen": -3.098668098449707,
      "logits/rejected": -3.1624746322631836,
      "logps/chosen": -21.219249725341797,
      "logps/rejected": -39.30792999267578,
      "loss": 0.3687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0347338914871216,
      "rewards/margins": 1.0347338914871216,
      "rewards/rejected": 0.0,
      "step": 317
    },
    {
      "epoch": 1.776536312849162,
      "grad_norm": 15.388957822571248,
      "learning_rate": 4.998214940817518e-06,
      "logits/chosen": -3.6507813930511475,
      "logits/rejected": -3.3265881538391113,
      "logps/chosen": -14.744409561157227,
      "logps/rejected": -53.850929260253906,
      "loss": 0.388,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5186793804168701,
      "rewards/margins": 0.5186793804168701,
      "rewards/rejected": 0.0,
      "step": 318
    },
    {
      "epoch": 1.7821229050279328,
      "grad_norm": 18.086600717769514,
      "learning_rate": 4.998097933048399e-06,
      "logits/chosen": -3.4258530139923096,
      "logits/rejected": -3.270634651184082,
      "logps/chosen": -17.92505645751953,
      "logps/rejected": -24.407217025756836,
      "loss": 0.367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3376582860946655,
      "rewards/margins": 1.3376582860946655,
      "rewards/rejected": 0.0,
      "step": 319
    },
    {
      "epoch": 1.7877094972067038,
      "grad_norm": 20.66447712575382,
      "learning_rate": 4.997977212660864e-06,
      "logits/chosen": -3.4117252826690674,
      "logits/rejected": -3.3323099613189697,
      "logps/chosen": -53.030208587646484,
      "logps/rejected": -40.295318603515625,
      "loss": 0.3942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5215930938720703,
      "rewards/margins": 2.5215930938720703,
      "rewards/rejected": 0.0,
      "step": 320
    },
    {
      "epoch": 1.7877094972067038,
      "eval_logits/chosen": -3.30950665473938,
      "eval_logits/rejected": -3.3252415657043457,
      "eval_logps/chosen": -23.134510040283203,
      "eval_logps/rejected": -34.79408264160156,
      "eval_loss": 0.49336764216423035,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": 0.6935532689094543,
      "eval_rewards/margins": 0.6935532689094543,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5532,
      "eval_samples_per_second": 12.132,
      "eval_steps_per_second": 0.391,
      "step": 320
    },
    {
      "epoch": 1.7932960893854748,
      "grad_norm": 14.36778671702645,
      "learning_rate": 4.997852779834325e-06,
      "logits/chosen": -3.480353355407715,
      "logits/rejected": -3.370297908782959,
      "logps/chosen": -31.88125228881836,
      "logps/rejected": -35.385711669921875,
      "loss": 0.3464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8841060400009155,
      "rewards/margins": 1.8841060400009155,
      "rewards/rejected": 0.0,
      "step": 321
    },
    {
      "epoch": 1.7988826815642458,
      "grad_norm": 14.877830756082828,
      "learning_rate": 4.997724634753713e-06,
      "logits/chosen": -3.5853922367095947,
      "logits/rejected": -3.46443510055542,
      "logps/chosen": -12.640785217285156,
      "logps/rejected": -38.16013717651367,
      "loss": 0.336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41125208139419556,
      "rewards/margins": 0.41125208139419556,
      "rewards/rejected": 0.0,
      "step": 322
    },
    {
      "epoch": 1.8044692737430168,
      "grad_norm": 14.021258767418043,
      "learning_rate": 4.997592777609472e-06,
      "logits/chosen": -3.3436641693115234,
      "logits/rejected": -3.411224365234375,
      "logps/chosen": -11.191061973571777,
      "logps/rejected": -29.387245178222656,
      "loss": 0.3823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.873626708984375,
      "rewards/margins": 0.873626708984375,
      "rewards/rejected": 0.0,
      "step": 323
    },
    {
      "epoch": 1.8100558659217878,
      "grad_norm": 13.970659236273097,
      "learning_rate": 4.997457208597567e-06,
      "logits/chosen": -3.2698748111724854,
      "logits/rejected": -3.1076247692108154,
      "logps/chosen": -7.707671165466309,
      "logps/rejected": -26.257455825805664,
      "loss": 0.3896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9408595561981201,
      "rewards/margins": 0.9408595561981201,
      "rewards/rejected": 0.0,
      "step": 324
    },
    {
      "epoch": 1.8156424581005588,
      "grad_norm": 14.705800946714467,
      "learning_rate": 4.997317927919479e-06,
      "logits/chosen": -3.735426664352417,
      "logits/rejected": -3.777461290359497,
      "logps/chosen": -19.857425689697266,
      "logps/rejected": -26.669639587402344,
      "loss": 0.3812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7880446910858154,
      "rewards/margins": 0.7880446910858154,
      "rewards/rejected": 0.0,
      "step": 325
    },
    {
      "epoch": 1.8212290502793296,
      "grad_norm": 9.57129847443544,
      "learning_rate": 4.9971749357822e-06,
      "logits/chosen": -3.5255823135375977,
      "logits/rejected": -3.386233329772949,
      "logps/chosen": -21.27496910095215,
      "logps/rejected": -35.06666946411133,
      "loss": 0.2768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.643841028213501,
      "rewards/margins": 1.643841028213501,
      "rewards/rejected": 0.0,
      "step": 326
    },
    {
      "epoch": 1.8268156424581006,
      "grad_norm": 14.940941311848736,
      "learning_rate": 4.997028232398244e-06,
      "logits/chosen": -2.7154922485351562,
      "logits/rejected": -2.7360289096832275,
      "logps/chosen": -41.853424072265625,
      "logps/rejected": -49.57958221435547,
      "loss": 0.3892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9426289796829224,
      "rewards/margins": 0.9426289796829224,
      "rewards/rejected": 0.0,
      "step": 327
    },
    {
      "epoch": 1.8324022346368714,
      "grad_norm": 10.72376790702105,
      "learning_rate": 4.99687781798564e-06,
      "logits/chosen": -3.560006856918335,
      "logits/rejected": -3.626863956451416,
      "logps/chosen": -20.073007583618164,
      "logps/rejected": -31.907115936279297,
      "loss": 0.3296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.125549554824829,
      "rewards/margins": 1.125549554824829,
      "rewards/rejected": 0.0,
      "step": 328
    },
    {
      "epoch": 1.8379888268156424,
      "grad_norm": 15.529958693147895,
      "learning_rate": 4.996723692767927e-06,
      "logits/chosen": -2.710724353790283,
      "logits/rejected": -2.7201733589172363,
      "logps/chosen": -23.987586975097656,
      "logps/rejected": -28.297626495361328,
      "loss": 0.391,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2644262313842773,
      "rewards/margins": 1.2644262313842773,
      "rewards/rejected": 0.0,
      "step": 329
    },
    {
      "epoch": 1.8435754189944134,
      "grad_norm": 18.91782402728586,
      "learning_rate": 4.9965658569741645e-06,
      "logits/chosen": -3.4068362712860107,
      "logits/rejected": -3.2787342071533203,
      "logps/chosen": -28.919971466064453,
      "logps/rejected": -31.637237548828125,
      "loss": 0.3964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7716529369354248,
      "rewards/margins": 1.7716529369354248,
      "rewards/rejected": 0.0,
      "step": 330
    },
    {
      "epoch": 1.8491620111731844,
      "grad_norm": 13.429337873930361,
      "learning_rate": 4.996404310838925e-06,
      "logits/chosen": -3.387408971786499,
      "logits/rejected": -3.35516619682312,
      "logps/chosen": -16.642911911010742,
      "logps/rejected": -40.36919403076172,
      "loss": 0.2981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4662256240844727,
      "rewards/margins": 1.4662256240844727,
      "rewards/rejected": 0.0,
      "step": 331
    },
    {
      "epoch": 1.8547486033519553,
      "grad_norm": 14.595481043952365,
      "learning_rate": 4.996239054602294e-06,
      "logits/chosen": -3.495936393737793,
      "logits/rejected": -3.7182323932647705,
      "logps/chosen": -15.437213897705078,
      "logps/rejected": -34.75167465209961,
      "loss": 0.3307,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.004394769668579,
      "rewards/margins": 1.004394769668579,
      "rewards/rejected": 0.0,
      "step": 332
    },
    {
      "epoch": 1.8603351955307263,
      "grad_norm": 17.817612094072796,
      "learning_rate": 4.9960700885098726e-06,
      "logits/chosen": -3.4978039264678955,
      "logits/rejected": -3.431283712387085,
      "logps/chosen": -14.202083587646484,
      "logps/rejected": -29.757308959960938,
      "loss": 0.3677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5254696607589722,
      "rewards/margins": 0.5254696607589722,
      "rewards/rejected": 0.0,
      "step": 333
    },
    {
      "epoch": 1.8659217877094973,
      "grad_norm": 12.000393583425339,
      "learning_rate": 4.995897412812773e-06,
      "logits/chosen": -3.3434362411499023,
      "logits/rejected": -3.3920764923095703,
      "logps/chosen": -13.252524375915527,
      "logps/rejected": -62.30060958862305,
      "loss": 0.36,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11595818400382996,
      "rewards/margins": -0.11595818400382996,
      "rewards/rejected": 0.0,
      "step": 334
    },
    {
      "epoch": 1.8715083798882681,
      "grad_norm": 13.865157267678176,
      "learning_rate": 4.995721027767624e-06,
      "logits/chosen": -3.8293161392211914,
      "logits/rejected": -3.4834883213043213,
      "logps/chosen": -17.968544006347656,
      "logps/rejected": -21.867589950561523,
      "loss": 0.3294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0035094022750854,
      "rewards/margins": 1.0035094022750854,
      "rewards/rejected": 0.0,
      "step": 335
    },
    {
      "epoch": 1.8770949720670391,
      "grad_norm": 13.772086207266817,
      "learning_rate": 4.995540933636564e-06,
      "logits/chosen": -3.4806265830993652,
      "logits/rejected": -3.5733625888824463,
      "logps/chosen": -20.447223663330078,
      "logps/rejected": -39.284210205078125,
      "loss": 0.3043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2055190801620483,
      "rewards/margins": 1.2055190801620483,
      "rewards/rejected": 0.0,
      "step": 336
    },
    {
      "epoch": 1.88268156424581,
      "grad_norm": 10.804877912853888,
      "learning_rate": 4.995357130687245e-06,
      "logits/chosen": -3.2895472049713135,
      "logits/rejected": -3.4035398960113525,
      "logps/chosen": -7.4063897132873535,
      "logps/rejected": -37.09493637084961,
      "loss": 0.3013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0431592464447021,
      "rewards/margins": 1.0431592464447021,
      "rewards/rejected": 0.0,
      "step": 337
    },
    {
      "epoch": 1.888268156424581,
      "grad_norm": 12.026186908916019,
      "learning_rate": 4.9951696191928305e-06,
      "logits/chosen": -3.4776182174682617,
      "logits/rejected": -3.308232545852661,
      "logps/chosen": -43.76352310180664,
      "logps/rejected": -37.68260192871094,
      "loss": 0.2747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8245394229888916,
      "rewards/margins": 1.8245394229888916,
      "rewards/rejected": 0.0,
      "step": 338
    },
    {
      "epoch": 1.893854748603352,
      "grad_norm": 17.916115456513282,
      "learning_rate": 4.994978399431997e-06,
      "logits/chosen": -3.4736733436584473,
      "logits/rejected": -3.4598498344421387,
      "logps/chosen": -19.13573455810547,
      "logps/rejected": -32.11339569091797,
      "loss": 0.3507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0839576721191406,
      "rewards/margins": 1.0839576721191406,
      "rewards/rejected": 0.0,
      "step": 339
    },
    {
      "epoch": 1.899441340782123,
      "grad_norm": 15.645546454037563,
      "learning_rate": 4.99478347168893e-06,
      "logits/chosen": -3.3215558528900146,
      "logits/rejected": -3.4204370975494385,
      "logps/chosen": -43.67162322998047,
      "logps/rejected": -26.428451538085938,
      "loss": 0.3671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7951416969299316,
      "rewards/margins": 1.7951416969299316,
      "rewards/rejected": 0.0,
      "step": 340
    },
    {
      "epoch": 1.899441340782123,
      "eval_logits/chosen": -3.3694515228271484,
      "eval_logits/rejected": -3.4182496070861816,
      "eval_logps/chosen": -23.025127410888672,
      "eval_logps/rejected": -36.128814697265625,
      "eval_loss": 0.5206051468849182,
      "eval_rewards/accuracies": 0.8500000238418579,
      "eval_rewards/chosen": 0.7044914960861206,
      "eval_rewards/margins": 0.7044914960861206,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5067,
      "eval_samples_per_second": 12.154,
      "eval_steps_per_second": 0.392,
      "step": 340
    },
    {
      "epoch": 1.905027932960894,
      "grad_norm": 12.497506592388476,
      "learning_rate": 4.994584836253328e-06,
      "logits/chosen": -3.596332311630249,
      "logits/rejected": -3.528698444366455,
      "logps/chosen": -18.721481323242188,
      "logps/rejected": -39.04826354980469,
      "loss": 0.3104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0283432006835938,
      "rewards/margins": 1.0283432006835938,
      "rewards/rejected": 0.0,
      "step": 341
    },
    {
      "epoch": 1.910614525139665,
      "grad_norm": 21.974652416592644,
      "learning_rate": 4.994382493420398e-06,
      "logits/chosen": -3.5680558681488037,
      "logits/rejected": -3.470879554748535,
      "logps/chosen": -16.953296661376953,
      "logps/rejected": -48.06925582885742,
      "loss": 0.4133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2990820407867432,
      "rewards/margins": 1.2990820407867432,
      "rewards/rejected": 0.0,
      "step": 342
    },
    {
      "epoch": 1.916201117318436,
      "grad_norm": 22.72623743838928,
      "learning_rate": 4.994176443490857e-06,
      "logits/chosen": -3.495945930480957,
      "logits/rejected": -3.3986639976501465,
      "logps/chosen": -14.993463516235352,
      "logps/rejected": -33.34233856201172,
      "loss": 0.4451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.460553914308548,
      "rewards/margins": 0.460553914308548,
      "rewards/rejected": 0.0,
      "step": 343
    },
    {
      "epoch": 1.9217877094972067,
      "grad_norm": 18.73312099701,
      "learning_rate": 4.993966686770933e-06,
      "logits/chosen": -3.6852200031280518,
      "logits/rejected": -3.4666152000427246,
      "logps/chosen": -51.89307403564453,
      "logps/rejected": -35.461151123046875,
      "loss": 0.3326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6873944401741028,
      "rewards/margins": 0.6873944401741028,
      "rewards/rejected": 0.0,
      "step": 344
    },
    {
      "epoch": 1.9273743016759777,
      "grad_norm": 12.906065736862885,
      "learning_rate": 4.993753223572361e-06,
      "logits/chosen": -3.3969438076019287,
      "logits/rejected": -3.4193525314331055,
      "logps/chosen": -14.411123275756836,
      "logps/rejected": -23.936962127685547,
      "loss": 0.3376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3854618966579437,
      "rewards/margins": 0.3854618966579437,
      "rewards/rejected": 0.0,
      "step": 345
    },
    {
      "epoch": 1.9329608938547485,
      "grad_norm": 13.561614882877821,
      "learning_rate": 4.993536054212386e-06,
      "logits/chosen": -3.667783737182617,
      "logits/rejected": -3.1447315216064453,
      "logps/chosen": -15.251993179321289,
      "logps/rejected": -52.242347717285156,
      "loss": 0.353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.767013430595398,
      "rewards/margins": 0.767013430595398,
      "rewards/rejected": 0.0,
      "step": 346
    },
    {
      "epoch": 1.9385474860335195,
      "grad_norm": 16.21265317710707,
      "learning_rate": 4.99331517901376e-06,
      "logits/chosen": -3.4292824268341064,
      "logits/rejected": -3.295081615447998,
      "logps/chosen": -15.493255615234375,
      "logps/rejected": -34.9805793762207,
      "loss": 0.3445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1420081853866577,
      "rewards/margins": 1.1420081853866577,
      "rewards/rejected": 0.0,
      "step": 347
    },
    {
      "epoch": 1.9441340782122905,
      "grad_norm": 17.907497634523672,
      "learning_rate": 4.993090598304743e-06,
      "logits/chosen": -3.250702142715454,
      "logits/rejected": -3.3699018955230713,
      "logps/chosen": -31.262832641601562,
      "logps/rejected": -36.74143981933594,
      "loss": 0.3714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.235573172569275,
      "rewards/margins": 1.235573172569275,
      "rewards/rejected": 0.0,
      "step": 348
    },
    {
      "epoch": 1.9497206703910615,
      "grad_norm": 16.203444108948897,
      "learning_rate": 4.9928623124191015e-06,
      "logits/chosen": -3.3870089054107666,
      "logits/rejected": -3.450190782546997,
      "logps/chosen": -10.818788528442383,
      "logps/rejected": -30.897445678710938,
      "loss": 0.3971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5637869238853455,
      "rewards/margins": 0.5637869238853455,
      "rewards/rejected": 0.0,
      "step": 349
    },
    {
      "epoch": 1.9553072625698324,
      "grad_norm": 16.101350329069152,
      "learning_rate": 4.992630321696108e-06,
      "logits/chosen": -3.5528903007507324,
      "logits/rejected": -3.571624994277954,
      "logps/chosen": -28.142803192138672,
      "logps/rejected": -34.094566345214844,
      "loss": 0.3152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2241624593734741,
      "rewards/margins": 1.2241624593734741,
      "rewards/rejected": 0.0,
      "step": 350
    },
    {
      "epoch": 1.9608938547486034,
      "grad_norm": 14.207810968834684,
      "learning_rate": 4.992394626480544e-06,
      "logits/chosen": -3.0443267822265625,
      "logits/rejected": -2.894282341003418,
      "logps/chosen": -54.00589370727539,
      "logps/rejected": -40.57790756225586,
      "loss": 0.3412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5150294303894043,
      "rewards/margins": 1.5150294303894043,
      "rewards/rejected": 0.0,
      "step": 351
    },
    {
      "epoch": 1.9664804469273744,
      "grad_norm": 14.131304020293587,
      "learning_rate": 4.992155227122695e-06,
      "logits/chosen": -3.633695602416992,
      "logits/rejected": -3.517167091369629,
      "logps/chosen": -21.563661575317383,
      "logps/rejected": -30.619632720947266,
      "loss": 0.3309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3301699459552765,
      "rewards/margins": 0.3301699459552765,
      "rewards/rejected": 0.0,
      "step": 352
    },
    {
      "epoch": 1.9720670391061452,
      "grad_norm": 11.262680057914912,
      "learning_rate": 4.991912123978349e-06,
      "logits/chosen": -3.466942310333252,
      "logits/rejected": -3.342298984527588,
      "logps/chosen": -20.345712661743164,
      "logps/rejected": -71.0086898803711,
      "loss": 0.2943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3203604221343994,
      "rewards/margins": 1.3203604221343994,
      "rewards/rejected": 0.0,
      "step": 353
    },
    {
      "epoch": 1.9776536312849162,
      "grad_norm": 17.18753458655627,
      "learning_rate": 4.991665317408802e-06,
      "logits/chosen": -3.438047409057617,
      "logits/rejected": -3.3485469818115234,
      "logps/chosen": -31.720592498779297,
      "logps/rejected": -24.745643615722656,
      "loss": 0.3438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.320259928703308,
      "rewards/margins": 1.320259928703308,
      "rewards/rejected": 0.0,
      "step": 354
    },
    {
      "epoch": 1.983240223463687,
      "grad_norm": 14.03041812023355,
      "learning_rate": 4.991414807780852e-06,
      "logits/chosen": -3.6923153400421143,
      "logits/rejected": -3.7172601222991943,
      "logps/chosen": -14.719075202941895,
      "logps/rejected": -21.720918655395508,
      "loss": 0.3938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7664157152175903,
      "rewards/margins": 0.7664157152175903,
      "rewards/rejected": 0.0,
      "step": 355
    },
    {
      "epoch": 1.988826815642458,
      "grad_norm": 14.290437961408182,
      "learning_rate": 4.9911605954668e-06,
      "logits/chosen": -3.747605323791504,
      "logits/rejected": -3.7034707069396973,
      "logps/chosen": -16.45557975769043,
      "logps/rejected": -26.212390899658203,
      "loss": 0.3221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9023047089576721,
      "rewards/margins": 0.9023047089576721,
      "rewards/rejected": 0.0,
      "step": 356
    },
    {
      "epoch": 1.994413407821229,
      "grad_norm": 14.311086738226326,
      "learning_rate": 4.990902680844452e-06,
      "logits/chosen": -3.540126085281372,
      "logits/rejected": -3.1784915924072266,
      "logps/chosen": -25.121631622314453,
      "logps/rejected": -26.96269989013672,
      "loss": 0.3765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7375798225402832,
      "rewards/margins": 1.7375798225402832,
      "rewards/rejected": 0.0,
      "step": 357
    },
    {
      "epoch": 2.0,
      "grad_norm": 7.671926730245016,
      "learning_rate": 4.990641064297116e-06,
      "logits/chosen": -3.394439458847046,
      "logits/rejected": -3.2642557621002197,
      "logps/chosen": -16.163419723510742,
      "logps/rejected": -32.79155731201172,
      "loss": 0.2442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3764185905456543,
      "rewards/margins": 1.3764185905456543,
      "rewards/rejected": 0.0,
      "step": 358
    },
    {
      "epoch": 2.005586592178771,
      "grad_norm": 7.736165742086428,
      "learning_rate": 4.9903757462135984e-06,
      "logits/chosen": -2.95119309425354,
      "logits/rejected": -3.115893840789795,
      "logps/chosen": -19.889610290527344,
      "logps/rejected": -28.365379333496094,
      "loss": 0.2383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2685694694519043,
      "rewards/margins": 2.2685694694519043,
      "rewards/rejected": 0.0,
      "step": 359
    },
    {
      "epoch": 2.011173184357542,
      "grad_norm": 4.949730194001517,
      "learning_rate": 4.99010672698821e-06,
      "logits/chosen": -3.6400949954986572,
      "logits/rejected": -3.3829128742218018,
      "logps/chosen": -5.0058417320251465,
      "logps/rejected": -26.69578742980957,
      "loss": 0.1985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2562910318374634,
      "rewards/margins": 1.2562910318374634,
      "rewards/rejected": 0.0,
      "step": 360
    },
    {
      "epoch": 2.011173184357542,
      "eval_logits/chosen": -3.345062255859375,
      "eval_logits/rejected": -3.397439956665039,
      "eval_logps/chosen": -22.856285095214844,
      "eval_logps/rejected": -36.67879104614258,
      "eval_loss": 0.5171103477478027,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": 0.7213755249977112,
      "eval_rewards/margins": 0.7213755249977112,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5665,
      "eval_samples_per_second": 12.125,
      "eval_steps_per_second": 0.391,
      "step": 360
    },
    {
      "epoch": 2.016759776536313,
      "grad_norm": 9.507030336692578,
      "learning_rate": 4.989834007020763e-06,
      "logits/chosen": -3.609931707382202,
      "logits/rejected": -3.608849287033081,
      "logps/chosen": -11.501104354858398,
      "logps/rejected": -32.71945571899414,
      "loss": 0.2339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8313283920288086,
      "rewards/margins": 1.8313283920288086,
      "rewards/rejected": 0.0,
      "step": 361
    },
    {
      "epoch": 2.022346368715084,
      "grad_norm": 8.615466803861164,
      "learning_rate": 4.989557586716565e-06,
      "logits/chosen": -3.3424911499023438,
      "logits/rejected": -3.446420669555664,
      "logps/chosen": -23.39049530029297,
      "logps/rejected": -46.949188232421875,
      "loss": 0.236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7406585216522217,
      "rewards/margins": 2.7406585216522217,
      "rewards/rejected": 0.0,
      "step": 362
    },
    {
      "epoch": 2.0279329608938546,
      "grad_norm": 4.702123263787773,
      "learning_rate": 4.989277466486427e-06,
      "logits/chosen": -3.5967466831207275,
      "logits/rejected": -3.37713885307312,
      "logps/chosen": -20.169044494628906,
      "logps/rejected": -40.779457092285156,
      "loss": 0.1817,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0893311500549316,
      "rewards/margins": 2.0893311500549316,
      "rewards/rejected": 0.0,
      "step": 363
    },
    {
      "epoch": 2.0335195530726256,
      "grad_norm": 7.657859048849203,
      "learning_rate": 4.988993646746659e-06,
      "logits/chosen": -3.628753662109375,
      "logits/rejected": -3.7111990451812744,
      "logps/chosen": -18.812368392944336,
      "logps/rejected": -40.05157470703125,
      "loss": 0.1935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7823641300201416,
      "rewards/margins": 1.7823641300201416,
      "rewards/rejected": 0.0,
      "step": 364
    },
    {
      "epoch": 2.0391061452513966,
      "grad_norm": 8.43244290208756,
      "learning_rate": 4.9887061279190656e-06,
      "logits/chosen": -3.7191309928894043,
      "logits/rejected": -3.5845999717712402,
      "logps/chosen": -11.508506774902344,
      "logps/rejected": -38.850502014160156,
      "loss": 0.217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.308178424835205,
      "rewards/margins": 1.308178424835205,
      "rewards/rejected": 0.0,
      "step": 365
    },
    {
      "epoch": 2.0446927374301676,
      "grad_norm": 14.312321521846322,
      "learning_rate": 4.988414910430952e-06,
      "logits/chosen": -3.454660654067993,
      "logits/rejected": -3.2817726135253906,
      "logps/chosen": -12.423985481262207,
      "logps/rejected": -28.90095329284668,
      "loss": 0.2229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0065531730651855,
      "rewards/margins": 2.0065531730651855,
      "rewards/rejected": 0.0,
      "step": 366
    },
    {
      "epoch": 2.0502793296089385,
      "grad_norm": 9.745525837425294,
      "learning_rate": 4.9881199947151195e-06,
      "logits/chosen": -3.4120941162109375,
      "logits/rejected": -3.7092175483703613,
      "logps/chosen": -8.277473449707031,
      "logps/rejected": -31.6805477142334,
      "loss": 0.2123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2950546741485596,
      "rewards/margins": 1.2950546741485596,
      "rewards/rejected": 0.0,
      "step": 367
    },
    {
      "epoch": 2.0558659217877095,
      "grad_norm": 10.024269536513055,
      "learning_rate": 4.9878213812098646e-06,
      "logits/chosen": -3.553208589553833,
      "logits/rejected": -3.5948784351348877,
      "logps/chosen": -7.506690979003906,
      "logps/rejected": -65.11898803710938,
      "loss": 0.2466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.3200761079788208,
      "rewards/margins": 1.3200761079788208,
      "rewards/rejected": 0.0,
      "step": 368
    },
    {
      "epoch": 2.0614525139664805,
      "grad_norm": 13.423124736306795,
      "learning_rate": 4.987519070358979e-06,
      "logits/chosen": -3.807222604751587,
      "logits/rejected": -3.755756139755249,
      "logps/chosen": -4.988807201385498,
      "logps/rejected": -38.59445571899414,
      "loss": 0.2976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6037044525146484,
      "rewards/margins": 1.6037044525146484,
      "rewards/rejected": 0.0,
      "step": 369
    },
    {
      "epoch": 2.0670391061452515,
      "grad_norm": 9.787111931892932,
      "learning_rate": 4.987213062611754e-06,
      "logits/chosen": -3.7777392864227295,
      "logits/rejected": -3.5216798782348633,
      "logps/chosen": -5.750980854034424,
      "logps/rejected": -35.04076385498047,
      "loss": 0.2057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3980554342269897,
      "rewards/margins": 1.3980554342269897,
      "rewards/rejected": 0.0,
      "step": 370
    },
    {
      "epoch": 2.0726256983240225,
      "grad_norm": 11.530510326732038,
      "learning_rate": 4.986903358422969e-06,
      "logits/chosen": -3.4087440967559814,
      "logits/rejected": -3.52323317527771,
      "logps/chosen": -13.646749496459961,
      "logps/rejected": -50.54429244995117,
      "loss": 0.2053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2211568355560303,
      "rewards/margins": 2.2211568355560303,
      "rewards/rejected": 0.0,
      "step": 371
    },
    {
      "epoch": 2.078212290502793,
      "grad_norm": 7.46152081567692,
      "learning_rate": 4.9865899582528995e-06,
      "logits/chosen": -3.6993322372436523,
      "logits/rejected": -3.653618812561035,
      "logps/chosen": -48.68275451660156,
      "logps/rejected": -56.838401794433594,
      "loss": 0.167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4521231651306152,
      "rewards/margins": 3.4521231651306152,
      "rewards/rejected": 0.0,
      "step": 372
    },
    {
      "epoch": 2.083798882681564,
      "grad_norm": 7.6165625721124615,
      "learning_rate": 4.986272862567315e-06,
      "logits/chosen": -3.3805606365203857,
      "logits/rejected": -3.760756731033325,
      "logps/chosen": -9.737466812133789,
      "logps/rejected": -56.42272186279297,
      "loss": 0.2057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7444933652877808,
      "rewards/margins": 1.7444933652877808,
      "rewards/rejected": 0.0,
      "step": 373
    },
    {
      "epoch": 2.089385474860335,
      "grad_norm": 16.888541632666943,
      "learning_rate": 4.9859520718374755e-06,
      "logits/chosen": -3.1844379901885986,
      "logits/rejected": -3.4456865787506104,
      "logps/chosen": -35.880733489990234,
      "logps/rejected": -26.866573333740234,
      "loss": 0.2979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8470935821533203,
      "rewards/margins": 1.8470935821533203,
      "rewards/rejected": 0.0,
      "step": 374
    },
    {
      "epoch": 2.094972067039106,
      "grad_norm": 7.440824356399274,
      "learning_rate": 4.9856275865401335e-06,
      "logits/chosen": -3.616004228591919,
      "logits/rejected": -3.558302164077759,
      "logps/chosen": -10.929966926574707,
      "logps/rejected": -32.761905670166016,
      "loss": 0.1902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7534706592559814,
      "rewards/margins": 1.7534706592559814,
      "rewards/rejected": 0.0,
      "step": 375
    },
    {
      "epoch": 2.100558659217877,
      "grad_norm": 5.727995282103713,
      "learning_rate": 4.9852994071575325e-06,
      "logits/chosen": -3.4788522720336914,
      "logits/rejected": -3.2857964038848877,
      "logps/chosen": -16.220478057861328,
      "logps/rejected": -33.40386962890625,
      "loss": 0.1878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.200493097305298,
      "rewards/margins": 2.200493097305298,
      "rewards/rejected": 0.0,
      "step": 376
    },
    {
      "epoch": 2.106145251396648,
      "grad_norm": 6.224974995103918,
      "learning_rate": 4.984967534177404e-06,
      "logits/chosen": -3.7683591842651367,
      "logits/rejected": -3.754582405090332,
      "logps/chosen": -8.510100364685059,
      "logps/rejected": -38.595863342285156,
      "loss": 0.1472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6437320709228516,
      "rewards/margins": 1.6437320709228516,
      "rewards/rejected": 0.0,
      "step": 377
    },
    {
      "epoch": 2.111731843575419,
      "grad_norm": 10.190073759412073,
      "learning_rate": 4.984631968092972e-06,
      "logits/chosen": -3.531526803970337,
      "logits/rejected": -3.712568998336792,
      "logps/chosen": -28.678802490234375,
      "logps/rejected": -48.99562454223633,
      "loss": 0.257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7308201789855957,
      "rewards/margins": 2.7308201789855957,
      "rewards/rejected": 0.0,
      "step": 378
    },
    {
      "epoch": 2.11731843575419,
      "grad_norm": 7.721085085144927,
      "learning_rate": 4.984292709402946e-06,
      "logits/chosen": -3.5616002082824707,
      "logits/rejected": -3.439574718475342,
      "logps/chosen": -12.080522537231445,
      "logps/rejected": -34.05799865722656,
      "loss": 0.1807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5355209112167358,
      "rewards/margins": 1.5355209112167358,
      "rewards/rejected": 0.0,
      "step": 379
    },
    {
      "epoch": 2.122905027932961,
      "grad_norm": 8.157832826463533,
      "learning_rate": 4.983949758611526e-06,
      "logits/chosen": -2.8917479515075684,
      "logits/rejected": -2.8648452758789062,
      "logps/chosen": -12.098701477050781,
      "logps/rejected": -44.413597106933594,
      "loss": 0.2023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3973195552825928,
      "rewards/margins": 1.3973195552825928,
      "rewards/rejected": 0.0,
      "step": 380
    },
    {
      "epoch": 2.122905027932961,
      "eval_logits/chosen": -3.4510910511016846,
      "eval_logits/rejected": -3.5360169410705566,
      "eval_logps/chosen": -24.04014015197754,
      "eval_logps/rejected": -39.39387893676758,
      "eval_loss": 0.6485124230384827,
      "eval_rewards/accuracies": 0.800000011920929,
      "eval_rewards/chosen": 0.6029903292655945,
      "eval_rewards/margins": 0.6029903292655945,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5346,
      "eval_samples_per_second": 12.14,
      "eval_steps_per_second": 0.392,
      "step": 380
    },
    {
      "epoch": 2.1284916201117317,
      "grad_norm": 12.926809817809152,
      "learning_rate": 4.983603116228397e-06,
      "logits/chosen": -3.401309013366699,
      "logits/rejected": -3.5806772708892822,
      "logps/chosen": -15.675642013549805,
      "logps/rejected": -30.914247512817383,
      "loss": 0.2212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.853121757507324,
      "rewards/margins": 2.853121757507324,
      "rewards/rejected": 0.0,
      "step": 381
    },
    {
      "epoch": 2.1340782122905027,
      "grad_norm": 5.4326332104070785,
      "learning_rate": 4.983252782768732e-06,
      "logits/chosen": -3.599668025970459,
      "logits/rejected": -3.7165205478668213,
      "logps/chosen": -21.272512435913086,
      "logps/rejected": -53.48712921142578,
      "loss": 0.1509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2882392406463623,
      "rewards/margins": 2.2882392406463623,
      "rewards/rejected": 0.0,
      "step": 382
    },
    {
      "epoch": 2.1396648044692737,
      "grad_norm": 6.394599007528729,
      "learning_rate": 4.982898758753189e-06,
      "logits/chosen": -3.6522810459136963,
      "logits/rejected": -3.5281193256378174,
      "logps/chosen": -7.145430564880371,
      "logps/rejected": -44.66184997558594,
      "loss": 0.1836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2574849128723145,
      "rewards/margins": 2.2574849128723145,
      "rewards/rejected": 0.0,
      "step": 383
    },
    {
      "epoch": 2.1452513966480447,
      "grad_norm": 8.332308909024558,
      "learning_rate": 4.982541044707909e-06,
      "logits/chosen": -3.7109761238098145,
      "logits/rejected": -3.5894107818603516,
      "logps/chosen": -9.778807640075684,
      "logps/rejected": -52.05250549316406,
      "loss": 0.1818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.300947666168213,
      "rewards/margins": 2.300947666168213,
      "rewards/rejected": 0.0,
      "step": 384
    },
    {
      "epoch": 2.1508379888268156,
      "grad_norm": 7.888471551807452,
      "learning_rate": 4.982179641164521e-06,
      "logits/chosen": -3.8357346057891846,
      "logits/rejected": -3.8061907291412354,
      "logps/chosen": -6.41518497467041,
      "logps/rejected": -36.57960510253906,
      "loss": 0.2298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.845055341720581,
      "rewards/margins": 0.845055341720581,
      "rewards/rejected": 0.0,
      "step": 385
    },
    {
      "epoch": 2.1564245810055866,
      "grad_norm": 12.148938666419296,
      "learning_rate": 4.981814548660135e-06,
      "logits/chosen": -3.536987543106079,
      "logits/rejected": -3.6213369369506836,
      "logps/chosen": -17.743207931518555,
      "logps/rejected": -55.41523361206055,
      "loss": 0.2336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1810901165008545,
      "rewards/margins": 2.1810901165008545,
      "rewards/rejected": 0.0,
      "step": 386
    },
    {
      "epoch": 2.1620111731843576,
      "grad_norm": 20.84667450814865,
      "learning_rate": 4.981445767737342e-06,
      "logits/chosen": -3.8179726600646973,
      "logits/rejected": -3.937417984008789,
      "logps/chosen": -14.141307830810547,
      "logps/rejected": -42.041847229003906,
      "loss": 0.3005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3580573797225952,
      "rewards/margins": 0.3580573797225952,
      "rewards/rejected": 0.0,
      "step": 387
    },
    {
      "epoch": 2.1675977653631286,
      "grad_norm": 5.721917400132948,
      "learning_rate": 4.981073298944217e-06,
      "logits/chosen": -3.7798266410827637,
      "logits/rejected": -3.82353138923645,
      "logps/chosen": -25.213279724121094,
      "logps/rejected": -39.17259979248047,
      "loss": 0.161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9487380981445312,
      "rewards/margins": 2.9487380981445312,
      "rewards/rejected": 0.0,
      "step": 388
    },
    {
      "epoch": 2.1731843575418996,
      "grad_norm": 9.534440073849643,
      "learning_rate": 4.980697142834315e-06,
      "logits/chosen": -3.5279486179351807,
      "logits/rejected": -3.3731577396392822,
      "logps/chosen": -12.353391647338867,
      "logps/rejected": -35.85765075683594,
      "loss": 0.2135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7470382452011108,
      "rewards/margins": 1.7470382452011108,
      "rewards/rejected": 0.0,
      "step": 389
    },
    {
      "epoch": 2.17877094972067,
      "grad_norm": 8.71792261728629,
      "learning_rate": 4.98031729996667e-06,
      "logits/chosen": -3.746387243270874,
      "logits/rejected": -3.853478193283081,
      "logps/chosen": -20.033931732177734,
      "logps/rejected": -30.82926368713379,
      "loss": 0.197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.079129219055176,
      "rewards/margins": 2.079129219055176,
      "rewards/rejected": 0.0,
      "step": 390
    },
    {
      "epoch": 2.184357541899441,
      "grad_norm": 5.627969796056245,
      "learning_rate": 4.979933770905798e-06,
      "logits/chosen": -3.7321577072143555,
      "logits/rejected": -3.7556521892547607,
      "logps/chosen": -14.45535659790039,
      "logps/rejected": -44.04999542236328,
      "loss": 0.1525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2587504386901855,
      "rewards/margins": 2.2587504386901855,
      "rewards/rejected": 0.0,
      "step": 391
    },
    {
      "epoch": 2.189944134078212,
      "grad_norm": 14.222485550235392,
      "learning_rate": 4.979546556221689e-06,
      "logits/chosen": -3.750295639038086,
      "logits/rejected": -3.5511999130249023,
      "logps/chosen": -15.29178524017334,
      "logps/rejected": -29.6248722076416,
      "loss": 0.3946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.936311721801758,
      "rewards/margins": 2.936311721801758,
      "rewards/rejected": 0.0,
      "step": 392
    },
    {
      "epoch": 2.195530726256983,
      "grad_norm": 10.577587345860108,
      "learning_rate": 4.979155656489816e-06,
      "logits/chosen": -3.862234592437744,
      "logits/rejected": -3.9011456966400146,
      "logps/chosen": -14.507349014282227,
      "logps/rejected": -50.72199630737305,
      "loss": 0.2383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8096565008163452,
      "rewards/margins": 1.8096565008163452,
      "rewards/rejected": 0.0,
      "step": 393
    },
    {
      "epoch": 2.201117318435754,
      "grad_norm": 12.45968686888698,
      "learning_rate": 4.978761072291123e-06,
      "logits/chosen": -3.6359851360321045,
      "logits/rejected": -3.886000394821167,
      "logps/chosen": -30.291730880737305,
      "logps/rejected": -85.19866943359375,
      "loss": 0.221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.118631362915039,
      "rewards/margins": 3.118631362915039,
      "rewards/rejected": 0.0,
      "step": 394
    },
    {
      "epoch": 2.206703910614525,
      "grad_norm": 10.028707023512952,
      "learning_rate": 4.978362804212033e-06,
      "logits/chosen": -3.4917001724243164,
      "logits/rejected": -3.4894049167633057,
      "logps/chosen": -11.436698913574219,
      "logps/rejected": -23.996070861816406,
      "loss": 0.2578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2970213890075684,
      "rewards/margins": 2.2970213890075684,
      "rewards/rejected": 0.0,
      "step": 395
    },
    {
      "epoch": 2.212290502793296,
      "grad_norm": 13.896455391776701,
      "learning_rate": 4.977960852844445e-06,
      "logits/chosen": -3.527330160140991,
      "logits/rejected": -3.5103859901428223,
      "logps/chosen": -35.23580551147461,
      "logps/rejected": -36.89857482910156,
      "loss": 0.2331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.258470058441162,
      "rewards/margins": 3.258470058441162,
      "rewards/rejected": 0.0,
      "step": 396
    },
    {
      "epoch": 2.217877094972067,
      "grad_norm": 7.62732680840098,
      "learning_rate": 4.977555218785728e-06,
      "logits/chosen": -3.607927083969116,
      "logits/rejected": -3.654386043548584,
      "logps/chosen": -16.212186813354492,
      "logps/rejected": -53.32461166381836,
      "loss": 0.2385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0884411334991455,
      "rewards/margins": 1.0884411334991455,
      "rewards/rejected": 0.0,
      "step": 397
    },
    {
      "epoch": 2.223463687150838,
      "grad_norm": 12.394625180872367,
      "learning_rate": 4.977145902638727e-06,
      "logits/chosen": -3.7416322231292725,
      "logits/rejected": -3.7018003463745117,
      "logps/chosen": -14.267093658447266,
      "logps/rejected": -40.313072204589844,
      "loss": 0.2917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.990336298942566,
      "rewards/margins": 1.990336298942566,
      "rewards/rejected": 0.0,
      "step": 398
    },
    {
      "epoch": 2.2290502793296088,
      "grad_norm": 7.420466262668125,
      "learning_rate": 4.976732905011761e-06,
      "logits/chosen": -3.7681636810302734,
      "logits/rejected": -3.7365427017211914,
      "logps/chosen": -8.992073059082031,
      "logps/rejected": -59.62089157104492,
      "loss": 0.2211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0773417949676514,
      "rewards/margins": 2.0773417949676514,
      "rewards/rejected": 0.0,
      "step": 399
    },
    {
      "epoch": 2.2346368715083798,
      "grad_norm": 9.233589299392456,
      "learning_rate": 4.976316226518614e-06,
      "logits/chosen": -3.3901925086975098,
      "logits/rejected": -3.499669313430786,
      "logps/chosen": -6.8948974609375,
      "logps/rejected": -40.24817657470703,
      "loss": 0.2189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7660081386566162,
      "rewards/margins": 1.7660081386566162,
      "rewards/rejected": 0.0,
      "step": 400
    },
    {
      "epoch": 2.2346368715083798,
      "eval_logits/chosen": -3.436840772628784,
      "eval_logits/rejected": -3.5077157020568848,
      "eval_logps/chosen": -24.858745574951172,
      "eval_logps/rejected": -37.954002380371094,
      "eval_loss": 0.6177390813827515,
      "eval_rewards/accuracies": 0.7749999761581421,
      "eval_rewards/chosen": 0.5211297273635864,
      "eval_rewards/margins": 0.5211297273635864,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5687,
      "eval_samples_per_second": 12.124,
      "eval_steps_per_second": 0.391,
      "step": 400
    },
    {
      "epoch": 2.2402234636871508,
      "grad_norm": 6.243561606747056,
      "learning_rate": 4.975895867778547e-06,
      "logits/chosen": -3.4033730030059814,
      "logits/rejected": -3.6369223594665527,
      "logps/chosen": -35.43971252441406,
      "logps/rejected": -48.03842544555664,
      "loss": 0.1794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.815727472305298,
      "rewards/margins": 2.815727472305298,
      "rewards/rejected": 0.0,
      "step": 401
    },
    {
      "epoch": 2.2458100558659218,
      "grad_norm": 11.829199198167112,
      "learning_rate": 4.975471829416288e-06,
      "logits/chosen": -3.561713933944702,
      "logits/rejected": -3.344815969467163,
      "logps/chosen": -6.803017616271973,
      "logps/rejected": -33.57386779785156,
      "loss": 0.3027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.824026346206665,
      "rewards/margins": 1.824026346206665,
      "rewards/rejected": 0.0,
      "step": 402
    },
    {
      "epoch": 2.2513966480446927,
      "grad_norm": 6.4557214476974485,
      "learning_rate": 4.975044112062034e-06,
      "logits/chosen": -3.8228676319122314,
      "logits/rejected": -3.6451609134674072,
      "logps/chosen": -6.916016101837158,
      "logps/rejected": -44.41805648803711,
      "loss": 0.1877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.470684289932251,
      "rewards/margins": 1.470684289932251,
      "rewards/rejected": 0.0,
      "step": 403
    },
    {
      "epoch": 2.2569832402234637,
      "grad_norm": 7.930798259433725,
      "learning_rate": 4.974612716351446e-06,
      "logits/chosen": -3.1935999393463135,
      "logits/rejected": -3.430670738220215,
      "logps/chosen": -24.053115844726562,
      "logps/rejected": -56.394317626953125,
      "loss": 0.2157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6914401054382324,
      "rewards/margins": 1.6914401054382324,
      "rewards/rejected": 0.0,
      "step": 404
    },
    {
      "epoch": 2.2625698324022347,
      "grad_norm": 12.717609074525784,
      "learning_rate": 4.974177642925659e-06,
      "logits/chosen": -3.8572421073913574,
      "logits/rejected": -3.6785976886749268,
      "logps/chosen": -12.757099151611328,
      "logps/rejected": -25.289871215820312,
      "loss": 0.2823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.067211151123047,
      "rewards/margins": 2.067211151123047,
      "rewards/rejected": 0.0,
      "step": 405
    },
    {
      "epoch": 2.2681564245810057,
      "grad_norm": 15.653090070676225,
      "learning_rate": 4.973738892431268e-06,
      "logits/chosen": -3.5904452800750732,
      "logits/rejected": -3.5950732231140137,
      "logps/chosen": -27.672115325927734,
      "logps/rejected": -45.62098693847656,
      "loss": 0.3044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9824707508087158,
      "rewards/margins": 1.9824707508087158,
      "rewards/rejected": 0.0,
      "step": 406
    },
    {
      "epoch": 2.2737430167597763,
      "grad_norm": 7.454205425287796,
      "learning_rate": 4.973296465520333e-06,
      "logits/chosen": -3.5127499103546143,
      "logits/rejected": -3.7231948375701904,
      "logps/chosen": -14.017836570739746,
      "logps/rejected": -28.810386657714844,
      "loss": 0.2261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.124819278717041,
      "rewards/margins": 2.124819278717041,
      "rewards/rejected": 0.0,
      "step": 407
    },
    {
      "epoch": 2.2793296089385473,
      "grad_norm": 8.413599620301857,
      "learning_rate": 4.97285036285038e-06,
      "logits/chosen": -3.586087226867676,
      "logits/rejected": -3.57080078125,
      "logps/chosen": -10.82162857055664,
      "logps/rejected": -38.39168167114258,
      "loss": 0.2252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.3672852516174316,
      "rewards/margins": 1.3672852516174316,
      "rewards/rejected": 0.0,
      "step": 408
    },
    {
      "epoch": 2.2849162011173183,
      "grad_norm": 7.219231286616838,
      "learning_rate": 4.972400585084398e-06,
      "logits/chosen": -3.729053497314453,
      "logits/rejected": -3.8185019493103027,
      "logps/chosen": -13.987716674804688,
      "logps/rejected": -45.3639030456543,
      "loss": 0.1963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.941501498222351,
      "rewards/margins": 1.941501498222351,
      "rewards/rejected": 0.0,
      "step": 409
    },
    {
      "epoch": 2.2905027932960893,
      "grad_norm": 10.073441675356646,
      "learning_rate": 4.971947132890836e-06,
      "logits/chosen": -2.9088027477264404,
      "logits/rejected": -2.9671642780303955,
      "logps/chosen": -25.704235076904297,
      "logps/rejected": -55.46349334716797,
      "loss": 0.2515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3396357297897339,
      "rewards/margins": 1.3396357297897339,
      "rewards/rejected": 0.0,
      "step": 410
    },
    {
      "epoch": 2.2960893854748603,
      "grad_norm": 12.398371236575066,
      "learning_rate": 4.971490006943604e-06,
      "logits/chosen": -3.1757729053497314,
      "logits/rejected": -3.282437324523926,
      "logps/chosen": -15.582036972045898,
      "logps/rejected": -62.56745147705078,
      "loss": 0.2935,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.326956033706665,
      "rewards/margins": 1.326956033706665,
      "rewards/rejected": 0.0,
      "step": 411
    },
    {
      "epoch": 2.3016759776536313,
      "grad_norm": 7.115882582182102,
      "learning_rate": 4.971029207922074e-06,
      "logits/chosen": -3.656519889831543,
      "logits/rejected": -3.4952328205108643,
      "logps/chosen": -27.151092529296875,
      "logps/rejected": -55.331138610839844,
      "loss": 0.1669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.885730743408203,
      "rewards/margins": 2.885730743408203,
      "rewards/rejected": 0.0,
      "step": 412
    },
    {
      "epoch": 2.3072625698324023,
      "grad_norm": 9.305696593070364,
      "learning_rate": 4.9705647365110746e-06,
      "logits/chosen": -3.5597293376922607,
      "logits/rejected": -3.6816155910491943,
      "logps/chosen": -6.089199542999268,
      "logps/rejected": -57.91256332397461,
      "loss": 0.2477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7718863487243652,
      "rewards/margins": 1.7718863487243652,
      "rewards/rejected": 0.0,
      "step": 413
    },
    {
      "epoch": 2.3128491620111733,
      "grad_norm": 17.83647431120677,
      "learning_rate": 4.970096593400894e-06,
      "logits/chosen": -3.605591297149658,
      "logits/rejected": -3.480286121368408,
      "logps/chosen": -13.26904010772705,
      "logps/rejected": -68.32940673828125,
      "loss": 0.3126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.518927574157715,
      "rewards/margins": 2.518927574157715,
      "rewards/rejected": 0.0,
      "step": 414
    },
    {
      "epoch": 2.3184357541899443,
      "grad_norm": 6.749008432879513,
      "learning_rate": 4.969624779287275e-06,
      "logits/chosen": -3.596463918685913,
      "logits/rejected": -3.480217933654785,
      "logps/chosen": -10.4995698928833,
      "logps/rejected": -28.05653953552246,
      "loss": 0.189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3841400146484375,
      "rewards/margins": 1.3841400146484375,
      "rewards/rejected": 0.0,
      "step": 415
    },
    {
      "epoch": 2.3240223463687153,
      "grad_norm": 15.98856211507587,
      "learning_rate": 4.969149294871417e-06,
      "logits/chosen": -3.752018451690674,
      "logits/rejected": -3.8669352531433105,
      "logps/chosen": -12.775226593017578,
      "logps/rejected": -38.96854782104492,
      "loss": 0.2925,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7472655773162842,
      "rewards/margins": 0.7472655773162842,
      "rewards/rejected": 0.0,
      "step": 416
    },
    {
      "epoch": 2.329608938547486,
      "grad_norm": 8.6244156827601,
      "learning_rate": 4.9686701408599765e-06,
      "logits/chosen": -3.6045093536376953,
      "logits/rejected": -3.40248703956604,
      "logps/chosen": -9.619041442871094,
      "logps/rejected": -37.123966217041016,
      "loss": 0.1661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2577977180480957,
      "rewards/margins": 2.2577977180480957,
      "rewards/rejected": 0.0,
      "step": 417
    },
    {
      "epoch": 2.335195530726257,
      "grad_norm": 9.336125273347784,
      "learning_rate": 4.96818731796506e-06,
      "logits/chosen": -3.2583911418914795,
      "logits/rejected": -3.25252103805542,
      "logps/chosen": -23.357982635498047,
      "logps/rejected": -28.66636848449707,
      "loss": 0.203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1874477863311768,
      "rewards/margins": 2.1874477863311768,
      "rewards/rejected": 0.0,
      "step": 418
    },
    {
      "epoch": 2.340782122905028,
      "grad_norm": 8.575391779240812,
      "learning_rate": 4.967700826904229e-06,
      "logits/chosen": -3.544948101043701,
      "logits/rejected": -3.547779083251953,
      "logps/chosen": -23.153709411621094,
      "logps/rejected": -42.894622802734375,
      "loss": 0.2285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.7197084426879883,
      "rewards/margins": 1.7197084426879883,
      "rewards/rejected": 0.0,
      "step": 419
    },
    {
      "epoch": 2.346368715083799,
      "grad_norm": 7.474446475426693,
      "learning_rate": 4.9672106684004965e-06,
      "logits/chosen": -3.7313382625579834,
      "logits/rejected": -3.2398791313171387,
      "logps/chosen": -11.048622131347656,
      "logps/rejected": -45.04474639892578,
      "loss": 0.243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.980872631072998,
      "rewards/margins": 0.980872631072998,
      "rewards/rejected": 0.0,
      "step": 420
    },
    {
      "epoch": 2.346368715083799,
      "eval_logits/chosen": -3.430154800415039,
      "eval_logits/rejected": -3.4917099475860596,
      "eval_logps/chosen": -25.22188377380371,
      "eval_logps/rejected": -39.00457763671875,
      "eval_loss": 0.6466967463493347,
      "eval_rewards/accuracies": 0.7749999761581421,
      "eval_rewards/chosen": 0.4848159849643707,
      "eval_rewards/margins": 0.4848159849643707,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5777,
      "eval_samples_per_second": 12.12,
      "eval_steps_per_second": 0.391,
      "step": 420
    },
    {
      "epoch": 2.35195530726257,
      "grad_norm": 16.304574663681915,
      "learning_rate": 4.9667168431823234e-06,
      "logits/chosen": -3.611527681350708,
      "logits/rejected": -3.278815507888794,
      "logps/chosen": -18.972021102905273,
      "logps/rejected": -23.859703063964844,
      "loss": 0.3237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0662593841552734,
      "rewards/margins": 1.0662593841552734,
      "rewards/rejected": 0.0,
      "step": 421
    },
    {
      "epoch": 2.357541899441341,
      "grad_norm": 6.185705465228968,
      "learning_rate": 4.966219351983623e-06,
      "logits/chosen": -3.7344651222229004,
      "logits/rejected": -3.7314536571502686,
      "logps/chosen": -10.771251678466797,
      "logps/rejected": -55.106929779052734,
      "loss": 0.1722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9542665481567383,
      "rewards/margins": 1.9542665481567383,
      "rewards/rejected": 0.0,
      "step": 422
    },
    {
      "epoch": 2.363128491620112,
      "grad_norm": 9.365864742773509,
      "learning_rate": 4.965718195543757e-06,
      "logits/chosen": -3.6108205318450928,
      "logits/rejected": -3.258544683456421,
      "logps/chosen": -7.752781391143799,
      "logps/rejected": -34.68027877807617,
      "loss": 0.2442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8499546051025391,
      "rewards/margins": 0.8499546051025391,
      "rewards/rejected": 0.0,
      "step": 423
    },
    {
      "epoch": 2.368715083798883,
      "grad_norm": 7.616949990085439,
      "learning_rate": 4.965213374607532e-06,
      "logits/chosen": -3.5425667762756348,
      "logits/rejected": -3.661477565765381,
      "logps/chosen": -15.192893028259277,
      "logps/rejected": -23.2348690032959,
      "loss": 0.2067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4291563034057617,
      "rewards/margins": 2.4291563034057617,
      "rewards/rejected": 0.0,
      "step": 424
    },
    {
      "epoch": 2.3743016759776534,
      "grad_norm": 11.377116105849238,
      "learning_rate": 4.964704889925201e-06,
      "logits/chosen": -3.4569437503814697,
      "logits/rejected": -3.5592877864837646,
      "logps/chosen": -13.247268676757812,
      "logps/rejected": -56.331214904785156,
      "loss": 0.2922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6894659996032715,
      "rewards/margins": 2.6894659996032715,
      "rewards/rejected": 0.0,
      "step": 425
    },
    {
      "epoch": 2.3798882681564244,
      "grad_norm": 6.66429798416961,
      "learning_rate": 4.964192742252464e-06,
      "logits/chosen": -3.7106165885925293,
      "logits/rejected": -3.631795883178711,
      "logps/chosen": -12.781500816345215,
      "logps/rejected": -37.01472854614258,
      "loss": 0.2191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1149332523345947,
      "rewards/margins": 1.1149332523345947,
      "rewards/rejected": 0.0,
      "step": 426
    },
    {
      "epoch": 2.3854748603351954,
      "grad_norm": 15.595591481654179,
      "learning_rate": 4.963676932350463e-06,
      "logits/chosen": -3.813253879547119,
      "logits/rejected": -3.7080752849578857,
      "logps/chosen": -14.919236183166504,
      "logps/rejected": -25.616317749023438,
      "loss": 0.3064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.76207435131073,
      "rewards/margins": 0.76207435131073,
      "rewards/rejected": 0.0,
      "step": 427
    },
    {
      "epoch": 2.3910614525139664,
      "grad_norm": 9.210627626401765,
      "learning_rate": 4.963157460985784e-06,
      "logits/chosen": -3.0816783905029297,
      "logits/rejected": -3.2561745643615723,
      "logps/chosen": -17.636070251464844,
      "logps/rejected": -39.017860412597656,
      "loss": 0.2265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5654361248016357,
      "rewards/margins": 1.5654361248016357,
      "rewards/rejected": 0.0,
      "step": 428
    },
    {
      "epoch": 2.3966480446927374,
      "grad_norm": 6.276897985503736,
      "learning_rate": 4.9626343289304536e-06,
      "logits/chosen": -3.685936450958252,
      "logits/rejected": -3.7742390632629395,
      "logps/chosen": -6.460122585296631,
      "logps/rejected": -26.266361236572266,
      "loss": 0.1915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2810802459716797,
      "rewards/margins": 1.2810802459716797,
      "rewards/rejected": 0.0,
      "step": 429
    },
    {
      "epoch": 2.4022346368715084,
      "grad_norm": 8.480348968574571,
      "learning_rate": 4.9621075369619394e-06,
      "logits/chosen": -3.392352819442749,
      "logits/rejected": -3.182173728942871,
      "logps/chosen": -17.05902671813965,
      "logps/rejected": -27.338346481323242,
      "loss": 0.2294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.5215271711349487,
      "rewards/margins": 1.5215271711349487,
      "rewards/rejected": 0.0,
      "step": 430
    },
    {
      "epoch": 2.4078212290502794,
      "grad_norm": 8.728683182632818,
      "learning_rate": 4.961577085863148e-06,
      "logits/chosen": -3.7438673973083496,
      "logits/rejected": -2.9077682495117188,
      "logps/chosen": -9.9461030960083,
      "logps/rejected": -41.39183044433594,
      "loss": 0.2382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8264210224151611,
      "rewards/margins": 1.8264210224151611,
      "rewards/rejected": 0.0,
      "step": 431
    },
    {
      "epoch": 2.4134078212290504,
      "grad_norm": 15.87432974979997,
      "learning_rate": 4.9610429764224234e-06,
      "logits/chosen": -3.4335601329803467,
      "logits/rejected": -3.637930154800415,
      "logps/chosen": -19.071182250976562,
      "logps/rejected": -31.80838394165039,
      "loss": 0.3113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6482436656951904,
      "rewards/margins": 1.6482436656951904,
      "rewards/rejected": 0.0,
      "step": 432
    },
    {
      "epoch": 2.4189944134078214,
      "grad_norm": 14.065556036637448,
      "learning_rate": 4.960505209433549e-06,
      "logits/chosen": -3.930577039718628,
      "logits/rejected": -3.863051414489746,
      "logps/chosen": -24.816280364990234,
      "logps/rejected": -29.330059051513672,
      "loss": 0.3424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1597429513931274,
      "rewards/margins": 1.1597429513931274,
      "rewards/rejected": 0.0,
      "step": 433
    },
    {
      "epoch": 2.4245810055865924,
      "grad_norm": 5.987718956175602,
      "learning_rate": 4.95996378569574e-06,
      "logits/chosen": -3.3813023567199707,
      "logits/rejected": -3.3773443698883057,
      "logps/chosen": -7.563570499420166,
      "logps/rejected": -33.30601501464844,
      "loss": 0.1647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6568121910095215,
      "rewards/margins": 1.6568121910095215,
      "rewards/rejected": 0.0,
      "step": 434
    },
    {
      "epoch": 2.430167597765363,
      "grad_norm": 5.79198085765812,
      "learning_rate": 4.959418706013651e-06,
      "logits/chosen": -3.7298686504364014,
      "logits/rejected": -3.5471932888031006,
      "logps/chosen": -12.287824630737305,
      "logps/rejected": -50.811927795410156,
      "loss": 0.1894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.650527238845825,
      "rewards/margins": 2.650527238845825,
      "rewards/rejected": 0.0,
      "step": 435
    },
    {
      "epoch": 2.435754189944134,
      "grad_norm": 8.468262669140856,
      "learning_rate": 4.958869971197367e-06,
      "logits/chosen": -3.5113391876220703,
      "logits/rejected": -3.538116693496704,
      "logps/chosen": -9.906305313110352,
      "logps/rejected": -43.73832702636719,
      "loss": 0.2345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.25618577003479,
      "rewards/margins": 1.25618577003479,
      "rewards/rejected": 0.0,
      "step": 436
    },
    {
      "epoch": 2.441340782122905,
      "grad_norm": 12.564253590004972,
      "learning_rate": 4.9583175820624035e-06,
      "logits/chosen": -3.7604453563690186,
      "logits/rejected": -3.9554851055145264,
      "logps/chosen": -12.04368782043457,
      "logps/rejected": -62.88062286376953,
      "loss": 0.2555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.144263505935669,
      "rewards/margins": 1.144263505935669,
      "rewards/rejected": 0.0,
      "step": 437
    },
    {
      "epoch": 2.446927374301676,
      "grad_norm": 11.015120503353197,
      "learning_rate": 4.957761539429712e-06,
      "logits/chosen": -3.139028549194336,
      "logits/rejected": -3.3601629734039307,
      "logps/chosen": -19.023967742919922,
      "logps/rejected": -23.190258026123047,
      "loss": 0.3192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.242623805999756,
      "rewards/margins": 2.242623805999756,
      "rewards/rejected": 0.0,
      "step": 438
    },
    {
      "epoch": 2.452513966480447,
      "grad_norm": 8.86306153938212,
      "learning_rate": 4.957201844125668e-06,
      "logits/chosen": -3.415107488632202,
      "logits/rejected": -3.3250598907470703,
      "logps/chosen": -12.171310424804688,
      "logps/rejected": -73.0188980102539,
      "loss": 0.2192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.87996244430542,
      "rewards/margins": 1.87996244430542,
      "rewards/rejected": 0.0,
      "step": 439
    },
    {
      "epoch": 2.458100558659218,
      "grad_norm": 5.604219232851792,
      "learning_rate": 4.956638496982081e-06,
      "logits/chosen": -3.6978025436401367,
      "logits/rejected": -3.440706729888916,
      "logps/chosen": -18.612173080444336,
      "logps/rejected": -45.73664474487305,
      "loss": 0.1532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.127530813217163,
      "rewards/margins": 3.127530813217163,
      "rewards/rejected": 0.0,
      "step": 440
    },
    {
      "epoch": 2.458100558659218,
      "eval_logits/chosen": -3.4241397380828857,
      "eval_logits/rejected": -3.479137897491455,
      "eval_logps/chosen": -25.918277740478516,
      "eval_logps/rejected": -38.23731994628906,
      "eval_loss": 0.6663566827774048,
      "eval_rewards/accuracies": 0.7749999761581421,
      "eval_rewards/chosen": 0.41517654061317444,
      "eval_rewards/margins": 0.41517654061317444,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5685,
      "eval_samples_per_second": 12.124,
      "eval_steps_per_second": 0.391,
      "step": 440
    },
    {
      "epoch": 2.463687150837989,
      "grad_norm": 13.640629508412836,
      "learning_rate": 4.956071498836184e-06,
      "logits/chosen": -3.763352155685425,
      "logits/rejected": -3.682645797729492,
      "logps/chosen": -40.50086975097656,
      "logps/rejected": -29.64499282836914,
      "loss": 0.2721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.925788402557373,
      "rewards/margins": 2.925788402557373,
      "rewards/rejected": 0.0,
      "step": 441
    },
    {
      "epoch": 2.46927374301676,
      "grad_norm": 8.204506136102413,
      "learning_rate": 4.955500850530637e-06,
      "logits/chosen": -3.5452334880828857,
      "logits/rejected": -3.5358643531799316,
      "logps/chosen": -6.444394111633301,
      "logps/rejected": -26.7586669921875,
      "loss": 0.1916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4184943437576294,
      "rewards/margins": 1.4184943437576294,
      "rewards/rejected": 0.0,
      "step": 442
    },
    {
      "epoch": 2.4748603351955305,
      "grad_norm": 7.622584846774958,
      "learning_rate": 4.954926552913525e-06,
      "logits/chosen": -3.698632001876831,
      "logits/rejected": -3.5752391815185547,
      "logps/chosen": -7.806251525878906,
      "logps/rejected": -40.67234420776367,
      "loss": 0.2315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3782196044921875,
      "rewards/margins": 1.3782196044921875,
      "rewards/rejected": 0.0,
      "step": 443
    },
    {
      "epoch": 2.4804469273743015,
      "grad_norm": 12.191658529489187,
      "learning_rate": 4.954348606838358e-06,
      "logits/chosen": -3.532078266143799,
      "logits/rejected": -3.476181983947754,
      "logps/chosen": -23.74327850341797,
      "logps/rejected": -49.2454719543457,
      "loss": 0.2803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.971900463104248,
      "rewards/margins": 1.971900463104248,
      "rewards/rejected": 0.0,
      "step": 444
    },
    {
      "epoch": 2.4860335195530725,
      "grad_norm": 15.666505693719596,
      "learning_rate": 4.953767013164065e-06,
      "logits/chosen": -3.6603217124938965,
      "logits/rejected": -3.7852041721343994,
      "logps/chosen": -8.152360916137695,
      "logps/rejected": -89.96377563476562,
      "loss": 0.2765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1948775053024292,
      "rewards/margins": 1.1948775053024292,
      "rewards/rejected": 0.0,
      "step": 445
    },
    {
      "epoch": 2.4916201117318435,
      "grad_norm": 9.855342921053886,
      "learning_rate": 4.953181772754998e-06,
      "logits/chosen": -3.715217351913452,
      "logits/rejected": -3.638705015182495,
      "logps/chosen": -9.717981338500977,
      "logps/rejected": -29.45804214477539,
      "loss": 0.2229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8973733186721802,
      "rewards/margins": 1.8973733186721802,
      "rewards/rejected": 0.0,
      "step": 446
    },
    {
      "epoch": 2.4972067039106145,
      "grad_norm": 9.88973344067588,
      "learning_rate": 4.952592886480929e-06,
      "logits/chosen": -3.5098559856414795,
      "logits/rejected": -3.4249813556671143,
      "logps/chosen": -36.327423095703125,
      "logps/rejected": -33.338836669921875,
      "loss": 0.2304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4162535667419434,
      "rewards/margins": 2.4162535667419434,
      "rewards/rejected": 0.0,
      "step": 447
    },
    {
      "epoch": 2.5027932960893855,
      "grad_norm": 10.31561906151943,
      "learning_rate": 4.952000355217049e-06,
      "logits/chosen": -3.7112200260162354,
      "logits/rejected": -3.5888285636901855,
      "logps/chosen": -16.69519805908203,
      "logps/rejected": -34.421112060546875,
      "loss": 0.262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1247596740722656,
      "rewards/margins": 2.1247596740722656,
      "rewards/rejected": 0.0,
      "step": 448
    },
    {
      "epoch": 2.5083798882681565,
      "grad_norm": 9.694260789038083,
      "learning_rate": 4.951404179843963e-06,
      "logits/chosen": -3.37312912940979,
      "logits/rejected": -3.363196849822998,
      "logps/chosen": -9.445751190185547,
      "logps/rejected": -59.14949417114258,
      "loss": 0.2752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8089487552642822,
      "rewards/margins": 1.8089487552642822,
      "rewards/rejected": 0.0,
      "step": 449
    },
    {
      "epoch": 2.5139664804469275,
      "grad_norm": 10.383685347036307,
      "learning_rate": 4.950804361247694e-06,
      "logits/chosen": -3.8457746505737305,
      "logits/rejected": -3.815129280090332,
      "logps/chosen": -9.37015151977539,
      "logps/rejected": -33.414947509765625,
      "loss": 0.2809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8543004989624023,
      "rewards/margins": 1.8543004989624023,
      "rewards/rejected": 0.0,
      "step": 450
    },
    {
      "epoch": 2.5195530726256985,
      "grad_norm": 7.869319417770778,
      "learning_rate": 4.95020090031968e-06,
      "logits/chosen": -3.7749829292297363,
      "logits/rejected": -3.585822820663452,
      "logps/chosen": -9.538869857788086,
      "logps/rejected": -38.587520599365234,
      "loss": 0.221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1329652070999146,
      "rewards/margins": 1.1329652070999146,
      "rewards/rejected": 0.0,
      "step": 451
    },
    {
      "epoch": 2.5251396648044695,
      "grad_norm": 8.743347682099401,
      "learning_rate": 4.949593797956772e-06,
      "logits/chosen": -3.0479891300201416,
      "logits/rejected": -2.8850865364074707,
      "logps/chosen": -22.390270233154297,
      "logps/rejected": -42.44178009033203,
      "loss": 0.2468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7790570259094238,
      "rewards/margins": 1.7790570259094238,
      "rewards/rejected": 0.0,
      "step": 452
    },
    {
      "epoch": 2.5307262569832405,
      "grad_norm": 11.402812909713921,
      "learning_rate": 4.948983055061231e-06,
      "logits/chosen": -3.7795796394348145,
      "logits/rejected": -3.730720043182373,
      "logps/chosen": -10.82972526550293,
      "logps/rejected": -37.638641357421875,
      "loss": 0.2724,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.3601937294006348,
      "rewards/margins": 1.3601937294006348,
      "rewards/rejected": 0.0,
      "step": 453
    },
    {
      "epoch": 2.536312849162011,
      "grad_norm": 7.977591654894314,
      "learning_rate": 4.9483686725407295e-06,
      "logits/chosen": -3.573173999786377,
      "logits/rejected": -3.651446580886841,
      "logps/chosen": -7.591944694519043,
      "logps/rejected": -28.446674346923828,
      "loss": 0.195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3893461227416992,
      "rewards/margins": 1.3893461227416992,
      "rewards/rejected": 0.0,
      "step": 454
    },
    {
      "epoch": 2.541899441340782,
      "grad_norm": 9.299688374855851,
      "learning_rate": 4.947750651308351e-06,
      "logits/chosen": -3.5672802925109863,
      "logits/rejected": -3.5629284381866455,
      "logps/chosen": -9.63311767578125,
      "logps/rejected": -34.95433807373047,
      "loss": 0.2422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6046555042266846,
      "rewards/margins": 1.6046555042266846,
      "rewards/rejected": 0.0,
      "step": 455
    },
    {
      "epoch": 2.547486033519553,
      "grad_norm": 10.747071707790564,
      "learning_rate": 4.9471289922825835e-06,
      "logits/chosen": -3.5354928970336914,
      "logits/rejected": -3.714805841445923,
      "logps/chosen": -11.917631149291992,
      "logps/rejected": -48.64508819580078,
      "loss": 0.2764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5865811109542847,
      "rewards/margins": 1.5865811109542847,
      "rewards/rejected": 0.0,
      "step": 456
    },
    {
      "epoch": 2.553072625698324,
      "grad_norm": 10.183896096457042,
      "learning_rate": 4.9465036963873234e-06,
      "logits/chosen": -3.6406567096710205,
      "logits/rejected": -3.7196829319000244,
      "logps/chosen": -6.481830596923828,
      "logps/rejected": -30.47899627685547,
      "loss": 0.217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9919025897979736,
      "rewards/margins": 0.9919025897979736,
      "rewards/rejected": 0.0,
      "step": 457
    },
    {
      "epoch": 2.558659217877095,
      "grad_norm": 14.34131473914047,
      "learning_rate": 4.945874764551873e-06,
      "logits/chosen": -3.480870485305786,
      "logits/rejected": -3.5965325832366943,
      "logps/chosen": -5.0016326904296875,
      "logps/rejected": -62.639801025390625,
      "loss": 0.2936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.963644802570343,
      "rewards/margins": 0.963644802570343,
      "rewards/rejected": 0.0,
      "step": 458
    },
    {
      "epoch": 2.564245810055866,
      "grad_norm": 11.145716584300128,
      "learning_rate": 4.945242197710935e-06,
      "logits/chosen": -3.41308856010437,
      "logits/rejected": -3.52736759185791,
      "logps/chosen": -14.65832233428955,
      "logps/rejected": -44.82463073730469,
      "loss": 0.271,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4274589419364929,
      "rewards/margins": 0.4274589419364929,
      "rewards/rejected": 0.0,
      "step": 459
    },
    {
      "epoch": 2.5698324022346366,
      "grad_norm": 11.582236461354283,
      "learning_rate": 4.944605996804618e-06,
      "logits/chosen": -3.500563383102417,
      "logits/rejected": -3.51568603515625,
      "logps/chosen": -26.387428283691406,
      "logps/rejected": -86.21025085449219,
      "loss": 0.2572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7578389644622803,
      "rewards/margins": 1.7578389644622803,
      "rewards/rejected": 0.0,
      "step": 460
    },
    {
      "epoch": 2.5698324022346366,
      "eval_logits/chosen": -3.4121031761169434,
      "eval_logits/rejected": -3.4722087383270264,
      "eval_logps/chosen": -26.48482322692871,
      "eval_logps/rejected": -38.31188201904297,
      "eval_loss": 0.6773480772972107,
      "eval_rewards/accuracies": 0.7250000238418579,
      "eval_rewards/chosen": 0.358521968126297,
      "eval_rewards/margins": 0.358521968126297,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.585,
      "eval_samples_per_second": 12.116,
      "eval_steps_per_second": 0.391,
      "step": 460
    },
    {
      "epoch": 2.5754189944134076,
      "grad_norm": 9.9519075131971,
      "learning_rate": 4.9439661627784295e-06,
      "logits/chosen": -3.450260639190674,
      "logits/rejected": -3.51298189163208,
      "logps/chosen": -32.49373245239258,
      "logps/rejected": -38.14031982421875,
      "loss": 0.2655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6734542846679688,
      "rewards/margins": 2.6734542846679688,
      "rewards/rejected": 0.0,
      "step": 461
    },
    {
      "epoch": 2.5810055865921786,
      "grad_norm": 12.464303941966826,
      "learning_rate": 4.943322696583276e-06,
      "logits/chosen": -3.488281488418579,
      "logits/rejected": -3.6389501094818115,
      "logps/chosen": -18.7330322265625,
      "logps/rejected": -47.90739059448242,
      "loss": 0.2478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7415714263916016,
      "rewards/margins": 1.7415714263916016,
      "rewards/rejected": 0.0,
      "step": 462
    },
    {
      "epoch": 2.5865921787709496,
      "grad_norm": 14.121677503395844,
      "learning_rate": 4.942675599175464e-06,
      "logits/chosen": -3.2792911529541016,
      "logits/rejected": -3.2519619464874268,
      "logps/chosen": -12.003146171569824,
      "logps/rejected": -53.46558380126953,
      "loss": 0.3102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3137681484222412,
      "rewards/margins": 1.3137681484222412,
      "rewards/rejected": 0.0,
      "step": 463
    },
    {
      "epoch": 2.5921787709497206,
      "grad_norm": 16.55502089846945,
      "learning_rate": 4.942024871516694e-06,
      "logits/chosen": -3.467578172683716,
      "logits/rejected": -3.2962722778320312,
      "logps/chosen": -25.32575798034668,
      "logps/rejected": -56.88692092895508,
      "loss": 0.3913,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09861046075820923,
      "rewards/margins": 0.09861046075820923,
      "rewards/rejected": 0.0,
      "step": 464
    },
    {
      "epoch": 2.5977653631284916,
      "grad_norm": 7.984618783608261,
      "learning_rate": 4.941370514574064e-06,
      "logits/chosen": -3.3088600635528564,
      "logits/rejected": -3.3386614322662354,
      "logps/chosen": -44.124534606933594,
      "logps/rejected": -41.44841003417969,
      "loss": 0.1895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5208206176757812,
      "rewards/margins": 3.5208206176757812,
      "rewards/rejected": 0.0,
      "step": 465
    },
    {
      "epoch": 2.6033519553072626,
      "grad_norm": 9.59665812094458,
      "learning_rate": 4.940712529320066e-06,
      "logits/chosen": -3.668675422668457,
      "logits/rejected": -3.6745896339416504,
      "logps/chosen": -20.23447036743164,
      "logps/rejected": -42.10270690917969,
      "loss": 0.2758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3388655185699463,
      "rewards/margins": 1.3388655185699463,
      "rewards/rejected": 0.0,
      "step": 466
    },
    {
      "epoch": 2.6089385474860336,
      "grad_norm": 8.216069152596615,
      "learning_rate": 4.940050916732582e-06,
      "logits/chosen": -3.6291463375091553,
      "logits/rejected": -3.7243592739105225,
      "logps/chosen": -7.182065963745117,
      "logps/rejected": -31.376358032226562,
      "loss": 0.215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4207298755645752,
      "rewards/margins": 1.4207298755645752,
      "rewards/rejected": 0.0,
      "step": 467
    },
    {
      "epoch": 2.6145251396648046,
      "grad_norm": 9.528386884255825,
      "learning_rate": 4.9393856777948855e-06,
      "logits/chosen": -3.7146143913269043,
      "logits/rejected": -3.6014597415924072,
      "logps/chosen": -32.09270477294922,
      "logps/rejected": -49.89198303222656,
      "loss": 0.2297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1596074104309082,
      "rewards/margins": 1.1596074104309082,
      "rewards/rejected": 0.0,
      "step": 468
    },
    {
      "epoch": 2.6201117318435756,
      "grad_norm": 11.04818745405567,
      "learning_rate": 4.938716813495642e-06,
      "logits/chosen": -3.6191561222076416,
      "logits/rejected": -3.6325783729553223,
      "logps/chosen": -8.461179733276367,
      "logps/rejected": -49.84785461425781,
      "loss": 0.3798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9204212427139282,
      "rewards/margins": 1.9204212427139282,
      "rewards/rejected": 0.0,
      "step": 469
    },
    {
      "epoch": 2.6256983240223466,
      "grad_norm": 6.656274782148886,
      "learning_rate": 4.9380443248289025e-06,
      "logits/chosen": -4.020930767059326,
      "logits/rejected": -4.010366916656494,
      "logps/chosen": -10.222768783569336,
      "logps/rejected": -24.6385440826416,
      "loss": 0.2384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.559239149093628,
      "rewards/margins": 1.559239149093628,
      "rewards/rejected": 0.0,
      "step": 470
    },
    {
      "epoch": 2.631284916201117,
      "grad_norm": 9.044383795959122,
      "learning_rate": 4.937368212794103e-06,
      "logits/chosen": -3.7412068843841553,
      "logits/rejected": -3.807184934616089,
      "logps/chosen": -10.33376407623291,
      "logps/rejected": -39.550514221191406,
      "loss": 0.2267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.195363998413086,
      "rewards/margins": 2.195363998413086,
      "rewards/rejected": 0.0,
      "step": 471
    },
    {
      "epoch": 2.636871508379888,
      "grad_norm": 9.153636738745826,
      "learning_rate": 4.936688478396068e-06,
      "logits/chosen": -3.543262004852295,
      "logits/rejected": -3.5626437664031982,
      "logps/chosen": -13.64102840423584,
      "logps/rejected": -26.64143180847168,
      "loss": 0.2425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0355925559997559,
      "rewards/margins": 1.0355925559997559,
      "rewards/rejected": 0.0,
      "step": 472
    },
    {
      "epoch": 2.642458100558659,
      "grad_norm": 7.577096041982333,
      "learning_rate": 4.936005122645004e-06,
      "logits/chosen": -3.789921760559082,
      "logits/rejected": -3.533780813217163,
      "logps/chosen": -11.747702598571777,
      "logps/rejected": -26.601421356201172,
      "loss": 0.185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.963315486907959,
      "rewards/margins": 1.963315486907959,
      "rewards/rejected": 0.0,
      "step": 473
    },
    {
      "epoch": 2.64804469273743,
      "grad_norm": 10.119833803992162,
      "learning_rate": 4.935318146556499e-06,
      "logits/chosen": -3.8856709003448486,
      "logits/rejected": -3.736361265182495,
      "logps/chosen": -17.659000396728516,
      "logps/rejected": -36.72257995605469,
      "loss": 0.2724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5814303159713745,
      "rewards/margins": 1.5814303159713745,
      "rewards/rejected": 0.0,
      "step": 474
    },
    {
      "epoch": 2.653631284916201,
      "grad_norm": 11.020569131476357,
      "learning_rate": 4.934627551151522e-06,
      "logits/chosen": -3.688471794128418,
      "logits/rejected": -3.6178042888641357,
      "logps/chosen": -17.2575740814209,
      "logps/rejected": -39.61146545410156,
      "loss": 0.3276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6362807750701904,
      "rewards/margins": 1.6362807750701904,
      "rewards/rejected": 0.0,
      "step": 475
    },
    {
      "epoch": 2.659217877094972,
      "grad_norm": 5.982156132488162,
      "learning_rate": 4.93393333745642e-06,
      "logits/chosen": -3.5872161388397217,
      "logits/rejected": -3.5497727394104004,
      "logps/chosen": -34.132503509521484,
      "logps/rejected": -28.261878967285156,
      "loss": 0.1978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9942636489868164,
      "rewards/margins": 1.9942636489868164,
      "rewards/rejected": 0.0,
      "step": 476
    },
    {
      "epoch": 2.664804469273743,
      "grad_norm": 6.704635146537391,
      "learning_rate": 4.933235506502919e-06,
      "logits/chosen": -3.447739362716675,
      "logits/rejected": -3.410402536392212,
      "logps/chosen": -26.60841178894043,
      "logps/rejected": -38.12532043457031,
      "loss": 0.2171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3351829051971436,
      "rewards/margins": 2.3351829051971436,
      "rewards/rejected": 0.0,
      "step": 477
    },
    {
      "epoch": 2.6703910614525137,
      "grad_norm": 7.5723632181214535,
      "learning_rate": 4.93253405932812e-06,
      "logits/chosen": -3.7006425857543945,
      "logits/rejected": -3.4994475841522217,
      "logps/chosen": -14.325606346130371,
      "logps/rejected": -65.43325805664062,
      "loss": 0.2166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.278433918952942,
      "rewards/margins": 1.278433918952942,
      "rewards/rejected": 0.0,
      "step": 478
    },
    {
      "epoch": 2.6759776536312847,
      "grad_norm": 9.51480704145188,
      "learning_rate": 4.931828996974498e-06,
      "logits/chosen": -3.49611234664917,
      "logits/rejected": -3.3973209857940674,
      "logps/chosen": -4.141186714172363,
      "logps/rejected": -29.483539581298828,
      "loss": 0.2734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1052652597427368,
      "rewards/margins": 1.1052652597427368,
      "rewards/rejected": 0.0,
      "step": 479
    },
    {
      "epoch": 2.6815642458100557,
      "grad_norm": 11.876114808876137,
      "learning_rate": 4.931120320489902e-06,
      "logits/chosen": -3.39361310005188,
      "logits/rejected": -3.5273473262786865,
      "logps/chosen": -20.93730354309082,
      "logps/rejected": -56.397342681884766,
      "loss": 0.2698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.2912646532058716,
      "rewards/margins": 1.2912646532058716,
      "rewards/rejected": 0.0,
      "step": 480
    },
    {
      "epoch": 2.6815642458100557,
      "eval_logits/chosen": -3.4202568531036377,
      "eval_logits/rejected": -3.499009370803833,
      "eval_logps/chosen": -26.18922996520996,
      "eval_logps/rejected": -39.039676666259766,
      "eval_loss": 0.6857407093048096,
      "eval_rewards/accuracies": 0.7749999761581421,
      "eval_rewards/chosen": 0.38808125257492065,
      "eval_rewards/margins": 0.38808125257492065,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.633,
      "eval_samples_per_second": 12.094,
      "eval_steps_per_second": 0.39,
      "step": 480
    },
    {
      "epoch": 2.6871508379888267,
      "grad_norm": 12.97823928616358,
      "learning_rate": 4.930408030927549e-06,
      "logits/chosen": -3.772036075592041,
      "logits/rejected": -3.9154014587402344,
      "logps/chosen": -15.563390731811523,
      "logps/rejected": -37.88628387451172,
      "loss": 0.2857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.305469036102295,
      "rewards/margins": 1.305469036102295,
      "rewards/rejected": 0.0,
      "step": 481
    },
    {
      "epoch": 2.6927374301675977,
      "grad_norm": 12.459298783598237,
      "learning_rate": 4.929692129346032e-06,
      "logits/chosen": -3.6585543155670166,
      "logits/rejected": -3.4301023483276367,
      "logps/chosen": -17.022111892700195,
      "logps/rejected": -48.912635803222656,
      "loss": 0.2793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2236939668655396,
      "rewards/margins": 1.2236939668655396,
      "rewards/rejected": 0.0,
      "step": 482
    },
    {
      "epoch": 2.6983240223463687,
      "grad_norm": 8.718524644149529,
      "learning_rate": 4.928972616809306e-06,
      "logits/chosen": -3.4232821464538574,
      "logits/rejected": -3.3921141624450684,
      "logps/chosen": -25.136520385742188,
      "logps/rejected": -48.49579620361328,
      "loss": 0.2285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4094929695129395,
      "rewards/margins": 2.4094929695129395,
      "rewards/rejected": 0.0,
      "step": 483
    },
    {
      "epoch": 2.7039106145251397,
      "grad_norm": 14.768960485933134,
      "learning_rate": 4.928249494386694e-06,
      "logits/chosen": -3.743952751159668,
      "logits/rejected": -3.8500096797943115,
      "logps/chosen": -6.842565059661865,
      "logps/rejected": -51.62577819824219,
      "loss": 0.3319,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9490782022476196,
      "rewards/margins": 0.9490782022476196,
      "rewards/rejected": 0.0,
      "step": 484
    },
    {
      "epoch": 2.7094972067039107,
      "grad_norm": 14.469963146242783,
      "learning_rate": 4.927522763152886e-06,
      "logits/chosen": -3.6242997646331787,
      "logits/rejected": -3.332655191421509,
      "logps/chosen": -8.909425735473633,
      "logps/rejected": -23.911720275878906,
      "loss": 0.3465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4682164192199707,
      "rewards/margins": 1.4682164192199707,
      "rewards/rejected": 0.0,
      "step": 485
    },
    {
      "epoch": 2.7150837988826817,
      "grad_norm": 7.462685406687471,
      "learning_rate": 4.926792424187935e-06,
      "logits/chosen": -3.838343858718872,
      "logits/rejected": -3.7386252880096436,
      "logps/chosen": -9.146822929382324,
      "logps/rejected": -21.344528198242188,
      "loss": 0.2398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2866406440734863,
      "rewards/margins": 1.2866406440734863,
      "rewards/rejected": 0.0,
      "step": 486
    },
    {
      "epoch": 2.7206703910614527,
      "grad_norm": 7.148204527169305,
      "learning_rate": 4.9260584785772535e-06,
      "logits/chosen": -3.875157594680786,
      "logits/rejected": -3.725428342819214,
      "logps/chosen": -10.537574768066406,
      "logps/rejected": -36.27181625366211,
      "loss": 0.2113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6521003246307373,
      "rewards/margins": 1.6521003246307373,
      "rewards/rejected": 0.0,
      "step": 487
    },
    {
      "epoch": 2.7262569832402237,
      "grad_norm": 10.987404382513079,
      "learning_rate": 4.925320927411614e-06,
      "logits/chosen": -3.607668876647949,
      "logits/rejected": -3.4913244247436523,
      "logps/chosen": -13.60542106628418,
      "logps/rejected": -39.43303680419922,
      "loss": 0.2577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1527724266052246,
      "rewards/margins": 2.1527724266052246,
      "rewards/rejected": 0.0,
      "step": 488
    },
    {
      "epoch": 2.7318435754189943,
      "grad_norm": 10.323562226932104,
      "learning_rate": 4.924579771787151e-06,
      "logits/chosen": -3.6909854412078857,
      "logits/rejected": -3.6763968467712402,
      "logps/chosen": -17.746667861938477,
      "logps/rejected": -31.590293884277344,
      "loss": 0.2496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8706515431404114,
      "rewards/margins": 0.8706515431404114,
      "rewards/rejected": 0.0,
      "step": 489
    },
    {
      "epoch": 2.7374301675977653,
      "grad_norm": 9.321767993528761,
      "learning_rate": 4.9238350128053536e-06,
      "logits/chosen": -3.4091458320617676,
      "logits/rejected": -3.626840591430664,
      "logps/chosen": -14.06284236907959,
      "logps/rejected": -38.00033187866211,
      "loss": 0.2298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2537178993225098,
      "rewards/margins": 1.2537178993225098,
      "rewards/rejected": 0.0,
      "step": 490
    },
    {
      "epoch": 2.7430167597765363,
      "grad_norm": 8.428012779509059,
      "learning_rate": 4.9230866515730655e-06,
      "logits/chosen": -3.5724446773529053,
      "logits/rejected": -3.3720438480377197,
      "logps/chosen": -11.452644348144531,
      "logps/rejected": -31.765769958496094,
      "loss": 0.2442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0843114852905273,
      "rewards/margins": 1.0843114852905273,
      "rewards/rejected": 0.0,
      "step": 491
    },
    {
      "epoch": 2.7486033519553073,
      "grad_norm": 5.569123370468062,
      "learning_rate": 4.922334689202485e-06,
      "logits/chosen": -3.8108203411102295,
      "logits/rejected": -3.8188564777374268,
      "logps/chosen": -8.907066345214844,
      "logps/rejected": -53.68848419189453,
      "loss": 0.1787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0030553340911865,
      "rewards/margins": 2.0030553340911865,
      "rewards/rejected": 0.0,
      "step": 492
    },
    {
      "epoch": 2.7541899441340782,
      "grad_norm": 9.761864659532193,
      "learning_rate": 4.921579126811162e-06,
      "logits/chosen": -3.800229549407959,
      "logits/rejected": -3.7908823490142822,
      "logps/chosen": -10.309123992919922,
      "logps/rejected": -34.62474060058594,
      "loss": 0.2122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4503856897354126,
      "rewards/margins": 1.4503856897354126,
      "rewards/rejected": 0.0,
      "step": 493
    },
    {
      "epoch": 2.7597765363128492,
      "grad_norm": 7.027664145662218,
      "learning_rate": 4.920819965521997e-06,
      "logits/chosen": -3.4141292572021484,
      "logits/rejected": -3.5322158336639404,
      "logps/chosen": -4.670951843261719,
      "logps/rejected": -45.06983947753906,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1251702308654785,
      "rewards/margins": 1.1251702308654785,
      "rewards/rejected": 0.0,
      "step": 494
    },
    {
      "epoch": 2.7653631284916202,
      "grad_norm": 12.608137024479488,
      "learning_rate": 4.920057206463239e-06,
      "logits/chosen": -3.794767141342163,
      "logits/rejected": -3.726362943649292,
      "logps/chosen": -15.259572982788086,
      "logps/rejected": -41.00498580932617,
      "loss": 0.2582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5710067749023438,
      "rewards/margins": 0.5710067749023438,
      "rewards/rejected": 0.0,
      "step": 495
    },
    {
      "epoch": 2.770949720670391,
      "grad_norm": 13.363080361307707,
      "learning_rate": 4.9192908507684835e-06,
      "logits/chosen": -3.7646498680114746,
      "logits/rejected": -3.8265600204467773,
      "logps/chosen": -20.700878143310547,
      "logps/rejected": -35.19379806518555,
      "loss": 0.2719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.490762710571289,
      "rewards/margins": 2.490762710571289,
      "rewards/rejected": 0.0,
      "step": 496
    },
    {
      "epoch": 2.776536312849162,
      "grad_norm": 9.183927248198609,
      "learning_rate": 4.918520899576671e-06,
      "logits/chosen": -3.6508829593658447,
      "logits/rejected": -3.5203747749328613,
      "logps/chosen": -12.967455863952637,
      "logps/rejected": -39.27170181274414,
      "loss": 0.2517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4772241115570068,
      "rewards/margins": 1.4772241115570068,
      "rewards/rejected": 0.0,
      "step": 497
    },
    {
      "epoch": 2.782122905027933,
      "grad_norm": 9.53564834374126,
      "learning_rate": 4.917747354032087e-06,
      "logits/chosen": -3.591912269592285,
      "logits/rejected": -3.60760235786438,
      "logps/chosen": -6.920201301574707,
      "logps/rejected": -58.45861053466797,
      "loss": 0.2456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4739320278167725,
      "rewards/margins": 1.4739320278167725,
      "rewards/rejected": 0.0,
      "step": 498
    },
    {
      "epoch": 2.787709497206704,
      "grad_norm": 12.330089141072532,
      "learning_rate": 4.916970215284358e-06,
      "logits/chosen": -3.3424952030181885,
      "logits/rejected": -3.4522621631622314,
      "logps/chosen": -14.62384033203125,
      "logps/rejected": -38.694358825683594,
      "loss": 0.2366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8516249656677246,
      "rewards/margins": 1.8516249656677246,
      "rewards/rejected": 0.0,
      "step": 499
    },
    {
      "epoch": 2.793296089385475,
      "grad_norm": 7.665630790046399,
      "learning_rate": 4.916189484488451e-06,
      "logits/chosen": -3.664515256881714,
      "logits/rejected": -3.7281200885772705,
      "logps/chosen": -18.788148880004883,
      "logps/rejected": -34.65311050415039,
      "loss": 0.2003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1718568801879883,
      "rewards/margins": 2.1718568801879883,
      "rewards/rejected": 0.0,
      "step": 500
    },
    {
      "epoch": 2.793296089385475,
      "eval_logits/chosen": -3.3867688179016113,
      "eval_logits/rejected": -3.4570679664611816,
      "eval_logps/chosen": -27.489871978759766,
      "eval_logps/rejected": -40.19453048706055,
      "eval_loss": 0.6971476674079895,
      "eval_rewards/accuracies": 0.574999988079071,
      "eval_rewards/chosen": 0.25801703333854675,
      "eval_rewards/margins": 0.25801703333854675,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5694,
      "eval_samples_per_second": 12.124,
      "eval_steps_per_second": 0.391,
      "step": 500
    },
    {
      "epoch": 2.798882681564246,
      "grad_norm": 16.082531356749712,
      "learning_rate": 4.915405162804668e-06,
      "logits/chosen": -3.4399499893188477,
      "logits/rejected": -3.576658010482788,
      "logps/chosen": -11.506025314331055,
      "logps/rejected": -44.461082458496094,
      "loss": 0.3649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8119820356369019,
      "rewards/margins": 0.8119820356369019,
      "rewards/rejected": 0.0,
      "step": 501
    },
    {
      "epoch": 2.804469273743017,
      "grad_norm": 10.589506200582308,
      "learning_rate": 4.914617251398655e-06,
      "logits/chosen": -3.6179075241088867,
      "logits/rejected": -3.4947290420532227,
      "logps/chosen": -13.945972442626953,
      "logps/rejected": -34.221435546875,
      "loss": 0.2853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0035812854766846,
      "rewards/margins": 1.0035812854766846,
      "rewards/rejected": 0.0,
      "step": 502
    },
    {
      "epoch": 2.810055865921788,
      "grad_norm": 8.573162133366749,
      "learning_rate": 4.913825751441386e-06,
      "logits/chosen": -3.6278727054595947,
      "logits/rejected": -3.113569736480713,
      "logps/chosen": -10.819999694824219,
      "logps/rejected": -72.68250274658203,
      "loss": 0.2146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.265613079071045,
      "rewards/margins": 1.265613079071045,
      "rewards/rejected": 0.0,
      "step": 503
    },
    {
      "epoch": 2.815642458100559,
      "grad_norm": 7.380108948417155,
      "learning_rate": 4.913030664109171e-06,
      "logits/chosen": -3.16776442527771,
      "logits/rejected": -3.3179080486297607,
      "logps/chosen": -39.84401321411133,
      "logps/rejected": -36.29817199707031,
      "loss": 0.2028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3594346046447754,
      "rewards/margins": 2.3594346046447754,
      "rewards/rejected": 0.0,
      "step": 504
    },
    {
      "epoch": 2.82122905027933,
      "grad_norm": 12.61997271976118,
      "learning_rate": 4.912231990583653e-06,
      "logits/chosen": -3.664263963699341,
      "logits/rejected": -3.611185073852539,
      "logps/chosen": -7.35392951965332,
      "logps/rejected": -23.90955924987793,
      "loss": 0.2698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3570897579193115,
      "rewards/margins": 1.3570897579193115,
      "rewards/rejected": 0.0,
      "step": 505
    },
    {
      "epoch": 2.826815642458101,
      "grad_norm": 10.373729573716252,
      "learning_rate": 4.911429732051801e-06,
      "logits/chosen": -3.7463676929473877,
      "logits/rejected": -3.8650095462799072,
      "logps/chosen": -10.482114791870117,
      "logps/rejected": -33.49766540527344,
      "loss": 0.2423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3939738273620605,
      "rewards/margins": 1.3939738273620605,
      "rewards/rejected": 0.0,
      "step": 506
    },
    {
      "epoch": 2.8324022346368714,
      "grad_norm": 12.854915024228385,
      "learning_rate": 4.910623889705915e-06,
      "logits/chosen": -3.5927188396453857,
      "logits/rejected": -3.5549471378326416,
      "logps/chosen": -15.822578430175781,
      "logps/rejected": -43.65298843383789,
      "loss": 0.3002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9550178647041321,
      "rewards/margins": 0.9550178647041321,
      "rewards/rejected": 0.0,
      "step": 507
    },
    {
      "epoch": 2.8379888268156424,
      "grad_norm": 8.048029311730383,
      "learning_rate": 4.909814464743621e-06,
      "logits/chosen": -3.838860034942627,
      "logits/rejected": -3.6069447994232178,
      "logps/chosen": -20.455013275146484,
      "logps/rejected": -35.35968780517578,
      "loss": 0.2172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6534394025802612,
      "rewards/margins": 1.6534394025802612,
      "rewards/rejected": 0.0,
      "step": 508
    },
    {
      "epoch": 2.8435754189944134,
      "grad_norm": 8.295368223159954,
      "learning_rate": 4.909001458367867e-06,
      "logits/chosen": -3.8993685245513916,
      "logits/rejected": -3.7712860107421875,
      "logps/chosen": -10.490028381347656,
      "logps/rejected": -40.54615783691406,
      "loss": 0.2043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2777233123779297,
      "rewards/margins": 1.2777233123779297,
      "rewards/rejected": 0.0,
      "step": 509
    },
    {
      "epoch": 2.8491620111731844,
      "grad_norm": 9.142628735699985,
      "learning_rate": 4.908184871786927e-06,
      "logits/chosen": -3.327820301055908,
      "logits/rejected": -3.30405330657959,
      "logps/chosen": -37.276100158691406,
      "logps/rejected": -26.274696350097656,
      "loss": 0.2363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.18687105178833,
      "rewards/margins": 2.18687105178833,
      "rewards/rejected": 0.0,
      "step": 510
    },
    {
      "epoch": 2.8547486033519553,
      "grad_norm": 9.478609861905765,
      "learning_rate": 4.907364706214393e-06,
      "logits/chosen": -3.3292946815490723,
      "logits/rejected": -3.3119490146636963,
      "logps/chosen": -20.771568298339844,
      "logps/rejected": -39.83605194091797,
      "loss": 0.213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3654065132141113,
      "rewards/margins": 1.3654065132141113,
      "rewards/rejected": 0.0,
      "step": 511
    },
    {
      "epoch": 2.8603351955307263,
      "grad_norm": 9.384658294157877,
      "learning_rate": 4.9065409628691774e-06,
      "logits/chosen": -3.6397030353546143,
      "logits/rejected": -3.5852835178375244,
      "logps/chosen": -16.56783103942871,
      "logps/rejected": -33.01799011230469,
      "loss": 0.2315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3842573165893555,
      "rewards/margins": 2.3842573165893555,
      "rewards/rejected": 0.0,
      "step": 512
    },
    {
      "epoch": 2.8659217877094973,
      "grad_norm": 10.49869854705092,
      "learning_rate": 4.905713642975509e-06,
      "logits/chosen": -3.4858903884887695,
      "logits/rejected": -3.3324131965637207,
      "logps/chosen": -22.517074584960938,
      "logps/rejected": -33.249755859375,
      "loss": 0.2434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.005667209625244,
      "rewards/margins": 2.005667209625244,
      "rewards/rejected": 0.0,
      "step": 513
    },
    {
      "epoch": 2.871508379888268,
      "grad_norm": 6.1302693782711355,
      "learning_rate": 4.904882747762933e-06,
      "logits/chosen": -3.695497512817383,
      "logits/rejected": -3.416656255722046,
      "logps/chosen": -19.944780349731445,
      "logps/rejected": -65.5020523071289,
      "loss": 0.2,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2357497215270996,
      "rewards/margins": 2.2357497215270996,
      "rewards/rejected": 0.0,
      "step": 514
    },
    {
      "epoch": 2.877094972067039,
      "grad_norm": 14.341203192239014,
      "learning_rate": 4.9040482784663086e-06,
      "logits/chosen": -3.2222533226013184,
      "logits/rejected": -3.410007953643799,
      "logps/chosen": -16.77880096435547,
      "logps/rejected": -39.7156982421875,
      "loss": 0.2461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0438480377197266,
      "rewards/margins": 1.0438480377197266,
      "rewards/rejected": 0.0,
      "step": 515
    },
    {
      "epoch": 2.88268156424581,
      "grad_norm": 13.7042523871091,
      "learning_rate": 4.903210236325804e-06,
      "logits/chosen": -3.348184108734131,
      "logits/rejected": -3.28187894821167,
      "logps/chosen": -22.75292205810547,
      "logps/rejected": -40.93385314941406,
      "loss": 0.3378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.907666802406311,
      "rewards/margins": 1.907666802406311,
      "rewards/rejected": 0.0,
      "step": 516
    },
    {
      "epoch": 2.888268156424581,
      "grad_norm": 5.470930286512879,
      "learning_rate": 4.9023686225869e-06,
      "logits/chosen": -3.5765676498413086,
      "logits/rejected": -3.6867291927337646,
      "logps/chosen": -31.1104736328125,
      "logps/rejected": -31.68254280090332,
      "loss": 0.164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9578039646148682,
      "rewards/margins": 1.9578039646148682,
      "rewards/rejected": 0.0,
      "step": 517
    },
    {
      "epoch": 2.893854748603352,
      "grad_norm": 6.782103662613166,
      "learning_rate": 4.901523438500385e-06,
      "logits/chosen": -3.5101301670074463,
      "logits/rejected": -3.319049119949341,
      "logps/chosen": -33.24108123779297,
      "logps/rejected": -35.92305374145508,
      "loss": 0.1516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4425456523895264,
      "rewards/margins": 3.4425456523895264,
      "rewards/rejected": 0.0,
      "step": 518
    },
    {
      "epoch": 2.899441340782123,
      "grad_norm": 14.786214299449432,
      "learning_rate": 4.900674685322351e-06,
      "logits/chosen": -3.3603501319885254,
      "logits/rejected": -3.3649325370788574,
      "logps/chosen": -9.142024993896484,
      "logps/rejected": -53.43362808227539,
      "loss": 0.3005,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5436765551567078,
      "rewards/margins": 0.5436765551567078,
      "rewards/rejected": 0.0,
      "step": 519
    },
    {
      "epoch": 2.905027932960894,
      "grad_norm": 8.797237988477718,
      "learning_rate": 4.8998223643141996e-06,
      "logits/chosen": -3.501187324523926,
      "logits/rejected": -3.3236680030822754,
      "logps/chosen": -14.158652305603027,
      "logps/rejected": -22.43594741821289,
      "loss": 0.2242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.78994220495224,
      "rewards/margins": 0.78994220495224,
      "rewards/rejected": 0.0,
      "step": 520
    },
    {
      "epoch": 2.905027932960894,
      "eval_logits/chosen": -3.43141508102417,
      "eval_logits/rejected": -3.497451066970825,
      "eval_logps/chosen": -25.670452117919922,
      "eval_logps/rejected": -38.723793029785156,
      "eval_loss": 0.669161319732666,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": 0.43995895981788635,
      "eval_rewards/margins": 0.43995895981788635,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.635,
      "eval_samples_per_second": 12.093,
      "eval_steps_per_second": 0.39,
      "step": 520
    },
    {
      "epoch": 2.910614525139665,
      "grad_norm": 9.05558622183522,
      "learning_rate": 4.8989664767426285e-06,
      "logits/chosen": -3.8894541263580322,
      "logits/rejected": -3.7251250743865967,
      "logps/chosen": -19.159496307373047,
      "logps/rejected": -67.04489135742188,
      "loss": 0.1825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.8530590534210205,
      "rewards/margins": 1.8530590534210205,
      "rewards/rejected": 0.0,
      "step": 521
    },
    {
      "epoch": 2.916201117318436,
      "grad_norm": 11.942054922143942,
      "learning_rate": 4.8981070238796415e-06,
      "logits/chosen": -3.6824591159820557,
      "logits/rejected": -3.523838996887207,
      "logps/chosen": -25.197954177856445,
      "logps/rejected": -23.272476196289062,
      "loss": 0.2501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.16544771194458,
      "rewards/margins": 2.16544771194458,
      "rewards/rejected": 0.0,
      "step": 522
    },
    {
      "epoch": 2.921787709497207,
      "grad_norm": 11.155556011645738,
      "learning_rate": 4.897244007002536e-06,
      "logits/chosen": -3.624246835708618,
      "logits/rejected": -3.213735342025757,
      "logps/chosen": -21.01001739501953,
      "logps/rejected": -35.76752853393555,
      "loss": 0.2284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0376659631729126,
      "rewards/margins": 1.0376659631729126,
      "rewards/rejected": 0.0,
      "step": 523
    },
    {
      "epoch": 2.927374301675978,
      "grad_norm": 14.53461296385414,
      "learning_rate": 4.896377427393912e-06,
      "logits/chosen": -3.5456905364990234,
      "logits/rejected": -3.430098056793213,
      "logps/chosen": -24.2993106842041,
      "logps/rejected": -43.080299377441406,
      "loss": 0.3382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7260308265686035,
      "rewards/margins": 1.7260308265686035,
      "rewards/rejected": 0.0,
      "step": 524
    },
    {
      "epoch": 2.9329608938547485,
      "grad_norm": 12.043909533619347,
      "learning_rate": 4.895507286341659e-06,
      "logits/chosen": -3.7056233882904053,
      "logits/rejected": -3.094269037246704,
      "logps/chosen": -13.89625358581543,
      "logps/rejected": -47.47608184814453,
      "loss": 0.313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5578198432922363,
      "rewards/margins": 1.5578198432922363,
      "rewards/rejected": 0.0,
      "step": 525
    },
    {
      "epoch": 2.9385474860335195,
      "grad_norm": 6.055127350367941,
      "learning_rate": 4.894633585138961e-06,
      "logits/chosen": -3.705443859100342,
      "logits/rejected": -3.491806745529175,
      "logps/chosen": -13.456048011779785,
      "logps/rejected": -43.296539306640625,
      "loss": 0.191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7328287363052368,
      "rewards/margins": 1.7328287363052368,
      "rewards/rejected": 0.0,
      "step": 526
    },
    {
      "epoch": 2.9441340782122905,
      "grad_norm": 7.932007369593178,
      "learning_rate": 4.893756325084295e-06,
      "logits/chosen": -3.6564714908599854,
      "logits/rejected": -3.4105539321899414,
      "logps/chosen": -21.92789077758789,
      "logps/rejected": -45.58838653564453,
      "loss": 0.18,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.057879686355591,
      "rewards/margins": 2.057879686355591,
      "rewards/rejected": 0.0,
      "step": 527
    },
    {
      "epoch": 2.9497206703910615,
      "grad_norm": 12.05864489780205,
      "learning_rate": 4.892875507481426e-06,
      "logits/chosen": -3.8203251361846924,
      "logits/rejected": -3.787804365158081,
      "logps/chosen": -8.583724975585938,
      "logps/rejected": -33.650081634521484,
      "loss": 0.2415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6776840686798096,
      "rewards/margins": 1.6776840686798096,
      "rewards/rejected": 0.0,
      "step": 528
    },
    {
      "epoch": 2.9553072625698324,
      "grad_norm": 9.252766472462271,
      "learning_rate": 4.891991133639405e-06,
      "logits/chosen": -3.6949799060821533,
      "logits/rejected": -3.682075023651123,
      "logps/chosen": -22.265247344970703,
      "logps/rejected": -34.934349060058594,
      "loss": 0.2165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7904510498046875,
      "rewards/margins": 1.7904510498046875,
      "rewards/rejected": 0.0,
      "step": 529
    },
    {
      "epoch": 2.9608938547486034,
      "grad_norm": 13.799638861918032,
      "learning_rate": 4.8911032048725674e-06,
      "logits/chosen": -3.5691003799438477,
      "logits/rejected": -3.2430078983306885,
      "logps/chosen": -10.437606811523438,
      "logps/rejected": -44.936790466308594,
      "loss": 0.324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8133826851844788,
      "rewards/margins": 0.8133826851844788,
      "rewards/rejected": 0.0,
      "step": 530
    },
    {
      "epoch": 2.9664804469273744,
      "grad_norm": 9.663487677140589,
      "learning_rate": 4.8902117225005365e-06,
      "logits/chosen": -3.370802640914917,
      "logits/rejected": -3.381099224090576,
      "logps/chosen": -38.221519470214844,
      "logps/rejected": -56.81721878051758,
      "loss": 0.2504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8368101119995117,
      "rewards/margins": 2.8368101119995117,
      "rewards/rejected": 0.0,
      "step": 531
    },
    {
      "epoch": 2.972067039106145,
      "grad_norm": 10.727493068991237,
      "learning_rate": 4.889316687848212e-06,
      "logits/chosen": -3.592973470687866,
      "logits/rejected": -3.557828426361084,
      "logps/chosen": -11.489707946777344,
      "logps/rejected": -50.816917419433594,
      "loss": 0.2424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1928131580352783,
      "rewards/margins": 2.1928131580352783,
      "rewards/rejected": 0.0,
      "step": 532
    },
    {
      "epoch": 2.977653631284916,
      "grad_norm": 7.93787429567587,
      "learning_rate": 4.888418102245776e-06,
      "logits/chosen": -3.457979917526245,
      "logits/rejected": -3.5648603439331055,
      "logps/chosen": -21.949750900268555,
      "logps/rejected": -44.56542205810547,
      "loss": 0.2421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8180768489837646,
      "rewards/margins": 1.8180768489837646,
      "rewards/rejected": 0.0,
      "step": 533
    },
    {
      "epoch": 2.983240223463687,
      "grad_norm": 11.01715763123628,
      "learning_rate": 4.887515967028687e-06,
      "logits/chosen": -3.2931697368621826,
      "logits/rejected": -3.5548505783081055,
      "logps/chosen": -13.819613456726074,
      "logps/rejected": -36.72962188720703,
      "loss": 0.1947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5560989379882812,
      "rewards/margins": 1.5560989379882812,
      "rewards/rejected": 0.0,
      "step": 534
    },
    {
      "epoch": 2.988826815642458,
      "grad_norm": 7.126689381056586,
      "learning_rate": 4.886610283537676e-06,
      "logits/chosen": -3.181040048599243,
      "logits/rejected": -3.1639270782470703,
      "logps/chosen": -16.32206916809082,
      "logps/rejected": -42.69291305541992,
      "loss": 0.1859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4714102745056152,
      "rewards/margins": 2.4714102745056152,
      "rewards/rejected": 0.0,
      "step": 535
    },
    {
      "epoch": 2.994413407821229,
      "grad_norm": 11.979035718399677,
      "learning_rate": 4.885701053118752e-06,
      "logits/chosen": -3.5208916664123535,
      "logits/rejected": -3.524162769317627,
      "logps/chosen": -13.967312812805176,
      "logps/rejected": -50.641136169433594,
      "loss": 0.2757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5675158500671387,
      "rewards/margins": 1.5675158500671387,
      "rewards/rejected": 0.0,
      "step": 536
    },
    {
      "epoch": 3.0,
      "grad_norm": 13.934885870226699,
      "learning_rate": 4.884788277123192e-06,
      "logits/chosen": -3.3993563652038574,
      "logits/rejected": -3.3354499340057373,
      "logps/chosen": -21.883188247680664,
      "logps/rejected": -32.13275909423828,
      "loss": 0.3596,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.6513726115226746,
      "rewards/margins": 0.6513726115226746,
      "rewards/rejected": 0.0,
      "step": 537
    },
    {
      "epoch": 3.005586592178771,
      "grad_norm": 9.468381205977476,
      "learning_rate": 4.883871956907547e-06,
      "logits/chosen": -3.5253610610961914,
      "logits/rejected": -3.678168773651123,
      "logps/chosen": -5.820188999176025,
      "logps/rejected": -40.933650970458984,
      "loss": 0.2544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9378538131713867,
      "rewards/margins": 1.9378538131713867,
      "rewards/rejected": 0.0,
      "step": 538
    },
    {
      "epoch": 3.011173184357542,
      "grad_norm": 7.712722403184457,
      "learning_rate": 4.882952093833628e-06,
      "logits/chosen": -3.5587193965911865,
      "logits/rejected": -3.171604871749878,
      "logps/chosen": -10.045610427856445,
      "logps/rejected": -46.975982666015625,
      "loss": 0.2286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0477114915847778,
      "rewards/margins": 1.0477114915847778,
      "rewards/rejected": 0.0,
      "step": 539
    },
    {
      "epoch": 3.016759776536313,
      "grad_norm": 7.355908173587549,
      "learning_rate": 4.882028689268517e-06,
      "logits/chosen": -3.187481164932251,
      "logits/rejected": -3.1481637954711914,
      "logps/chosen": -27.207073211669922,
      "logps/rejected": -55.9260368347168,
      "loss": 0.21,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7007155418395996,
      "rewards/margins": 3.7007155418395996,
      "rewards/rejected": 0.0,
      "step": 540
    },
    {
      "epoch": 3.016759776536313,
      "eval_logits/chosen": -3.410640001296997,
      "eval_logits/rejected": -3.4756932258605957,
      "eval_logps/chosen": -26.39898109436035,
      "eval_logps/rejected": -38.23013687133789,
      "eval_loss": 0.6707049012184143,
      "eval_rewards/accuracies": 0.699999988079071,
      "eval_rewards/chosen": 0.36710605025291443,
      "eval_rewards/margins": 0.36710605025291443,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5192,
      "eval_samples_per_second": 12.148,
      "eval_steps_per_second": 0.392,
      "step": 540
    },
    {
      "epoch": 3.022346368715084,
      "grad_norm": 6.247681991184702,
      "learning_rate": 4.881101744584558e-06,
      "logits/chosen": -3.759469985961914,
      "logits/rejected": -3.646568775177002,
      "logps/chosen": -10.333898544311523,
      "logps/rejected": -32.87458038330078,
      "loss": 0.178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8493003845214844,
      "rewards/margins": 1.8493003845214844,
      "rewards/rejected": 0.0,
      "step": 541
    },
    {
      "epoch": 3.0279329608938546,
      "grad_norm": 4.332177593420507,
      "learning_rate": 4.880171261159356e-06,
      "logits/chosen": -3.7060694694519043,
      "logits/rejected": -3.787137508392334,
      "logps/chosen": -12.449996948242188,
      "logps/rejected": -42.495784759521484,
      "loss": 0.1471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.931723117828369,
      "rewards/margins": 2.931723117828369,
      "rewards/rejected": 0.0,
      "step": 542
    },
    {
      "epoch": 3.0335195530726256,
      "grad_norm": 11.529788898992017,
      "learning_rate": 4.879237240375774e-06,
      "logits/chosen": -3.8649415969848633,
      "logits/rejected": -3.760371208190918,
      "logps/chosen": -7.29527473449707,
      "logps/rejected": -47.72666931152344,
      "loss": 0.234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.575704574584961,
      "rewards/margins": 2.575704574584961,
      "rewards/rejected": 0.0,
      "step": 543
    },
    {
      "epoch": 3.0391061452513966,
      "grad_norm": 8.640982034482404,
      "learning_rate": 4.878299683621934e-06,
      "logits/chosen": -3.7257440090179443,
      "logits/rejected": -3.4540822505950928,
      "logps/chosen": -9.748004913330078,
      "logps/rejected": -47.148460388183594,
      "loss": 0.2156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.614722728729248,
      "rewards/margins": 2.614722728729248,
      "rewards/rejected": 0.0,
      "step": 544
    },
    {
      "epoch": 3.0446927374301676,
      "grad_norm": 7.2780987898379905,
      "learning_rate": 4.877358592291212e-06,
      "logits/chosen": -3.209075927734375,
      "logits/rejected": -3.2220041751861572,
      "logps/chosen": -25.274744033813477,
      "logps/rejected": -55.720890045166016,
      "loss": 0.1982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.39582896232605,
      "rewards/margins": 3.39582896232605,
      "rewards/rejected": 0.0,
      "step": 545
    },
    {
      "epoch": 3.0502793296089385,
      "grad_norm": 5.880995783840338,
      "learning_rate": 4.8764139677822374e-06,
      "logits/chosen": -3.795177698135376,
      "logits/rejected": -3.7467200756073,
      "logps/chosen": -27.91313934326172,
      "logps/rejected": -42.81288146972656,
      "loss": 0.1838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.1776957511901855,
      "rewards/margins": 2.1776957511901855,
      "rewards/rejected": 0.0,
      "step": 546
    },
    {
      "epoch": 3.0558659217877095,
      "grad_norm": 4.814124126894078,
      "learning_rate": 4.8754658114988905e-06,
      "logits/chosen": -3.7405507564544678,
      "logits/rejected": -3.5936484336853027,
      "logps/chosen": -23.008102416992188,
      "logps/rejected": -44.486732482910156,
      "loss": 0.1492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6368730068206787,
      "rewards/margins": 3.6368730068206787,
      "rewards/rejected": 0.0,
      "step": 547
    },
    {
      "epoch": 3.0614525139664805,
      "grad_norm": 4.17479401136516,
      "learning_rate": 4.8745141248503005e-06,
      "logits/chosen": -3.5337929725646973,
      "logits/rejected": -3.536234140396118,
      "logps/chosen": -7.369060516357422,
      "logps/rejected": -31.6191349029541,
      "loss": 0.1284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9985876083374023,
      "rewards/margins": 2.9985876083374023,
      "rewards/rejected": 0.0,
      "step": 548
    },
    {
      "epoch": 3.0670391061452515,
      "grad_norm": 10.922397729098504,
      "learning_rate": 4.873558909250844e-06,
      "logits/chosen": -3.6212995052337646,
      "logits/rejected": -3.61673903465271,
      "logps/chosen": -8.66882038116455,
      "logps/rejected": -29.65574073791504,
      "loss": 0.2217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.23750901222229,
      "rewards/margins": 1.23750901222229,
      "rewards/rejected": 0.0,
      "step": 549
    },
    {
      "epoch": 3.0726256983240225,
      "grad_norm": 6.118737075683679,
      "learning_rate": 4.8726001661201395e-06,
      "logits/chosen": -3.5655899047851562,
      "logits/rejected": -3.576315402984619,
      "logps/chosen": -11.216480255126953,
      "logps/rejected": -50.851837158203125,
      "loss": 0.175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4009108543395996,
      "rewards/margins": 2.4009108543395996,
      "rewards/rejected": 0.0,
      "step": 550
    },
    {
      "epoch": 3.078212290502793,
      "grad_norm": 5.454692537773599,
      "learning_rate": 4.871637896883052e-06,
      "logits/chosen": -4.037452220916748,
      "logits/rejected": -3.8560588359832764,
      "logps/chosen": -21.41240882873535,
      "logps/rejected": -45.8934326171875,
      "loss": 0.1853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.223761558532715,
      "rewards/margins": 2.223761558532715,
      "rewards/rejected": 0.0,
      "step": 551
    },
    {
      "epoch": 3.083798882681564,
      "grad_norm": 4.415191827437318,
      "learning_rate": 4.870672102969683e-06,
      "logits/chosen": -3.680630922317505,
      "logits/rejected": -3.2984368801116943,
      "logps/chosen": -7.039037704467773,
      "logps/rejected": -50.92243957519531,
      "loss": 0.1445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5671565532684326,
      "rewards/margins": 1.5671565532684326,
      "rewards/rejected": 0.0,
      "step": 552
    },
    {
      "epoch": 3.089385474860335,
      "grad_norm": 5.940521552364067,
      "learning_rate": 4.869702785815376e-06,
      "logits/chosen": -3.9007019996643066,
      "logits/rejected": -3.92747163772583,
      "logps/chosen": -9.258013725280762,
      "logps/rejected": -38.32275390625,
      "loss": 0.184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2246570587158203,
      "rewards/margins": 2.2246570587158203,
      "rewards/rejected": 0.0,
      "step": 553
    },
    {
      "epoch": 3.094972067039106,
      "grad_norm": 5.196175496218633,
      "learning_rate": 4.868729946860709e-06,
      "logits/chosen": -3.7878668308258057,
      "logits/rejected": -3.736729621887207,
      "logps/chosen": -4.276645660400391,
      "logps/rejected": -57.40045166015625,
      "loss": 0.1607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2027775049209595,
      "rewards/margins": 1.2027775049209595,
      "rewards/rejected": 0.0,
      "step": 554
    },
    {
      "epoch": 3.100558659217877,
      "grad_norm": 15.813575787601515,
      "learning_rate": 4.867753587551492e-06,
      "logits/chosen": -3.695995807647705,
      "logits/rejected": -3.5249111652374268,
      "logps/chosen": -4.9186601638793945,
      "logps/rejected": -59.02560043334961,
      "loss": 0.2931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.681835412979126,
      "rewards/margins": 1.681835412979126,
      "rewards/rejected": 0.0,
      "step": 555
    },
    {
      "epoch": 3.106145251396648,
      "grad_norm": 15.729690145059177,
      "learning_rate": 4.86677370933877e-06,
      "logits/chosen": -3.731395721435547,
      "logits/rejected": -3.90791392326355,
      "logps/chosen": -6.823447227478027,
      "logps/rejected": -80.85296630859375,
      "loss": 0.2873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.208854913711548,
      "rewards/margins": 2.208854913711548,
      "rewards/rejected": 0.0,
      "step": 556
    },
    {
      "epoch": 3.111731843575419,
      "grad_norm": 11.100901825569622,
      "learning_rate": 4.865790313678817e-06,
      "logits/chosen": -3.670299530029297,
      "logits/rejected": -3.3062636852264404,
      "logps/chosen": -4.103579521179199,
      "logps/rejected": -64.87287139892578,
      "loss": 0.2458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9264636039733887,
      "rewards/margins": 1.9264636039733887,
      "rewards/rejected": 0.0,
      "step": 557
    },
    {
      "epoch": 3.11731843575419,
      "grad_norm": 10.704228274759895,
      "learning_rate": 4.864803402033135e-06,
      "logits/chosen": -3.923351764678955,
      "logits/rejected": -3.8535244464874268,
      "logps/chosen": -5.866227149963379,
      "logps/rejected": -76.19624328613281,
      "loss": 0.2343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6085448265075684,
      "rewards/margins": 1.6085448265075684,
      "rewards/rejected": 0.0,
      "step": 558
    },
    {
      "epoch": 3.122905027932961,
      "grad_norm": 7.4821647795426784,
      "learning_rate": 4.863812975868447e-06,
      "logits/chosen": -3.5024571418762207,
      "logits/rejected": -3.6949100494384766,
      "logps/chosen": -2.2576706409454346,
      "logps/rejected": -51.040218353271484,
      "loss": 0.1727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3602182865142822,
      "rewards/margins": 1.3602182865142822,
      "rewards/rejected": 0.0,
      "step": 559
    },
    {
      "epoch": 3.1284916201117317,
      "grad_norm": 10.408096994165046,
      "learning_rate": 4.862819036656707e-06,
      "logits/chosen": -2.7580952644348145,
      "logits/rejected": -2.7123942375183105,
      "logps/chosen": -19.8362979888916,
      "logps/rejected": -46.12969207763672,
      "loss": 0.2307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6844029426574707,
      "rewards/margins": 2.6844029426574707,
      "rewards/rejected": 0.0,
      "step": 560
    },
    {
      "epoch": 3.1284916201117317,
      "eval_logits/chosen": -3.4141929149627686,
      "eval_logits/rejected": -3.5038647651672363,
      "eval_logps/chosen": -28.728002548217773,
      "eval_logps/rejected": -42.361228942871094,
      "eval_loss": 0.83282470703125,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": 0.1342039406299591,
      "eval_rewards/margins": 0.1342039406299591,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6106,
      "eval_samples_per_second": 12.104,
      "eval_steps_per_second": 0.39,
      "step": 560
    },
    {
      "epoch": 3.1340782122905027,
      "grad_norm": 6.841814206992419,
      "learning_rate": 4.861821585875082e-06,
      "logits/chosen": -3.457611560821533,
      "logits/rejected": -3.2773709297180176,
      "logps/chosen": -7.321425437927246,
      "logps/rejected": -46.62171173095703,
      "loss": 0.2256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9120540022850037,
      "rewards/margins": 0.9120540022850037,
      "rewards/rejected": 0.0,
      "step": 561
    },
    {
      "epoch": 3.1396648044692737,
      "grad_norm": 6.278585859615893,
      "learning_rate": 4.860820625005963e-06,
      "logits/chosen": -3.414132833480835,
      "logits/rejected": -3.559236526489258,
      "logps/chosen": -20.34001350402832,
      "logps/rejected": -36.212303161621094,
      "loss": 0.1742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3784234523773193,
      "rewards/margins": 2.3784234523773193,
      "rewards/rejected": 0.0,
      "step": 562
    },
    {
      "epoch": 3.1452513966480447,
      "grad_norm": 5.518580512338845,
      "learning_rate": 4.859816155536956e-06,
      "logits/chosen": -3.2832934856414795,
      "logits/rejected": -3.407667398452759,
      "logps/chosen": -6.2693891525268555,
      "logps/rejected": -37.39242935180664,
      "loss": 0.1915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.607302188873291,
      "rewards/margins": 1.607302188873291,
      "rewards/rejected": 0.0,
      "step": 563
    },
    {
      "epoch": 3.1508379888268156,
      "grad_norm": 7.264748978225968,
      "learning_rate": 4.858808178960882e-06,
      "logits/chosen": -3.4719502925872803,
      "logits/rejected": -3.3588247299194336,
      "logps/chosen": -10.092350959777832,
      "logps/rejected": -31.526832580566406,
      "loss": 0.233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4393583536148071,
      "rewards/margins": 1.4393583536148071,
      "rewards/rejected": 0.0,
      "step": 564
    },
    {
      "epoch": 3.1564245810055866,
      "grad_norm": 3.5688723033632286,
      "learning_rate": 4.857796696775773e-06,
      "logits/chosen": -3.6764042377471924,
      "logits/rejected": -3.795382022857666,
      "logps/chosen": -4.672013759613037,
      "logps/rejected": -47.67449951171875,
      "loss": 0.1294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.571660041809082,
      "rewards/margins": 1.571660041809082,
      "rewards/rejected": 0.0,
      "step": 565
    },
    {
      "epoch": 3.1620111731843576,
      "grad_norm": 3.51165910576415,
      "learning_rate": 4.856781710484873e-06,
      "logits/chosen": -3.549619436264038,
      "logits/rejected": -3.638277292251587,
      "logps/chosen": -5.066268444061279,
      "logps/rejected": -63.28614807128906,
      "loss": 0.1356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1747734546661377,
      "rewards/margins": 2.1747734546661377,
      "rewards/rejected": 0.0,
      "step": 566
    },
    {
      "epoch": 3.1675977653631286,
      "grad_norm": 5.226012568428419,
      "learning_rate": 4.85576322159663e-06,
      "logits/chosen": -3.636514663696289,
      "logits/rejected": -3.7578952312469482,
      "logps/chosen": -9.338247299194336,
      "logps/rejected": -34.957210540771484,
      "loss": 0.1428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.500556707382202,
      "rewards/margins": 2.500556707382202,
      "rewards/rejected": 0.0,
      "step": 567
    },
    {
      "epoch": 3.1731843575418996,
      "grad_norm": 6.164267424989158,
      "learning_rate": 4.854741231624702e-06,
      "logits/chosen": -3.9002723693847656,
      "logits/rejected": -3.899473190307617,
      "logps/chosen": -3.9378864765167236,
      "logps/rejected": -26.161727905273438,
      "loss": 0.1846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.419400930404663,
      "rewards/margins": 1.419400930404663,
      "rewards/rejected": 0.0,
      "step": 568
    },
    {
      "epoch": 3.17877094972067,
      "grad_norm": 6.106931287419482,
      "learning_rate": 4.853715742087947e-06,
      "logits/chosen": -3.376265287399292,
      "logits/rejected": -3.3810982704162598,
      "logps/chosen": -24.41690444946289,
      "logps/rejected": -40.49224090576172,
      "loss": 0.1585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3253402709960938,
      "rewards/margins": 2.3253402709960938,
      "rewards/rejected": 0.0,
      "step": 569
    },
    {
      "epoch": 3.184357541899441,
      "grad_norm": 10.011606180121923,
      "learning_rate": 4.852686754510425e-06,
      "logits/chosen": -3.6312410831451416,
      "logits/rejected": -3.2106258869171143,
      "logps/chosen": -2.5288639068603516,
      "logps/rejected": -65.45777130126953,
      "loss": 0.2066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8193674087524414,
      "rewards/margins": 1.8193674087524414,
      "rewards/rejected": 0.0,
      "step": 570
    },
    {
      "epoch": 3.189944134078212,
      "grad_norm": 7.552977812922125,
      "learning_rate": 4.851654270421396e-06,
      "logits/chosen": -3.7082202434539795,
      "logits/rejected": -3.579951524734497,
      "logps/chosen": -22.686237335205078,
      "logps/rejected": -37.06023025512695,
      "loss": 0.1915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.207521915435791,
      "rewards/margins": 2.207521915435791,
      "rewards/rejected": 0.0,
      "step": 571
    },
    {
      "epoch": 3.195530726256983,
      "grad_norm": 4.996051274460295,
      "learning_rate": 4.850618291355314e-06,
      "logits/chosen": -3.369415283203125,
      "logits/rejected": -3.662245750427246,
      "logps/chosen": -20.927631378173828,
      "logps/rejected": -47.603271484375,
      "loss": 0.1462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9784538745880127,
      "rewards/margins": 2.9784538745880127,
      "rewards/rejected": 0.0,
      "step": 572
    },
    {
      "epoch": 3.201117318435754,
      "grad_norm": 6.717020926474769,
      "learning_rate": 4.849578818851829e-06,
      "logits/chosen": -3.8511791229248047,
      "logits/rejected": -3.7243635654449463,
      "logps/chosen": -11.984662055969238,
      "logps/rejected": -53.98311233520508,
      "loss": 0.1537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.131770133972168,
      "rewards/margins": 1.131770133972168,
      "rewards/rejected": 0.0,
      "step": 573
    },
    {
      "epoch": 3.206703910614525,
      "grad_norm": 10.79389328997401,
      "learning_rate": 4.848535854455783e-06,
      "logits/chosen": -3.6315724849700928,
      "logits/rejected": -3.6444904804229736,
      "logps/chosen": -4.472777366638184,
      "logps/rejected": -39.4677734375,
      "loss": 0.2185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9192560911178589,
      "rewards/margins": 1.9192560911178589,
      "rewards/rejected": 0.0,
      "step": 574
    },
    {
      "epoch": 3.212290502793296,
      "grad_norm": 8.993504819532456,
      "learning_rate": 4.847489399717205e-06,
      "logits/chosen": -3.4644453525543213,
      "logits/rejected": -3.4123144149780273,
      "logps/chosen": -5.036550521850586,
      "logps/rejected": -45.17393112182617,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1105345487594604,
      "rewards/margins": 1.1105345487594604,
      "rewards/rejected": 0.0,
      "step": 575
    },
    {
      "epoch": 3.217877094972067,
      "grad_norm": 6.851650117718826,
      "learning_rate": 4.846439456191313e-06,
      "logits/chosen": -3.915717124938965,
      "logits/rejected": -3.915767192840576,
      "logps/chosen": -5.896653652191162,
      "logps/rejected": -27.38468360900879,
      "loss": 0.1746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5716564655303955,
      "rewards/margins": 1.5716564655303955,
      "rewards/rejected": 0.0,
      "step": 576
    },
    {
      "epoch": 3.223463687150838,
      "grad_norm": 6.3010583044500414,
      "learning_rate": 4.845386025438512e-06,
      "logits/chosen": -3.9139134883880615,
      "logits/rejected": -3.9300661087036133,
      "logps/chosen": -32.87173843383789,
      "logps/rejected": -32.0973014831543,
      "loss": 0.1903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.532440423965454,
      "rewards/margins": 3.532440423965454,
      "rewards/rejected": 0.0,
      "step": 577
    },
    {
      "epoch": 3.2290502793296088,
      "grad_norm": 5.723140144258453,
      "learning_rate": 4.844329109024387e-06,
      "logits/chosen": -3.30938458442688,
      "logits/rejected": -3.448523759841919,
      "logps/chosen": -33.807220458984375,
      "logps/rejected": -65.9272232055664,
      "loss": 0.1453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8323745727539062,
      "rewards/margins": 3.8323745727539062,
      "rewards/rejected": 0.0,
      "step": 578
    },
    {
      "epoch": 3.2346368715083798,
      "grad_norm": 10.55614665807878,
      "learning_rate": 4.843268708519704e-06,
      "logits/chosen": -3.6377575397491455,
      "logits/rejected": -3.769376754760742,
      "logps/chosen": -14.01274585723877,
      "logps/rejected": -53.496429443359375,
      "loss": 0.232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.331953763961792,
      "rewards/margins": 1.331953763961792,
      "rewards/rejected": 0.0,
      "step": 579
    },
    {
      "epoch": 3.2402234636871508,
      "grad_norm": 9.043415981199473,
      "learning_rate": 4.842204825500407e-06,
      "logits/chosen": -3.8999080657958984,
      "logits/rejected": -3.4530107975006104,
      "logps/chosen": -13.957342147827148,
      "logps/rejected": -56.34123229980469,
      "loss": 0.1974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5536835193634033,
      "rewards/margins": 1.5536835193634033,
      "rewards/rejected": 0.0,
      "step": 580
    },
    {
      "epoch": 3.2402234636871508,
      "eval_logits/chosen": -3.442417860031128,
      "eval_logits/rejected": -3.5201575756073,
      "eval_logps/chosen": -28.180187225341797,
      "eval_logps/rejected": -41.86280822753906,
      "eval_loss": 0.8160907030105591,
      "eval_rewards/accuracies": 0.6000000238418579,
      "eval_rewards/chosen": 0.1889854371547699,
      "eval_rewards/margins": 0.1889854371547699,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5123,
      "eval_samples_per_second": 12.151,
      "eval_steps_per_second": 0.392,
      "step": 580
    },
    {
      "epoch": 3.2458100558659218,
      "grad_norm": 10.588089800481137,
      "learning_rate": 4.841137461547615e-06,
      "logits/chosen": -3.2225306034088135,
      "logits/rejected": -3.376682996749878,
      "logps/chosen": -23.72100830078125,
      "logps/rejected": -40.83525466918945,
      "loss": 0.2901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5044362545013428,
      "rewards/margins": 3.5044362545013428,
      "rewards/rejected": 0.0,
      "step": 581
    },
    {
      "epoch": 3.2513966480446927,
      "grad_norm": 6.830042028017552,
      "learning_rate": 4.840066618247621e-06,
      "logits/chosen": -3.313023567199707,
      "logits/rejected": -3.5785162448883057,
      "logps/chosen": -6.095949172973633,
      "logps/rejected": -30.879350662231445,
      "loss": 0.2153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7924909591674805,
      "rewards/margins": 1.7924909591674805,
      "rewards/rejected": 0.0,
      "step": 582
    },
    {
      "epoch": 3.2569832402234637,
      "grad_norm": 6.220259756881848,
      "learning_rate": 4.8389922971918895e-06,
      "logits/chosen": -3.6942644119262695,
      "logits/rejected": -3.834214210510254,
      "logps/chosen": -11.204002380371094,
      "logps/rejected": -34.97370147705078,
      "loss": 0.1447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4118940830230713,
      "rewards/margins": 2.4118940830230713,
      "rewards/rejected": 0.0,
      "step": 583
    },
    {
      "epoch": 3.2625698324022347,
      "grad_norm": 9.370461569004458,
      "learning_rate": 4.8379144999770526e-06,
      "logits/chosen": -3.8331010341644287,
      "logits/rejected": -3.4110708236694336,
      "logps/chosen": -10.808813095092773,
      "logps/rejected": -38.93630599975586,
      "loss": 0.2668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5869708061218262,
      "rewards/margins": 1.5869708061218262,
      "rewards/rejected": 0.0,
      "step": 584
    },
    {
      "epoch": 3.2681564245810057,
      "grad_norm": 3.7522874704840796,
      "learning_rate": 4.836833228204909e-06,
      "logits/chosen": -3.307615280151367,
      "logits/rejected": -3.044975519180298,
      "logps/chosen": -20.079368591308594,
      "logps/rejected": -80.94166564941406,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8941289186477661,
      "rewards/margins": 1.8941289186477661,
      "rewards/rejected": 0.0,
      "step": 585
    },
    {
      "epoch": 3.2737430167597763,
      "grad_norm": 4.9431583372978976,
      "learning_rate": 4.835748483482422e-06,
      "logits/chosen": -3.5052108764648438,
      "logits/rejected": -3.4834671020507812,
      "logps/chosen": -20.137592315673828,
      "logps/rejected": -46.21788787841797,
      "loss": 0.1682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.930875539779663,
      "rewards/margins": 2.930875539779663,
      "rewards/rejected": 0.0,
      "step": 586
    },
    {
      "epoch": 3.2793296089385473,
      "grad_norm": 5.837756921396615,
      "learning_rate": 4.834660267421713e-06,
      "logits/chosen": -3.6790201663970947,
      "logits/rejected": -3.741736650466919,
      "logps/chosen": -20.636844635009766,
      "logps/rejected": -50.88914489746094,
      "loss": 0.1787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2214770317077637,
      "rewards/margins": 2.2214770317077637,
      "rewards/rejected": 0.0,
      "step": 587
    },
    {
      "epoch": 3.2849162011173183,
      "grad_norm": 8.935934671415875,
      "learning_rate": 4.833568581640066e-06,
      "logits/chosen": -3.5549521446228027,
      "logits/rejected": -3.4773905277252197,
      "logps/chosen": -17.166458129882812,
      "logps/rejected": -49.729042053222656,
      "loss": 0.1979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9430313110351562,
      "rewards/margins": 2.9430313110351562,
      "rewards/rejected": 0.0,
      "step": 588
    },
    {
      "epoch": 3.2905027932960893,
      "grad_norm": 13.11187098853704,
      "learning_rate": 4.8324734277599226e-06,
      "logits/chosen": -3.5677270889282227,
      "logits/rejected": -3.5250914096832275,
      "logps/chosen": -9.86683464050293,
      "logps/rejected": -46.90715789794922,
      "loss": 0.3013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8606154918670654,
      "rewards/margins": 1.8606154918670654,
      "rewards/rejected": 0.0,
      "step": 589
    },
    {
      "epoch": 3.2960893854748603,
      "grad_norm": 4.709845573086337,
      "learning_rate": 4.831374807408874e-06,
      "logits/chosen": -3.6192870140075684,
      "logits/rejected": -3.545923948287964,
      "logps/chosen": -13.351846694946289,
      "logps/rejected": -45.416603088378906,
      "loss": 0.1658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0968503952026367,
      "rewards/margins": 2.0968503952026367,
      "rewards/rejected": 0.0,
      "step": 590
    },
    {
      "epoch": 3.3016759776536313,
      "grad_norm": 7.656973517054263,
      "learning_rate": 4.830272722219666e-06,
      "logits/chosen": -3.650534152984619,
      "logits/rejected": -3.7059690952301025,
      "logps/chosen": -8.621637344360352,
      "logps/rejected": -53.87293243408203,
      "loss": 0.1944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7814040184020996,
      "rewards/margins": 1.7814040184020996,
      "rewards/rejected": 0.0,
      "step": 591
    },
    {
      "epoch": 3.3072625698324023,
      "grad_norm": 5.227706371515974,
      "learning_rate": 4.8291671738301945e-06,
      "logits/chosen": -2.930783748626709,
      "logits/rejected": -3.070378303527832,
      "logps/chosen": -12.419721603393555,
      "logps/rejected": -63.619346618652344,
      "loss": 0.1483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3155133724212646,
      "rewards/margins": 2.3155133724212646,
      "rewards/rejected": 0.0,
      "step": 592
    },
    {
      "epoch": 3.3128491620111733,
      "grad_norm": 7.804122074443649,
      "learning_rate": 4.8280581638835015e-06,
      "logits/chosen": -3.830213785171509,
      "logits/rejected": -3.9136784076690674,
      "logps/chosen": -12.369525909423828,
      "logps/rejected": -46.501365661621094,
      "loss": 0.177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8357553482055664,
      "rewards/margins": 2.8357553482055664,
      "rewards/rejected": 0.0,
      "step": 593
    },
    {
      "epoch": 3.3184357541899443,
      "grad_norm": 7.051197156179111,
      "learning_rate": 4.826945694027771e-06,
      "logits/chosen": -3.543114423751831,
      "logits/rejected": -3.55729603767395,
      "logps/chosen": -10.202515602111816,
      "logps/rejected": -41.71031951904297,
      "loss": 0.1874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.169372320175171,
      "rewards/margins": 1.169372320175171,
      "rewards/rejected": 0.0,
      "step": 594
    },
    {
      "epoch": 3.3240223463687153,
      "grad_norm": 4.55186840552668,
      "learning_rate": 4.825829765916334e-06,
      "logits/chosen": -3.489142894744873,
      "logits/rejected": -3.542714834213257,
      "logps/chosen": -21.402925491333008,
      "logps/rejected": -49.10740661621094,
      "loss": 0.1077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2347660064697266,
      "rewards/margins": 3.2347660064697266,
      "rewards/rejected": 0.0,
      "step": 595
    },
    {
      "epoch": 3.329608938547486,
      "grad_norm": 6.6351003068720065,
      "learning_rate": 4.824710381207656e-06,
      "logits/chosen": -3.6005606651306152,
      "logits/rejected": -3.607652187347412,
      "logps/chosen": -6.15773868560791,
      "logps/rejected": -40.244354248046875,
      "loss": 0.2185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2410125732421875,
      "rewards/margins": 2.2410125732421875,
      "rewards/rejected": 0.0,
      "step": 596
    },
    {
      "epoch": 3.335195530726257,
      "grad_norm": 10.120032918069912,
      "learning_rate": 4.823587541565344e-06,
      "logits/chosen": -3.4826314449310303,
      "logits/rejected": -3.6615326404571533,
      "logps/chosen": -5.860255718231201,
      "logps/rejected": -50.78681182861328,
      "loss": 0.2437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0913797616958618,
      "rewards/margins": 1.0913797616958618,
      "rewards/rejected": 0.0,
      "step": 597
    },
    {
      "epoch": 3.340782122905028,
      "grad_norm": 5.1727927289983375,
      "learning_rate": 4.822461248658137e-06,
      "logits/chosen": -3.579887866973877,
      "logits/rejected": -3.8263485431671143,
      "logps/chosen": -10.900751113891602,
      "logps/rejected": -99.0810775756836,
      "loss": 0.1757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.925130009651184,
      "rewards/margins": 1.925130009651184,
      "rewards/rejected": 0.0,
      "step": 598
    },
    {
      "epoch": 3.346368715083799,
      "grad_norm": 5.126641512635156,
      "learning_rate": 4.821331504159906e-06,
      "logits/chosen": -3.760937213897705,
      "logits/rejected": -3.4571523666381836,
      "logps/chosen": -7.296835422515869,
      "logps/rejected": -64.24574279785156,
      "loss": 0.159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9948863983154297,
      "rewards/margins": 1.9948863983154297,
      "rewards/rejected": 0.0,
      "step": 599
    },
    {
      "epoch": 3.35195530726257,
      "grad_norm": 6.778212051362365,
      "learning_rate": 4.820198309749654e-06,
      "logits/chosen": -3.715524435043335,
      "logits/rejected": -3.824496269226074,
      "logps/chosen": -6.801763534545898,
      "logps/rejected": -52.965492248535156,
      "loss": 0.1847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3065123558044434,
      "rewards/margins": 1.3065123558044434,
      "rewards/rejected": 0.0,
      "step": 600
    },
    {
      "epoch": 3.35195530726257,
      "eval_logits/chosen": -3.4392287731170654,
      "eval_logits/rejected": -3.513981580734253,
      "eval_logps/chosen": -27.701074600219727,
      "eval_logps/rejected": -40.36104965209961,
      "eval_loss": 0.7794815897941589,
      "eval_rewards/accuracies": 0.550000011920929,
      "eval_rewards/chosen": 0.23689691722393036,
      "eval_rewards/margins": 0.23689691722393036,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5801,
      "eval_samples_per_second": 12.119,
      "eval_steps_per_second": 0.391,
      "step": 600
    },
    {
      "epoch": 3.357541899441341,
      "grad_norm": 7.51680622173121,
      "learning_rate": 4.819061667111509e-06,
      "logits/chosen": -3.5339367389678955,
      "logits/rejected": -3.723849296569824,
      "logps/chosen": -4.610629081726074,
      "logps/rejected": -80.5616455078125,
      "loss": 0.213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5026719570159912,
      "rewards/margins": 1.5026719570159912,
      "rewards/rejected": 0.0,
      "step": 601
    },
    {
      "epoch": 3.363128491620112,
      "grad_norm": 8.941029777783285,
      "learning_rate": 4.8179215779347245e-06,
      "logits/chosen": -3.546337127685547,
      "logits/rejected": -3.5805158615112305,
      "logps/chosen": -18.599018096923828,
      "logps/rejected": -42.407169342041016,
      "loss": 0.2409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8827435970306396,
      "rewards/margins": 2.8827435970306396,
      "rewards/rejected": 0.0,
      "step": 602
    },
    {
      "epoch": 3.368715083798883,
      "grad_norm": 13.689530457320492,
      "learning_rate": 4.816778043913675e-06,
      "logits/chosen": -3.54258394241333,
      "logits/rejected": -3.401550769805908,
      "logps/chosen": -21.965412139892578,
      "logps/rejected": -30.089923858642578,
      "loss": 0.1834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5572893619537354,
      "rewards/margins": 2.5572893619537354,
      "rewards/rejected": 0.0,
      "step": 603
    },
    {
      "epoch": 3.3743016759776534,
      "grad_norm": 6.518891565070713,
      "learning_rate": 4.815631066747857e-06,
      "logits/chosen": -3.7631072998046875,
      "logits/rejected": -3.650299072265625,
      "logps/chosen": -14.426202774047852,
      "logps/rejected": -59.20515060424805,
      "loss": 0.1685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2215367555618286,
      "rewards/margins": 1.2215367555618286,
      "rewards/rejected": 0.0,
      "step": 604
    },
    {
      "epoch": 3.3798882681564244,
      "grad_norm": 6.250858997054796,
      "learning_rate": 4.8144806481418834e-06,
      "logits/chosen": -3.6882870197296143,
      "logits/rejected": -3.8128466606140137,
      "logps/chosen": -3.4728193283081055,
      "logps/rejected": -38.668556213378906,
      "loss": 0.1846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9175872802734375,
      "rewards/margins": 1.9175872802734375,
      "rewards/rejected": 0.0,
      "step": 605
    },
    {
      "epoch": 3.3854748603351954,
      "grad_norm": 12.916945389839379,
      "learning_rate": 4.813326789805479e-06,
      "logits/chosen": -3.55879282951355,
      "logits/rejected": -3.5864598751068115,
      "logps/chosen": -20.849552154541016,
      "logps/rejected": -69.16793823242188,
      "loss": 0.3376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.5289812088012695,
      "rewards/margins": 1.5289812088012695,
      "rewards/rejected": 0.0,
      "step": 606
    },
    {
      "epoch": 3.3910614525139664,
      "grad_norm": 10.19866619802567,
      "learning_rate": 4.812169493453485e-06,
      "logits/chosen": -3.5235719680786133,
      "logits/rejected": -3.5062103271484375,
      "logps/chosen": -14.67902660369873,
      "logps/rejected": -41.75846481323242,
      "loss": 0.2254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7493878602981567,
      "rewards/margins": 1.7493878602981567,
      "rewards/rejected": 0.0,
      "step": 607
    },
    {
      "epoch": 3.3966480446927374,
      "grad_norm": 7.276060011580945,
      "learning_rate": 4.811008760805848e-06,
      "logits/chosen": -3.607107162475586,
      "logits/rejected": -3.5017011165618896,
      "logps/chosen": -8.073440551757812,
      "logps/rejected": -37.355186462402344,
      "loss": 0.2104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8066673278808594,
      "rewards/margins": 2.8066673278808594,
      "rewards/rejected": 0.0,
      "step": 608
    },
    {
      "epoch": 3.4022346368715084,
      "grad_norm": 10.465615991813689,
      "learning_rate": 4.809844593587624e-06,
      "logits/chosen": -3.5396969318389893,
      "logits/rejected": -3.595827579498291,
      "logps/chosen": -6.576706886291504,
      "logps/rejected": -68.46046447753906,
      "loss": 0.2347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.957580804824829,
      "rewards/margins": 1.957580804824829,
      "rewards/rejected": 0.0,
      "step": 609
    },
    {
      "epoch": 3.4078212290502794,
      "grad_norm": 4.737899358984887,
      "learning_rate": 4.808676993528975e-06,
      "logits/chosen": -3.4853336811065674,
      "logits/rejected": -3.2059054374694824,
      "logps/chosen": -26.520641326904297,
      "logps/rejected": -27.343265533447266,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0303542613983154,
      "rewards/margins": 3.0303542613983154,
      "rewards/rejected": 0.0,
      "step": 610
    },
    {
      "epoch": 3.4134078212290504,
      "grad_norm": 6.966777502759305,
      "learning_rate": 4.80750596236516e-06,
      "logits/chosen": -3.874516487121582,
      "logits/rejected": -3.7118914127349854,
      "logps/chosen": -9.689661026000977,
      "logps/rejected": -26.867454528808594,
      "loss": 0.1968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4182868003845215,
      "rewards/margins": 2.4182868003845215,
      "rewards/rejected": 0.0,
      "step": 611
    },
    {
      "epoch": 3.4189944134078214,
      "grad_norm": 3.939927149202863,
      "learning_rate": 4.80633150183654e-06,
      "logits/chosen": -3.661717653274536,
      "logits/rejected": -3.623983144760132,
      "logps/chosen": -6.190341949462891,
      "logps/rejected": -32.013427734375,
      "loss": 0.1554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8315867185592651,
      "rewards/margins": 1.8315867185592651,
      "rewards/rejected": 0.0,
      "step": 612
    },
    {
      "epoch": 3.4245810055865924,
      "grad_norm": 8.743223642234721,
      "learning_rate": 4.805153613688573e-06,
      "logits/chosen": -4.1319451332092285,
      "logits/rejected": -4.000922203063965,
      "logps/chosen": -26.528215408325195,
      "logps/rejected": -36.14392852783203,
      "loss": 0.1557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6150026321411133,
      "rewards/margins": 3.6150026321411133,
      "rewards/rejected": 0.0,
      "step": 613
    },
    {
      "epoch": 3.430167597765363,
      "grad_norm": 11.523037905190172,
      "learning_rate": 4.803972299671811e-06,
      "logits/chosen": -3.80461049079895,
      "logits/rejected": -3.5560615062713623,
      "logps/chosen": -6.905467510223389,
      "logps/rejected": -41.518184661865234,
      "loss": 0.3381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8341403007507324,
      "rewards/margins": 1.8341403007507324,
      "rewards/rejected": 0.0,
      "step": 614
    },
    {
      "epoch": 3.435754189944134,
      "grad_norm": 5.5150252468939875,
      "learning_rate": 4.802787561541896e-06,
      "logits/chosen": -3.4444825649261475,
      "logits/rejected": -3.437539577484131,
      "logps/chosen": -10.180009841918945,
      "logps/rejected": -45.739715576171875,
      "loss": 0.1561,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5233802795410156,
      "rewards/margins": 2.5233802795410156,
      "rewards/rejected": 0.0,
      "step": 615
    },
    {
      "epoch": 3.441340782122905,
      "grad_norm": 5.994860730934066,
      "learning_rate": 4.801599401059561e-06,
      "logits/chosen": -3.722310781478882,
      "logits/rejected": -3.7925031185150146,
      "logps/chosen": -6.436112403869629,
      "logps/rejected": -34.28742980957031,
      "loss": 0.1766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9283618927001953,
      "rewards/margins": 1.9283618927001953,
      "rewards/rejected": 0.0,
      "step": 616
    },
    {
      "epoch": 3.446927374301676,
      "grad_norm": 4.372100915929567,
      "learning_rate": 4.800407819990625e-06,
      "logits/chosen": -3.601862907409668,
      "logits/rejected": -3.7100462913513184,
      "logps/chosen": -14.612257957458496,
      "logps/rejected": -37.144283294677734,
      "loss": 0.1365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.62644624710083,
      "rewards/margins": 2.62644624710083,
      "rewards/rejected": 0.0,
      "step": 617
    },
    {
      "epoch": 3.452513966480447,
      "grad_norm": 6.090242444036939,
      "learning_rate": 4.799212820105986e-06,
      "logits/chosen": -3.554027795791626,
      "logits/rejected": -3.5970399379730225,
      "logps/chosen": -10.000203132629395,
      "logps/rejected": -39.477237701416016,
      "loss": 0.1992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2379651069641113,
      "rewards/margins": 2.2379651069641113,
      "rewards/rejected": 0.0,
      "step": 618
    },
    {
      "epoch": 3.458100558659218,
      "grad_norm": 7.289187966207348,
      "learning_rate": 4.7980144031816304e-06,
      "logits/chosen": -3.8722686767578125,
      "logits/rejected": -3.705322504043579,
      "logps/chosen": -15.548469543457031,
      "logps/rejected": -51.8181266784668,
      "loss": 0.1583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6380810737609863,
      "rewards/margins": 2.6380810737609863,
      "rewards/rejected": 0.0,
      "step": 619
    },
    {
      "epoch": 3.463687150837989,
      "grad_norm": 19.474089127324948,
      "learning_rate": 4.796812570998617e-06,
      "logits/chosen": -3.7076618671417236,
      "logits/rejected": -3.5529708862304688,
      "logps/chosen": -6.00963020324707,
      "logps/rejected": -36.00750732421875,
      "loss": 0.2343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0916075706481934,
      "rewards/margins": 2.0916075706481934,
      "rewards/rejected": 0.0,
      "step": 620
    },
    {
      "epoch": 3.463687150837989,
      "eval_logits/chosen": -3.4437155723571777,
      "eval_logits/rejected": -3.5191574096679688,
      "eval_logps/chosen": -28.467052459716797,
      "eval_logps/rejected": -41.61983871459961,
      "eval_loss": 0.7994943857192993,
      "eval_rewards/accuracies": 0.550000011920929,
      "eval_rewards/chosen": 0.16029894351959229,
      "eval_rewards/margins": 0.16029894351959229,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5463,
      "eval_samples_per_second": 12.135,
      "eval_steps_per_second": 0.391,
      "step": 620
    },
    {
      "epoch": 3.46927374301676,
      "grad_norm": 6.484219536365286,
      "learning_rate": 4.795607325343084e-06,
      "logits/chosen": -3.3029422760009766,
      "logits/rejected": -3.3994438648223877,
      "logps/chosen": -10.461865425109863,
      "logps/rejected": -47.5035400390625,
      "loss": 0.1702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6674866676330566,
      "rewards/margins": 1.6674866676330566,
      "rewards/rejected": 0.0,
      "step": 621
    },
    {
      "epoch": 3.4748603351955305,
      "grad_norm": 8.090478153689919,
      "learning_rate": 4.7943986680062384e-06,
      "logits/chosen": -3.705012559890747,
      "logits/rejected": -3.706943988800049,
      "logps/chosen": -13.556625366210938,
      "logps/rejected": -42.50819396972656,
      "loss": 0.19,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0345641374588013,
      "rewards/margins": 1.0345641374588013,
      "rewards/rejected": 0.0,
      "step": 622
    },
    {
      "epoch": 3.4804469273743015,
      "grad_norm": 8.52552313327546,
      "learning_rate": 4.793186600784364e-06,
      "logits/chosen": -3.4044454097747803,
      "logits/rejected": -3.501453161239624,
      "logps/chosen": -5.9863786697387695,
      "logps/rejected": -58.89891815185547,
      "loss": 0.1814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8412182331085205,
      "rewards/margins": 1.8412182331085205,
      "rewards/rejected": 0.0,
      "step": 623
    },
    {
      "epoch": 3.4860335195530725,
      "grad_norm": 10.136641577841798,
      "learning_rate": 4.791971125478806e-06,
      "logits/chosen": -3.6125845909118652,
      "logits/rejected": -3.416051149368286,
      "logps/chosen": -8.56518840789795,
      "logps/rejected": -37.60308074951172,
      "loss": 0.2517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8007904291152954,
      "rewards/margins": 1.8007904291152954,
      "rewards/rejected": 0.0,
      "step": 624
    },
    {
      "epoch": 3.4916201117318435,
      "grad_norm": 5.362836179213478,
      "learning_rate": 4.7907522438959775e-06,
      "logits/chosen": -3.7791085243225098,
      "logits/rejected": -3.5737106800079346,
      "logps/chosen": -8.412175178527832,
      "logps/rejected": -39.17524719238281,
      "loss": 0.1896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.166753888130188,
      "rewards/margins": 1.166753888130188,
      "rewards/rejected": 0.0,
      "step": 625
    },
    {
      "epoch": 3.4972067039106145,
      "grad_norm": 12.769899942508014,
      "learning_rate": 4.789529957847355e-06,
      "logits/chosen": -3.3971569538116455,
      "logits/rejected": -3.4277312755584717,
      "logps/chosen": -14.376250267028809,
      "logps/rejected": -33.547542572021484,
      "loss": 0.3112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1252312660217285,
      "rewards/margins": 2.1252312660217285,
      "rewards/rejected": 0.0,
      "step": 626
    },
    {
      "epoch": 3.5027932960893855,
      "grad_norm": 4.881973039463472,
      "learning_rate": 4.7883042691494705e-06,
      "logits/chosen": -3.420297145843506,
      "logits/rejected": -3.5313587188720703,
      "logps/chosen": -5.320849895477295,
      "logps/rejected": -47.995399475097656,
      "loss": 0.1613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4922823905944824,
      "rewards/margins": 1.4922823905944824,
      "rewards/rejected": 0.0,
      "step": 627
    },
    {
      "epoch": 3.5083798882681565,
      "grad_norm": 6.4910076199303095,
      "learning_rate": 4.787075179623918e-06,
      "logits/chosen": -3.6691818237304688,
      "logits/rejected": -3.738051414489746,
      "logps/chosen": -18.210773468017578,
      "logps/rejected": -51.83380889892578,
      "loss": 0.1507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9907724857330322,
      "rewards/margins": 1.9907724857330322,
      "rewards/rejected": 0.0,
      "step": 628
    },
    {
      "epoch": 3.5139664804469275,
      "grad_norm": 9.680798962534313,
      "learning_rate": 4.7858426910973435e-06,
      "logits/chosen": -3.349256753921509,
      "logits/rejected": -3.3463711738586426,
      "logps/chosen": -7.359307289123535,
      "logps/rejected": -41.967376708984375,
      "loss": 0.2598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9793089032173157,
      "rewards/margins": 0.9793089032173157,
      "rewards/rejected": 0.0,
      "step": 629
    },
    {
      "epoch": 3.5195530726256985,
      "grad_norm": 11.77916164446173,
      "learning_rate": 4.784606805401443e-06,
      "logits/chosen": -3.251560688018799,
      "logits/rejected": -3.2776331901550293,
      "logps/chosen": -25.092506408691406,
      "logps/rejected": -57.283966064453125,
      "loss": 0.3096,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.0780842304229736,
      "rewards/margins": 1.0780842304229736,
      "rewards/rejected": 0.0,
      "step": 630
    },
    {
      "epoch": 3.5251396648044695,
      "grad_norm": 4.9288046416758995,
      "learning_rate": 4.783367524372963e-06,
      "logits/chosen": -3.7195534706115723,
      "logits/rejected": -3.701195478439331,
      "logps/chosen": -8.048587799072266,
      "logps/rejected": -34.88921356201172,
      "loss": 0.1631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.284801483154297,
      "rewards/margins": 2.284801483154297,
      "rewards/rejected": 0.0,
      "step": 631
    },
    {
      "epoch": 3.5307262569832405,
      "grad_norm": 5.926137682452823,
      "learning_rate": 4.782124849853697e-06,
      "logits/chosen": -3.34482479095459,
      "logits/rejected": -3.4644083976745605,
      "logps/chosen": -10.446487426757812,
      "logps/rejected": -47.967525482177734,
      "loss": 0.1526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4711217880249023,
      "rewards/margins": 2.4711217880249023,
      "rewards/rejected": 0.0,
      "step": 632
    },
    {
      "epoch": 3.536312849162011,
      "grad_norm": 4.553349472064368,
      "learning_rate": 4.780878783690479e-06,
      "logits/chosen": -3.3005213737487793,
      "logits/rejected": -3.4283134937286377,
      "logps/chosen": -7.037881374359131,
      "logps/rejected": -32.19195556640625,
      "loss": 0.1544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8646538257598877,
      "rewards/margins": 1.8646538257598877,
      "rewards/rejected": 0.0,
      "step": 633
    },
    {
      "epoch": 3.541899441340782,
      "grad_norm": 10.066796313434525,
      "learning_rate": 4.779629327735188e-06,
      "logits/chosen": -3.637822389602661,
      "logits/rejected": -3.7103750705718994,
      "logps/chosen": -7.017227649688721,
      "logps/rejected": -66.68121337890625,
      "loss": 0.2147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.657154083251953,
      "rewards/margins": 2.657154083251953,
      "rewards/rejected": 0.0,
      "step": 634
    },
    {
      "epoch": 3.547486033519553,
      "grad_norm": 5.276020471814379,
      "learning_rate": 4.778376483844734e-06,
      "logits/chosen": -3.5323410034179688,
      "logits/rejected": -3.445220947265625,
      "logps/chosen": -13.836925506591797,
      "logps/rejected": -46.64960479736328,
      "loss": 0.1824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8257861137390137,
      "rewards/margins": 1.8257861137390137,
      "rewards/rejected": 0.0,
      "step": 635
    },
    {
      "epoch": 3.553072625698324,
      "grad_norm": 6.4980819663943805,
      "learning_rate": 4.77712025388107e-06,
      "logits/chosen": -3.2712175846099854,
      "logits/rejected": -3.4408462047576904,
      "logps/chosen": -5.465388774871826,
      "logps/rejected": -44.783470153808594,
      "loss": 0.1972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.090186595916748,
      "rewards/margins": 2.090186595916748,
      "rewards/rejected": 0.0,
      "step": 636
    },
    {
      "epoch": 3.558659217877095,
      "grad_norm": 5.641269142633293,
      "learning_rate": 4.775860639711176e-06,
      "logits/chosen": -3.5800206661224365,
      "logits/rejected": -3.3923377990722656,
      "logps/chosen": -12.307031631469727,
      "logps/rejected": -92.94241333007812,
      "loss": 0.167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.933529019355774,
      "rewards/margins": 1.933529019355774,
      "rewards/rejected": 0.0,
      "step": 637
    },
    {
      "epoch": 3.564245810055866,
      "grad_norm": 9.454088589146645,
      "learning_rate": 4.7745976432070615e-06,
      "logits/chosen": -3.267191171646118,
      "logits/rejected": -3.465709924697876,
      "logps/chosen": -25.384586334228516,
      "logps/rejected": -54.314247131347656,
      "loss": 0.1834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.691128730773926,
      "rewards/margins": 2.691128730773926,
      "rewards/rejected": 0.0,
      "step": 638
    },
    {
      "epoch": 3.5698324022346366,
      "grad_norm": 5.690395177292742,
      "learning_rate": 4.773331266245767e-06,
      "logits/chosen": -3.7093660831451416,
      "logits/rejected": -3.7013697624206543,
      "logps/chosen": -2.8740358352661133,
      "logps/rejected": -27.017486572265625,
      "loss": 0.1978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2889208793640137,
      "rewards/margins": 1.2889208793640137,
      "rewards/rejected": 0.0,
      "step": 639
    },
    {
      "epoch": 3.5754189944134076,
      "grad_norm": 5.5560724425485954,
      "learning_rate": 4.772061510709351e-06,
      "logits/chosen": -3.464585542678833,
      "logits/rejected": -3.3191869258880615,
      "logps/chosen": -13.93330192565918,
      "logps/rejected": -43.48509216308594,
      "loss": 0.1629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.344024658203125,
      "rewards/margins": 2.344024658203125,
      "rewards/rejected": 0.0,
      "step": 640
    },
    {
      "epoch": 3.5754189944134076,
      "eval_logits/chosen": -3.42974591255188,
      "eval_logits/rejected": -3.5119357109069824,
      "eval_logps/chosen": -28.130298614501953,
      "eval_logps/rejected": -39.89588165283203,
      "eval_loss": 0.8061860203742981,
      "eval_rewards/accuracies": 0.574999988079071,
      "eval_rewards/chosen": 0.19397439062595367,
      "eval_rewards/margins": 0.19397439062595367,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5414,
      "eval_samples_per_second": 12.137,
      "eval_steps_per_second": 0.392,
      "step": 640
    },
    {
      "epoch": 3.5810055865921786,
      "grad_norm": 5.754740772606839,
      "learning_rate": 4.7707883784848985e-06,
      "logits/chosen": -3.5592145919799805,
      "logits/rejected": -3.6638150215148926,
      "logps/chosen": -18.642562866210938,
      "logps/rejected": -53.02440643310547,
      "loss": 0.154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7951316833496094,
      "rewards/margins": 2.7951316833496094,
      "rewards/rejected": 0.0,
      "step": 641
    },
    {
      "epoch": 3.5865921787709496,
      "grad_norm": 6.852153848925176,
      "learning_rate": 4.769511871464511e-06,
      "logits/chosen": -3.711050271987915,
      "logits/rejected": -3.554518699645996,
      "logps/chosen": -8.704967498779297,
      "logps/rejected": -42.61420440673828,
      "loss": 0.1569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.129051685333252,
      "rewards/margins": 2.129051685333252,
      "rewards/rejected": 0.0,
      "step": 642
    },
    {
      "epoch": 3.5921787709497206,
      "grad_norm": 4.787235067838402,
      "learning_rate": 4.768231991545304e-06,
      "logits/chosen": -3.543412446975708,
      "logits/rejected": -3.7957582473754883,
      "logps/chosen": -6.597164630889893,
      "logps/rejected": -45.936767578125,
      "loss": 0.1674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0774264335632324,
      "rewards/margins": 2.0774264335632324,
      "rewards/rejected": 0.0,
      "step": 643
    },
    {
      "epoch": 3.5977653631284916,
      "grad_norm": 7.19536782006079,
      "learning_rate": 4.7669487406294076e-06,
      "logits/chosen": -3.440114736557007,
      "logits/rejected": -3.2933778762817383,
      "logps/chosen": -12.182418823242188,
      "logps/rejected": -78.68785858154297,
      "loss": 0.1763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7328377962112427,
      "rewards/margins": 1.7328377962112427,
      "rewards/rejected": 0.0,
      "step": 644
    },
    {
      "epoch": 3.6033519553072626,
      "grad_norm": 6.729841263356223,
      "learning_rate": 4.7656621206239615e-06,
      "logits/chosen": -3.553126811981201,
      "logits/rejected": -3.596619129180908,
      "logps/chosen": -21.246604919433594,
      "logps/rejected": -37.672462463378906,
      "loss": 0.1642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4170236587524414,
      "rewards/margins": 2.4170236587524414,
      "rewards/rejected": 0.0,
      "step": 645
    },
    {
      "epoch": 3.6089385474860336,
      "grad_norm": 7.555779563811953,
      "learning_rate": 4.764372133441112e-06,
      "logits/chosen": -3.791560411453247,
      "logits/rejected": -3.7655673027038574,
      "logps/chosen": -22.781898498535156,
      "logps/rejected": -34.68488311767578,
      "loss": 0.1922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1395468711853027,
      "rewards/margins": 3.1395468711853027,
      "rewards/rejected": 0.0,
      "step": 646
    },
    {
      "epoch": 3.6145251396648046,
      "grad_norm": 8.669514219926418,
      "learning_rate": 4.763078780998009e-06,
      "logits/chosen": -3.968066692352295,
      "logits/rejected": -3.9108896255493164,
      "logps/chosen": -20.934789657592773,
      "logps/rejected": -43.53668212890625,
      "loss": 0.2047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.964392900466919,
      "rewards/margins": 1.964392900466919,
      "rewards/rejected": 0.0,
      "step": 647
    },
    {
      "epoch": 3.6201117318435756,
      "grad_norm": 5.387665100123607,
      "learning_rate": 4.761782065216805e-06,
      "logits/chosen": -3.7943878173828125,
      "logits/rejected": -3.763796329498291,
      "logps/chosen": -3.6658310890197754,
      "logps/rejected": -32.52326202392578,
      "loss": 0.1631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6639413833618164,
      "rewards/margins": 1.6639413833618164,
      "rewards/rejected": 0.0,
      "step": 648
    },
    {
      "epoch": 3.6256983240223466,
      "grad_norm": 6.4776889949801815,
      "learning_rate": 4.760481988024651e-06,
      "logits/chosen": -3.4631223678588867,
      "logits/rejected": -3.247921943664551,
      "logps/chosen": -17.532135009765625,
      "logps/rejected": -65.61273193359375,
      "loss": 0.1975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4457545280456543,
      "rewards/margins": 2.4457545280456543,
      "rewards/rejected": 0.0,
      "step": 649
    },
    {
      "epoch": 3.631284916201117,
      "grad_norm": 5.6484016483031425,
      "learning_rate": 4.759178551353694e-06,
      "logits/chosen": -3.65202260017395,
      "logits/rejected": -3.723752737045288,
      "logps/chosen": -4.067886829376221,
      "logps/rejected": -43.589027404785156,
      "loss": 0.185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.378831386566162,
      "rewards/margins": 1.378831386566162,
      "rewards/rejected": 0.0,
      "step": 650
    },
    {
      "epoch": 3.636871508379888,
      "grad_norm": 6.58193960778403,
      "learning_rate": 4.7578717571410705e-06,
      "logits/chosen": -3.8458619117736816,
      "logits/rejected": -3.6737735271453857,
      "logps/chosen": -8.196861267089844,
      "logps/rejected": -42.294097900390625,
      "loss": 0.2223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9927579164505005,
      "rewards/margins": 1.9927579164505005,
      "rewards/rejected": 0.0,
      "step": 651
    },
    {
      "epoch": 3.642458100558659,
      "grad_norm": 16.17951917566104,
      "learning_rate": 4.756561607328911e-06,
      "logits/chosen": -3.7244534492492676,
      "logits/rejected": -3.4003283977508545,
      "logps/chosen": -8.448439598083496,
      "logps/rejected": -49.753143310546875,
      "loss": 0.3034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.602919101715088,
      "rewards/margins": 1.602919101715088,
      "rewards/rejected": 0.0,
      "step": 652
    },
    {
      "epoch": 3.64804469273743,
      "grad_norm": 6.620464539845927,
      "learning_rate": 4.755248103864331e-06,
      "logits/chosen": -3.6745693683624268,
      "logits/rejected": -3.6463005542755127,
      "logps/chosen": -13.520362854003906,
      "logps/rejected": -40.0255126953125,
      "loss": 0.192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.253948926925659,
      "rewards/margins": 2.253948926925659,
      "rewards/rejected": 0.0,
      "step": 653
    },
    {
      "epoch": 3.653631284916201,
      "grad_norm": 9.394838382068366,
      "learning_rate": 4.7539312486994326e-06,
      "logits/chosen": -3.8310256004333496,
      "logits/rejected": -3.786125659942627,
      "logps/chosen": -7.481823444366455,
      "logps/rejected": -35.95360565185547,
      "loss": 0.2215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7721312046051025,
      "rewards/margins": 1.7721312046051025,
      "rewards/rejected": 0.0,
      "step": 654
    },
    {
      "epoch": 3.659217877094972,
      "grad_norm": 13.003122664053388,
      "learning_rate": 4.752611043791295e-06,
      "logits/chosen": -3.7214272022247314,
      "logits/rejected": -3.786877393722534,
      "logps/chosen": -7.398440361022949,
      "logps/rejected": -40.49431228637695,
      "loss": 0.2559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.276411533355713,
      "rewards/margins": 2.276411533355713,
      "rewards/rejected": 0.0,
      "step": 655
    },
    {
      "epoch": 3.664804469273743,
      "grad_norm": 11.49225519823964,
      "learning_rate": 4.751287491101978e-06,
      "logits/chosen": -3.799783229827881,
      "logits/rejected": -3.674262523651123,
      "logps/chosen": -14.060473442077637,
      "logps/rejected": -42.837059020996094,
      "loss": 0.3095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.073192834854126,
      "rewards/margins": 2.073192834854126,
      "rewards/rejected": 0.0,
      "step": 656
    },
    {
      "epoch": 3.6703910614525137,
      "grad_norm": 6.476106035316455,
      "learning_rate": 4.7499605925985165e-06,
      "logits/chosen": -3.367530584335327,
      "logits/rejected": -3.3188998699188232,
      "logps/chosen": -9.133729934692383,
      "logps/rejected": -44.469947814941406,
      "loss": 0.179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9812566637992859,
      "rewards/margins": 0.9812566637992859,
      "rewards/rejected": 0.0,
      "step": 657
    },
    {
      "epoch": 3.6759776536312847,
      "grad_norm": 8.926669450851408,
      "learning_rate": 4.748630350252919e-06,
      "logits/chosen": -3.413731336593628,
      "logits/rejected": -3.163193464279175,
      "logps/chosen": -9.118218421936035,
      "logps/rejected": -40.92002868652344,
      "loss": 0.2464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7572979927062988,
      "rewards/margins": 1.7572979927062988,
      "rewards/rejected": 0.0,
      "step": 658
    },
    {
      "epoch": 3.6815642458100557,
      "grad_norm": 7.075861798310473,
      "learning_rate": 4.747296766042161e-06,
      "logits/chosen": -3.6507365703582764,
      "logits/rejected": -3.660268783569336,
      "logps/chosen": -14.66695499420166,
      "logps/rejected": -56.828914642333984,
      "loss": 0.214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9934264421463013,
      "rewards/margins": 0.9934264421463013,
      "rewards/rejected": 0.0,
      "step": 659
    },
    {
      "epoch": 3.6871508379888267,
      "grad_norm": 9.68256235195399,
      "learning_rate": 4.745959841948188e-06,
      "logits/chosen": -3.4773383140563965,
      "logits/rejected": -3.5118839740753174,
      "logps/chosen": -3.1876373291015625,
      "logps/rejected": -35.81080627441406,
      "loss": 0.2216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2740757465362549,
      "rewards/margins": 1.2740757465362549,
      "rewards/rejected": 0.0,
      "step": 660
    },
    {
      "epoch": 3.6871508379888267,
      "eval_logits/chosen": -3.4099373817443848,
      "eval_logits/rejected": -3.499451160430908,
      "eval_logps/chosen": -28.823928833007812,
      "eval_logps/rejected": -42.009117126464844,
      "eval_loss": 0.8059613704681396,
      "eval_rewards/accuracies": 0.6000000238418579,
      "eval_rewards/chosen": 0.12461113929748535,
      "eval_rewards/margins": 0.12461113929748535,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6301,
      "eval_samples_per_second": 12.095,
      "eval_steps_per_second": 0.39,
      "step": 660
    },
    {
      "epoch": 3.6927374301675977,
      "grad_norm": 8.830079913692,
      "learning_rate": 4.744619579957906e-06,
      "logits/chosen": -3.6958189010620117,
      "logits/rejected": -3.663323163986206,
      "logps/chosen": -28.942039489746094,
      "logps/rejected": -31.3182430267334,
      "loss": 0.1939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4774270057678223,
      "rewards/margins": 2.4774270057678223,
      "rewards/rejected": 0.0,
      "step": 661
    },
    {
      "epoch": 3.6983240223463687,
      "grad_norm": 10.594385940186546,
      "learning_rate": 4.743275982063182e-06,
      "logits/chosen": -3.5354983806610107,
      "logits/rejected": -3.3999202251434326,
      "logps/chosen": -15.737643241882324,
      "logps/rejected": -44.88929748535156,
      "loss": 0.2378,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20578783750534058,
      "rewards/margins": 0.20578783750534058,
      "rewards/rejected": 0.0,
      "step": 662
    },
    {
      "epoch": 3.7039106145251397,
      "grad_norm": 9.113014991667976,
      "learning_rate": 4.741929050260844e-06,
      "logits/chosen": -3.656846046447754,
      "logits/rejected": -3.2788662910461426,
      "logps/chosen": -8.455415725708008,
      "logps/rejected": -60.30867004394531,
      "loss": 0.2161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8249247074127197,
      "rewards/margins": 1.8249247074127197,
      "rewards/rejected": 0.0,
      "step": 663
    },
    {
      "epoch": 3.7094972067039107,
      "grad_norm": 5.529170836105029,
      "learning_rate": 4.740578786552672e-06,
      "logits/chosen": -3.7287368774414062,
      "logits/rejected": -3.4007678031921387,
      "logps/chosen": -8.162199974060059,
      "logps/rejected": -36.2088623046875,
      "loss": 0.1806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4614628553390503,
      "rewards/margins": 1.4614628553390503,
      "rewards/rejected": 0.0,
      "step": 664
    },
    {
      "epoch": 3.7150837988826817,
      "grad_norm": 7.8861582703378055,
      "learning_rate": 4.7392251929453965e-06,
      "logits/chosen": -3.5962681770324707,
      "logits/rejected": -3.299670457839966,
      "logps/chosen": -19.040754318237305,
      "logps/rejected": -38.2381591796875,
      "loss": 0.2299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.138477087020874,
      "rewards/margins": 2.138477087020874,
      "rewards/rejected": 0.0,
      "step": 665
    },
    {
      "epoch": 3.7206703910614527,
      "grad_norm": 4.222116323241285,
      "learning_rate": 4.737868271450702e-06,
      "logits/chosen": -3.6776533126831055,
      "logits/rejected": -3.767024517059326,
      "logps/chosen": -7.625303268432617,
      "logps/rejected": -46.311378479003906,
      "loss": 0.1542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.252135753631592,
      "rewards/margins": 2.252135753631592,
      "rewards/rejected": 0.0,
      "step": 666
    },
    {
      "epoch": 3.7262569832402237,
      "grad_norm": 5.256995519355446,
      "learning_rate": 4.736508024085212e-06,
      "logits/chosen": -3.6813454627990723,
      "logits/rejected": -3.594747304916382,
      "logps/chosen": -10.16135025024414,
      "logps/rejected": -61.06520080566406,
      "loss": 0.2055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3369373083114624,
      "rewards/margins": 1.3369373083114624,
      "rewards/rejected": 0.0,
      "step": 667
    },
    {
      "epoch": 3.7318435754189943,
      "grad_norm": 6.508103779500445,
      "learning_rate": 4.7351444528705e-06,
      "logits/chosen": -3.47410249710083,
      "logits/rejected": -3.421844959259033,
      "logps/chosen": -11.938239097595215,
      "logps/rejected": -25.21226692199707,
      "loss": 0.166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3734335899353027,
      "rewards/margins": 3.3734335899353027,
      "rewards/rejected": 0.0,
      "step": 668
    },
    {
      "epoch": 3.7374301675977653,
      "grad_norm": 5.070576260721473,
      "learning_rate": 4.733777559833071e-06,
      "logits/chosen": -3.8622543811798096,
      "logits/rejected": -3.8269472122192383,
      "logps/chosen": -4.9154372215271,
      "logps/rejected": -53.057533264160156,
      "loss": 0.1652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6138534545898438,
      "rewards/margins": 1.6138534545898438,
      "rewards/rejected": 0.0,
      "step": 669
    },
    {
      "epoch": 3.7430167597765363,
      "grad_norm": 4.464733650295034,
      "learning_rate": 4.732407347004376e-06,
      "logits/chosen": -3.6370937824249268,
      "logits/rejected": -3.5830299854278564,
      "logps/chosen": -9.715575218200684,
      "logps/rejected": -30.9544677734375,
      "loss": 0.2062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.590008497238159,
      "rewards/margins": 2.590008497238159,
      "rewards/rejected": 0.0,
      "step": 670
    },
    {
      "epoch": 3.7486033519553073,
      "grad_norm": 9.52131665745009,
      "learning_rate": 4.731033816420792e-06,
      "logits/chosen": -3.6579346656799316,
      "logits/rejected": -3.705422878265381,
      "logps/chosen": -7.2313714027404785,
      "logps/rejected": -38.460453033447266,
      "loss": 0.2384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4676446914672852,
      "rewards/margins": 1.4676446914672852,
      "rewards/rejected": 0.0,
      "step": 671
    },
    {
      "epoch": 3.7541899441340782,
      "grad_norm": 8.063859201237237,
      "learning_rate": 4.729656970123633e-06,
      "logits/chosen": -3.2096598148345947,
      "logits/rejected": -3.4913713932037354,
      "logps/chosen": -22.601146697998047,
      "logps/rejected": -24.459911346435547,
      "loss": 0.2055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2946219444274902,
      "rewards/margins": 2.2946219444274902,
      "rewards/rejected": 0.0,
      "step": 672
    },
    {
      "epoch": 3.7597765363128492,
      "grad_norm": 5.513038533490759,
      "learning_rate": 4.728276810159135e-06,
      "logits/chosen": -3.65520977973938,
      "logits/rejected": -3.673644781112671,
      "logps/chosen": -8.51766586303711,
      "logps/rejected": -56.10723114013672,
      "loss": 0.1524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5842711925506592,
      "rewards/margins": 1.5842711925506592,
      "rewards/rejected": 0.0,
      "step": 673
    },
    {
      "epoch": 3.7653631284916202,
      "grad_norm": 5.883812032553302,
      "learning_rate": 4.726893338578463e-06,
      "logits/chosen": -2.8780782222747803,
      "logits/rejected": -2.811371088027954,
      "logps/chosen": -29.249425888061523,
      "logps/rejected": -55.12591552734375,
      "loss": 0.1685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.219766616821289,
      "rewards/margins": 2.219766616821289,
      "rewards/rejected": 0.0,
      "step": 674
    },
    {
      "epoch": 3.770949720670391,
      "grad_norm": 14.415184980882891,
      "learning_rate": 4.725506557437702e-06,
      "logits/chosen": -3.672071695327759,
      "logits/rejected": -3.5483992099761963,
      "logps/chosen": -22.602832794189453,
      "logps/rejected": -60.177978515625,
      "loss": 0.2451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.2049005031585693,
      "rewards/margins": 2.2049005031585693,
      "rewards/rejected": 0.0,
      "step": 675
    },
    {
      "epoch": 3.776536312849162,
      "grad_norm": 9.108263495620294,
      "learning_rate": 4.724116468797855e-06,
      "logits/chosen": -3.318855047225952,
      "logits/rejected": -3.250792980194092,
      "logps/chosen": -19.908926010131836,
      "logps/rejected": -37.940216064453125,
      "loss": 0.2135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6881332397460938,
      "rewards/margins": 2.6881332397460938,
      "rewards/rejected": 0.0,
      "step": 676
    },
    {
      "epoch": 3.782122905027933,
      "grad_norm": 14.008217326947207,
      "learning_rate": 4.7227230747248435e-06,
      "logits/chosen": -3.625833511352539,
      "logits/rejected": -3.4701781272888184,
      "logps/chosen": -16.629928588867188,
      "logps/rejected": -37.674896240234375,
      "loss": 0.3099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8993169069290161,
      "rewards/margins": 0.8993169069290161,
      "rewards/rejected": 0.0,
      "step": 677
    },
    {
      "epoch": 3.787709497206704,
      "grad_norm": 5.180797858005499,
      "learning_rate": 4.721326377289496e-06,
      "logits/chosen": -3.479795217514038,
      "logits/rejected": -3.3786027431488037,
      "logps/chosen": -33.86552810668945,
      "logps/rejected": -36.790184020996094,
      "loss": 0.156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2047317028045654,
      "rewards/margins": 2.2047317028045654,
      "rewards/rejected": 0.0,
      "step": 678
    },
    {
      "epoch": 3.793296089385475,
      "grad_norm": 9.904213920180629,
      "learning_rate": 4.719926378567555e-06,
      "logits/chosen": -3.5223164558410645,
      "logits/rejected": -3.450791120529175,
      "logps/chosen": -19.45485496520996,
      "logps/rejected": -37.448570251464844,
      "loss": 0.2584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0726404190063477,
      "rewards/margins": 3.0726404190063477,
      "rewards/rejected": 0.0,
      "step": 679
    },
    {
      "epoch": 3.798882681564246,
      "grad_norm": 11.033590740382449,
      "learning_rate": 4.71852308063967e-06,
      "logits/chosen": -3.370786666870117,
      "logits/rejected": -3.30019211769104,
      "logps/chosen": -7.9781694412231445,
      "logps/rejected": -50.222835540771484,
      "loss": 0.2982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8648262023925781,
      "rewards/margins": 0.8648262023925781,
      "rewards/rejected": 0.0,
      "step": 680
    },
    {
      "epoch": 3.798882681564246,
      "eval_logits/chosen": -3.3912758827209473,
      "eval_logits/rejected": -3.467822551727295,
      "eval_logps/chosen": -28.79001808166504,
      "eval_logps/rejected": -44.0811882019043,
      "eval_loss": 0.8372684717178345,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": 0.12800252437591553,
      "eval_rewards/margins": 0.12800252437591553,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5284,
      "eval_samples_per_second": 12.143,
      "eval_steps_per_second": 0.392,
      "step": 680
    },
    {
      "epoch": 3.804469273743017,
      "grad_norm": 10.83196722895827,
      "learning_rate": 4.717116485591389e-06,
      "logits/chosen": -2.8995375633239746,
      "logits/rejected": -2.811350107192993,
      "logps/chosen": -27.29461097717285,
      "logps/rejected": -54.837039947509766,
      "loss": 0.2542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.281766414642334,
      "rewards/margins": 2.281766414642334,
      "rewards/rejected": 0.0,
      "step": 681
    },
    {
      "epoch": 3.810055865921788,
      "grad_norm": 10.362356859011294,
      "learning_rate": 4.715706595513163e-06,
      "logits/chosen": -3.4626212120056152,
      "logits/rejected": -3.485919952392578,
      "logps/chosen": -15.549570083618164,
      "logps/rejected": -77.3575668334961,
      "loss": 0.264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2270989418029785,
      "rewards/margins": 1.2270989418029785,
      "rewards/rejected": 0.0,
      "step": 682
    },
    {
      "epoch": 3.815642458100559,
      "grad_norm": 8.022237949131544,
      "learning_rate": 4.71429341250034e-06,
      "logits/chosen": -3.7809271812438965,
      "logits/rejected": -3.7428815364837646,
      "logps/chosen": -9.234298706054688,
      "logps/rejected": -41.815181732177734,
      "loss": 0.1936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8767935037612915,
      "rewards/margins": 1.8767935037612915,
      "rewards/rejected": 0.0,
      "step": 683
    },
    {
      "epoch": 3.82122905027933,
      "grad_norm": 13.009885682216439,
      "learning_rate": 4.712876938653163e-06,
      "logits/chosen": -3.623527765274048,
      "logits/rejected": -3.599233865737915,
      "logps/chosen": -19.30927085876465,
      "logps/rejected": -41.447242736816406,
      "loss": 0.284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.211812973022461,
      "rewards/margins": 2.211812973022461,
      "rewards/rejected": 0.0,
      "step": 684
    },
    {
      "epoch": 3.826815642458101,
      "grad_norm": 7.454710159125081,
      "learning_rate": 4.7114571760767626e-06,
      "logits/chosen": -3.902747631072998,
      "logits/rejected": -3.572439432144165,
      "logps/chosen": -7.175136089324951,
      "logps/rejected": -81.03994750976562,
      "loss": 0.2175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.90696120262146,
      "rewards/margins": 1.90696120262146,
      "rewards/rejected": 0.0,
      "step": 685
    },
    {
      "epoch": 3.8324022346368714,
      "grad_norm": 4.74931589318241,
      "learning_rate": 4.71003412688116e-06,
      "logits/chosen": -3.293174982070923,
      "logits/rejected": -3.415393352508545,
      "logps/chosen": -35.39019012451172,
      "logps/rejected": -46.16090774536133,
      "loss": 0.1618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2765414714813232,
      "rewards/margins": 3.2765414714813232,
      "rewards/rejected": 0.0,
      "step": 686
    },
    {
      "epoch": 3.8379888268156424,
      "grad_norm": 9.398059079073091,
      "learning_rate": 4.708607793181259e-06,
      "logits/chosen": -3.7059051990509033,
      "logits/rejected": -3.6659412384033203,
      "logps/chosen": -9.546884536743164,
      "logps/rejected": -40.253013610839844,
      "loss": 0.225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.398149013519287,
      "rewards/margins": 2.398149013519287,
      "rewards/rejected": 0.0,
      "step": 687
    },
    {
      "epoch": 3.8435754189944134,
      "grad_norm": 5.219542583961691,
      "learning_rate": 4.707178177096846e-06,
      "logits/chosen": -3.5003347396850586,
      "logits/rejected": -3.4049160480499268,
      "logps/chosen": -8.82661247253418,
      "logps/rejected": -48.066429138183594,
      "loss": 0.1733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5053038597106934,
      "rewards/margins": 1.5053038597106934,
      "rewards/rejected": 0.0,
      "step": 688
    },
    {
      "epoch": 3.8491620111731844,
      "grad_norm": 7.8221617440163795,
      "learning_rate": 4.705745280752586e-06,
      "logits/chosen": -3.4771108627319336,
      "logits/rejected": -3.36594557762146,
      "logps/chosen": -7.674956321716309,
      "logps/rejected": -32.37380599975586,
      "loss": 0.234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2883589267730713,
      "rewards/margins": 1.2883589267730713,
      "rewards/rejected": 0.0,
      "step": 689
    },
    {
      "epoch": 3.8547486033519553,
      "grad_norm": 6.027270794912926,
      "learning_rate": 4.7043091062780155e-06,
      "logits/chosen": -3.093134880065918,
      "logits/rejected": -3.0702993869781494,
      "logps/chosen": -12.896870613098145,
      "logps/rejected": -39.74254608154297,
      "loss": 0.196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5619125366210938,
      "rewards/margins": 2.5619125366210938,
      "rewards/rejected": 0.0,
      "step": 690
    },
    {
      "epoch": 3.8603351955307263,
      "grad_norm": 9.244086858966387,
      "learning_rate": 4.702869655807547e-06,
      "logits/chosen": -3.428267240524292,
      "logits/rejected": -3.453425884246826,
      "logps/chosen": -8.589532852172852,
      "logps/rejected": -38.89219665527344,
      "loss": 0.2817,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7539572715759277,
      "rewards/margins": 1.7539572715759277,
      "rewards/rejected": 0.0,
      "step": 691
    },
    {
      "epoch": 3.8659217877094973,
      "grad_norm": 4.836871495102247,
      "learning_rate": 4.701426931480462e-06,
      "logits/chosen": -3.2299365997314453,
      "logits/rejected": -3.3217391967773438,
      "logps/chosen": -14.344983100891113,
      "logps/rejected": -43.33951187133789,
      "loss": 0.1523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4687182903289795,
      "rewards/margins": 2.4687182903289795,
      "rewards/rejected": 0.0,
      "step": 692
    },
    {
      "epoch": 3.871508379888268,
      "grad_norm": 7.188611972791977,
      "learning_rate": 4.699980935440902e-06,
      "logits/chosen": -3.654404878616333,
      "logits/rejected": -3.3018710613250732,
      "logps/chosen": -10.682149887084961,
      "logps/rejected": -52.184844970703125,
      "loss": 0.2022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2460694313049316,
      "rewards/margins": 2.2460694313049316,
      "rewards/rejected": 0.0,
      "step": 693
    },
    {
      "epoch": 3.877094972067039,
      "grad_norm": 6.227410962855612,
      "learning_rate": 4.698531669837878e-06,
      "logits/chosen": -3.353290557861328,
      "logits/rejected": -3.4187185764312744,
      "logps/chosen": -11.798328399658203,
      "logps/rejected": -38.110477447509766,
      "loss": 0.1707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.602259874343872,
      "rewards/margins": 2.602259874343872,
      "rewards/rejected": 0.0,
      "step": 694
    },
    {
      "epoch": 3.88268156424581,
      "grad_norm": 7.457324807020877,
      "learning_rate": 4.697079136825256e-06,
      "logits/chosen": -3.2213284969329834,
      "logits/rejected": -3.097011089324951,
      "logps/chosen": -33.14448547363281,
      "logps/rejected": -41.06036376953125,
      "loss": 0.1954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.465236186981201,
      "rewards/margins": 2.465236186981201,
      "rewards/rejected": 0.0,
      "step": 695
    },
    {
      "epoch": 3.888268156424581,
      "grad_norm": 15.686217118530022,
      "learning_rate": 4.6956233385617575e-06,
      "logits/chosen": -3.3892557621002197,
      "logits/rejected": -3.5463368892669678,
      "logps/chosen": -26.541030883789062,
      "logps/rejected": -48.816490173339844,
      "loss": 0.2609,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.075298547744751,
      "rewards/margins": 1.075298547744751,
      "rewards/rejected": 0.0,
      "step": 696
    },
    {
      "epoch": 3.893854748603352,
      "grad_norm": 7.976509225232093,
      "learning_rate": 4.6941642772109584e-06,
      "logits/chosen": -3.506533622741699,
      "logits/rejected": -3.685051202774048,
      "logps/chosen": -5.026763916015625,
      "logps/rejected": -42.10284423828125,
      "loss": 0.1963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7167224884033203,
      "rewards/margins": 1.7167224884033203,
      "rewards/rejected": 0.0,
      "step": 697
    },
    {
      "epoch": 3.899441340782123,
      "grad_norm": 5.2349442953990915,
      "learning_rate": 4.692701954941284e-06,
      "logits/chosen": -3.5950589179992676,
      "logits/rejected": -3.4048166275024414,
      "logps/chosen": -41.86760711669922,
      "logps/rejected": -45.50979232788086,
      "loss": 0.1603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.257052183151245,
      "rewards/margins": 3.257052183151245,
      "rewards/rejected": 0.0,
      "step": 698
    },
    {
      "epoch": 3.905027932960894,
      "grad_norm": 11.636727780086062,
      "learning_rate": 4.691236373926007e-06,
      "logits/chosen": -3.7524166107177734,
      "logits/rejected": -3.762364387512207,
      "logps/chosen": -8.896484375,
      "logps/rejected": -35.31393814086914,
      "loss": 0.237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7321981191635132,
      "rewards/margins": 1.7321981191635132,
      "rewards/rejected": 0.0,
      "step": 699
    },
    {
      "epoch": 3.910614525139665,
      "grad_norm": 5.313388631389935,
      "learning_rate": 4.68976753634324e-06,
      "logits/chosen": -3.9892921447753906,
      "logits/rejected": -3.8974342346191406,
      "logps/chosen": -10.671701431274414,
      "logps/rejected": -35.03058624267578,
      "loss": 0.1526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.321685791015625,
      "rewards/margins": 2.321685791015625,
      "rewards/rejected": 0.0,
      "step": 700
    },
    {
      "epoch": 3.910614525139665,
      "eval_logits/chosen": -3.3899314403533936,
      "eval_logits/rejected": -3.4697322845458984,
      "eval_logps/chosen": -28.17318344116211,
      "eval_logps/rejected": -42.10552215576172,
      "eval_loss": 0.840867280960083,
      "eval_rewards/accuracies": 0.6000000238418579,
      "eval_rewards/chosen": 0.1896861493587494,
      "eval_rewards/margins": 0.1896861493587494,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5563,
      "eval_samples_per_second": 12.13,
      "eval_steps_per_second": 0.391,
      "step": 700
    },
    {
      "epoch": 3.916201117318436,
      "grad_norm": 6.6956431964077385,
      "learning_rate": 4.6882954443759374e-06,
      "logits/chosen": -3.589661121368408,
      "logits/rejected": -3.569284200668335,
      "logps/chosen": -5.161067962646484,
      "logps/rejected": -75.24226379394531,
      "loss": 0.2049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.764440894126892,
      "rewards/margins": 1.764440894126892,
      "rewards/rejected": 0.0,
      "step": 701
    },
    {
      "epoch": 3.921787709497207,
      "grad_norm": 9.026191512303784,
      "learning_rate": 4.686820100211891e-06,
      "logits/chosen": -3.705820083618164,
      "logits/rejected": -3.6214001178741455,
      "logps/chosen": -13.898820877075195,
      "logps/rejected": -31.889625549316406,
      "loss": 0.2185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2923707962036133,
      "rewards/margins": 2.2923707962036133,
      "rewards/rejected": 0.0,
      "step": 702
    },
    {
      "epoch": 3.927374301675978,
      "grad_norm": 9.155420041458425,
      "learning_rate": 4.685341506043723e-06,
      "logits/chosen": -3.4805731773376465,
      "logits/rejected": -3.588463544845581,
      "logps/chosen": -26.118555068969727,
      "logps/rejected": -43.71784591674805,
      "loss": 0.2122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.208099126815796,
      "rewards/margins": 3.208099126815796,
      "rewards/rejected": 0.0,
      "step": 703
    },
    {
      "epoch": 3.9329608938547485,
      "grad_norm": 5.758242295193571,
      "learning_rate": 4.68385966406889e-06,
      "logits/chosen": -3.5621721744537354,
      "logits/rejected": -3.5439846515655518,
      "logps/chosen": -7.455489635467529,
      "logps/rejected": -43.101341247558594,
      "loss": 0.173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.858414649963379,
      "rewards/margins": 1.858414649963379,
      "rewards/rejected": 0.0,
      "step": 704
    },
    {
      "epoch": 3.9385474860335195,
      "grad_norm": 7.433749228856837,
      "learning_rate": 4.682374576489672e-06,
      "logits/chosen": -3.357069730758667,
      "logits/rejected": -3.6034626960754395,
      "logps/chosen": -12.881941795349121,
      "logps/rejected": -43.72645950317383,
      "loss": 0.1724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9001150131225586,
      "rewards/margins": 2.9001150131225586,
      "rewards/rejected": 0.0,
      "step": 705
    },
    {
      "epoch": 3.9441340782122905,
      "grad_norm": 11.798013729587996,
      "learning_rate": 4.6808862455131736e-06,
      "logits/chosen": -3.6326851844787598,
      "logits/rejected": -3.6483993530273438,
      "logps/chosen": -10.166780471801758,
      "logps/rejected": -29.399486541748047,
      "loss": 0.2428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6209464073181152,
      "rewards/margins": 1.6209464073181152,
      "rewards/rejected": 0.0,
      "step": 706
    },
    {
      "epoch": 3.9497206703910615,
      "grad_norm": 5.849497308798129,
      "learning_rate": 4.67939467335132e-06,
      "logits/chosen": -3.8629636764526367,
      "logits/rejected": -3.6983749866485596,
      "logps/chosen": -29.43489646911621,
      "logps/rejected": -49.81913375854492,
      "loss": 0.1684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.788332939147949,
      "rewards/margins": 2.788332939147949,
      "rewards/rejected": 0.0,
      "step": 707
    },
    {
      "epoch": 3.9553072625698324,
      "grad_norm": 9.458109277255613,
      "learning_rate": 4.677899862220852e-06,
      "logits/chosen": -3.600506067276001,
      "logits/rejected": -3.5317647457122803,
      "logps/chosen": -12.002758026123047,
      "logps/rejected": -26.302013397216797,
      "loss": 0.1917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.033355712890625,
      "rewards/margins": 3.033355712890625,
      "rewards/rejected": 0.0,
      "step": 708
    },
    {
      "epoch": 3.9608938547486034,
      "grad_norm": 9.819462291972224,
      "learning_rate": 4.676401814343327e-06,
      "logits/chosen": -3.7813313007354736,
      "logits/rejected": -3.5953094959259033,
      "logps/chosen": -10.52864933013916,
      "logps/rejected": -62.86785125732422,
      "loss": 0.2692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0931921005249023,
      "rewards/margins": 2.0931921005249023,
      "rewards/rejected": 0.0,
      "step": 709
    },
    {
      "epoch": 3.9664804469273744,
      "grad_norm": 6.700519180748594,
      "learning_rate": 4.6749005319451094e-06,
      "logits/chosen": -3.617539644241333,
      "logits/rejected": -3.566241502761841,
      "logps/chosen": -8.754382133483887,
      "logps/rejected": -47.679664611816406,
      "loss": 0.1844,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.6152373552322388,
      "rewards/margins": 1.6152373552322388,
      "rewards/rejected": 0.0,
      "step": 710
    },
    {
      "epoch": 3.972067039106145,
      "grad_norm": 12.154524497177125,
      "learning_rate": 4.673396017257373e-06,
      "logits/chosen": -3.4723129272460938,
      "logits/rejected": -3.362467050552368,
      "logps/chosen": -11.26205825805664,
      "logps/rejected": -41.029296875,
      "loss": 0.3101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.814652919769287,
      "rewards/margins": 1.814652919769287,
      "rewards/rejected": 0.0,
      "step": 711
    },
    {
      "epoch": 3.977653631284916,
      "grad_norm": 3.957221666018275,
      "learning_rate": 4.671888272516094e-06,
      "logits/chosen": -3.788918972015381,
      "logits/rejected": -3.3950653076171875,
      "logps/chosen": -12.145364761352539,
      "logps/rejected": -44.38713836669922,
      "loss": 0.1424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.189382791519165,
      "rewards/margins": 2.189382791519165,
      "rewards/rejected": 0.0,
      "step": 712
    },
    {
      "epoch": 3.983240223463687,
      "grad_norm": 7.555433351670286,
      "learning_rate": 4.67037729996205e-06,
      "logits/chosen": -3.3715338706970215,
      "logits/rejected": -3.253209114074707,
      "logps/chosen": -6.187528610229492,
      "logps/rejected": -39.947303771972656,
      "loss": 0.1791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9478427171707153,
      "rewards/margins": 1.9478427171707153,
      "rewards/rejected": 0.0,
      "step": 713
    },
    {
      "epoch": 3.988826815642458,
      "grad_norm": 6.8240301380460195,
      "learning_rate": 4.668863101840815e-06,
      "logits/chosen": -3.59162974357605,
      "logits/rejected": -3.661330223083496,
      "logps/chosen": -5.201066970825195,
      "logps/rejected": -49.548179626464844,
      "loss": 0.2014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.025956630706787,
      "rewards/margins": 2.025956630706787,
      "rewards/rejected": 0.0,
      "step": 714
    },
    {
      "epoch": 3.994413407821229,
      "grad_norm": 4.00407381669323,
      "learning_rate": 4.667345680402758e-06,
      "logits/chosen": -3.3789281845092773,
      "logits/rejected": -3.3798727989196777,
      "logps/chosen": -20.46272850036621,
      "logps/rejected": -37.77895736694336,
      "loss": 0.151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4636240005493164,
      "rewards/margins": 2.4636240005493164,
      "rewards/rejected": 0.0,
      "step": 715
    },
    {
      "epoch": 4.0,
      "grad_norm": 7.297384723000338,
      "learning_rate": 4.665825037903036e-06,
      "logits/chosen": -3.392892599105835,
      "logits/rejected": -3.2378597259521484,
      "logps/chosen": -9.669824600219727,
      "logps/rejected": -29.676254272460938,
      "loss": 0.1831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8937046527862549,
      "rewards/margins": 1.8937046527862549,
      "rewards/rejected": 0.0,
      "step": 716
    },
    {
      "epoch": 4.005586592178771,
      "grad_norm": 7.737953114721841,
      "learning_rate": 4.664301176601595e-06,
      "logits/chosen": -3.7267191410064697,
      "logits/rejected": -3.7557132244110107,
      "logps/chosen": -10.259057998657227,
      "logps/rejected": -44.56793975830078,
      "loss": 0.1751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.6343467235565186,
      "rewards/margins": 1.6343467235565186,
      "rewards/rejected": 0.0,
      "step": 717
    },
    {
      "epoch": 4.011173184357542,
      "grad_norm": 3.021852177078668,
      "learning_rate": 4.662774098763165e-06,
      "logits/chosen": -3.3707265853881836,
      "logits/rejected": -3.356926679611206,
      "logps/chosen": -8.665124893188477,
      "logps/rejected": -36.90772247314453,
      "loss": 0.1335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9941139221191406,
      "rewards/margins": 1.9941139221191406,
      "rewards/rejected": 0.0,
      "step": 718
    },
    {
      "epoch": 4.016759776536313,
      "grad_norm": 4.571994141927863,
      "learning_rate": 4.661243806657256e-06,
      "logits/chosen": -3.715198516845703,
      "logits/rejected": -3.3577497005462646,
      "logps/chosen": -4.460678577423096,
      "logps/rejected": -40.8712158203125,
      "loss": 0.1724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9652432203292847,
      "rewards/margins": 1.9652432203292847,
      "rewards/rejected": 0.0,
      "step": 719
    },
    {
      "epoch": 4.022346368715084,
      "grad_norm": 3.928909727232153,
      "learning_rate": 4.659710302558154e-06,
      "logits/chosen": -3.725205659866333,
      "logits/rejected": -3.493034839630127,
      "logps/chosen": -3.9738192558288574,
      "logps/rejected": -25.73491668701172,
      "loss": 0.1435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.007418155670166,
      "rewards/margins": 2.007418155670166,
      "rewards/rejected": 0.0,
      "step": 720
    },
    {
      "epoch": 4.022346368715084,
      "eval_logits/chosen": -3.3798511028289795,
      "eval_logits/rejected": -3.462406873703003,
      "eval_logps/chosen": -27.224308013916016,
      "eval_logps/rejected": -41.68564224243164,
      "eval_loss": 0.8062307238578796,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": 0.284573495388031,
      "eval_rewards/margins": 0.284573495388031,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5438,
      "eval_samples_per_second": 12.136,
      "eval_steps_per_second": 0.391,
      "step": 720
    },
    {
      "epoch": 4.027932960893855,
      "grad_norm": 5.115211595986476,
      "learning_rate": 4.658173588744917e-06,
      "logits/chosen": -3.4412646293640137,
      "logits/rejected": -3.1422479152679443,
      "logps/chosen": -9.848830223083496,
      "logps/rejected": -33.66573715209961,
      "loss": 0.1613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.613081455230713,
      "rewards/margins": 2.613081455230713,
      "rewards/rejected": 0.0,
      "step": 721
    },
    {
      "epoch": 4.033519553072626,
      "grad_norm": 4.132488665803409,
      "learning_rate": 4.65663366750138e-06,
      "logits/chosen": -3.6516289710998535,
      "logits/rejected": -3.6510491371154785,
      "logps/chosen": -11.460866928100586,
      "logps/rejected": -68.56126403808594,
      "loss": 0.1425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6857962608337402,
      "rewards/margins": 2.6857962608337402,
      "rewards/rejected": 0.0,
      "step": 722
    },
    {
      "epoch": 4.039106145251397,
      "grad_norm": 6.306885602436459,
      "learning_rate": 4.655090541116136e-06,
      "logits/chosen": -3.060114860534668,
      "logits/rejected": -2.764143466949463,
      "logps/chosen": -13.021773338317871,
      "logps/rejected": -27.23346710205078,
      "loss": 0.1654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9445297718048096,
      "rewards/margins": 2.9445297718048096,
      "rewards/rejected": 0.0,
      "step": 723
    },
    {
      "epoch": 4.044692737430168,
      "grad_norm": 3.3961354688682297,
      "learning_rate": 4.653544211882548e-06,
      "logits/chosen": -3.3757858276367188,
      "logits/rejected": -3.2025012969970703,
      "logps/chosen": -4.454441070556641,
      "logps/rejected": -40.420814514160156,
      "loss": 0.1232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6740658283233643,
      "rewards/margins": 1.6740658283233643,
      "rewards/rejected": 0.0,
      "step": 724
    },
    {
      "epoch": 4.050279329608939,
      "grad_norm": 5.1995159680495915,
      "learning_rate": 4.651994682098735e-06,
      "logits/chosen": -3.516906499862671,
      "logits/rejected": -3.6081442832946777,
      "logps/chosen": -9.058988571166992,
      "logps/rejected": -86.97225952148438,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.426926612854004,
      "rewards/margins": 2.426926612854004,
      "rewards/rejected": 0.0,
      "step": 725
    },
    {
      "epoch": 4.055865921787709,
      "grad_norm": 9.936777261080561,
      "learning_rate": 4.650441954067576e-06,
      "logits/chosen": -3.6151368618011475,
      "logits/rejected": -3.4337685108184814,
      "logps/chosen": -8.372771263122559,
      "logps/rejected": -40.75501251220703,
      "loss": 0.203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6097676753997803,
      "rewards/margins": 2.6097676753997803,
      "rewards/rejected": 0.0,
      "step": 726
    },
    {
      "epoch": 4.06145251396648,
      "grad_norm": 3.746836288315019,
      "learning_rate": 4.6488860300967e-06,
      "logits/chosen": -3.5283093452453613,
      "logits/rejected": -3.5352466106414795,
      "logps/chosen": -5.351473331451416,
      "logps/rejected": -28.875404357910156,
      "loss": 0.1423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2793641090393066,
      "rewards/margins": 2.2793641090393066,
      "rewards/rejected": 0.0,
      "step": 727
    },
    {
      "epoch": 4.067039106145251,
      "grad_norm": 7.456202136839523,
      "learning_rate": 4.647326912498488e-06,
      "logits/chosen": -3.655738353729248,
      "logits/rejected": -3.420414447784424,
      "logps/chosen": -3.972681999206543,
      "logps/rejected": -66.99506378173828,
      "loss": 0.1812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6251847743988037,
      "rewards/margins": 1.6251847743988037,
      "rewards/rejected": 0.0,
      "step": 728
    },
    {
      "epoch": 4.072625698324022,
      "grad_norm": 4.123799240860416,
      "learning_rate": 4.645764603590066e-06,
      "logits/chosen": -3.6910288333892822,
      "logits/rejected": -3.763230800628662,
      "logps/chosen": -3.5917537212371826,
      "logps/rejected": -44.62071990966797,
      "loss": 0.1409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4260940551757812,
      "rewards/margins": 1.4260940551757812,
      "rewards/rejected": 0.0,
      "step": 729
    },
    {
      "epoch": 4.078212290502793,
      "grad_norm": 4.493730331442008,
      "learning_rate": 4.644199105693303e-06,
      "logits/chosen": -3.9505717754364014,
      "logits/rejected": -3.8407318592071533,
      "logps/chosen": -6.027235984802246,
      "logps/rejected": -31.36172103881836,
      "loss": 0.1475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.617992877960205,
      "rewards/margins": 2.617992877960205,
      "rewards/rejected": 0.0,
      "step": 730
    },
    {
      "epoch": 4.083798882681564,
      "grad_norm": 6.446054969982822,
      "learning_rate": 4.642630421134808e-06,
      "logits/chosen": -3.528900384902954,
      "logits/rejected": -3.2422120571136475,
      "logps/chosen": -3.1293368339538574,
      "logps/rejected": -59.9114990234375,
      "loss": 0.1874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8860950469970703,
      "rewards/margins": 1.8860950469970703,
      "rewards/rejected": 0.0,
      "step": 731
    },
    {
      "epoch": 4.089385474860335,
      "grad_norm": 7.618306938138089,
      "learning_rate": 4.641058552245926e-06,
      "logits/chosen": -3.479328155517578,
      "logits/rejected": -3.630209445953369,
      "logps/chosen": -3.9469428062438965,
      "logps/rejected": -78.94319915771484,
      "loss": 0.1628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2718093395233154,
      "rewards/margins": 1.2718093395233154,
      "rewards/rejected": 0.0,
      "step": 732
    },
    {
      "epoch": 4.094972067039106,
      "grad_norm": 5.180172809559568,
      "learning_rate": 4.639483501362733e-06,
      "logits/chosen": -3.786367654800415,
      "logits/rejected": -3.8787198066711426,
      "logps/chosen": -10.175238609313965,
      "logps/rejected": -58.82747268676758,
      "loss": 0.1811,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8415729999542236,
      "rewards/margins": 1.8415729999542236,
      "rewards/rejected": 0.0,
      "step": 733
    },
    {
      "epoch": 4.100558659217877,
      "grad_norm": 2.3681895652349696,
      "learning_rate": 4.637905270826037e-06,
      "logits/chosen": -3.6310243606567383,
      "logits/rejected": -3.6082985401153564,
      "logps/chosen": -4.227251052856445,
      "logps/rejected": -52.22579574584961,
      "loss": 0.1026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2357778549194336,
      "rewards/margins": 2.2357778549194336,
      "rewards/rejected": 0.0,
      "step": 734
    },
    {
      "epoch": 4.106145251396648,
      "grad_norm": 5.504253693230685,
      "learning_rate": 4.636323862981367e-06,
      "logits/chosen": -3.5543110370635986,
      "logits/rejected": -3.700530529022217,
      "logps/chosen": -27.071819305419922,
      "logps/rejected": -45.268775939941406,
      "loss": 0.1697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.201329231262207,
      "rewards/margins": 3.201329231262207,
      "rewards/rejected": 0.0,
      "step": 735
    },
    {
      "epoch": 4.111731843575419,
      "grad_norm": 13.23800083633401,
      "learning_rate": 4.634739280178979e-06,
      "logits/chosen": -3.685631036758423,
      "logits/rejected": -3.4392623901367188,
      "logps/chosen": -16.671146392822266,
      "logps/rejected": -38.8029899597168,
      "loss": 0.2099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.3527164459228516,
      "rewards/margins": 2.3527164459228516,
      "rewards/rejected": 0.0,
      "step": 736
    },
    {
      "epoch": 4.11731843575419,
      "grad_norm": 3.7935636903481034,
      "learning_rate": 4.633151524773844e-06,
      "logits/chosen": -3.419198989868164,
      "logits/rejected": -3.5870704650878906,
      "logps/chosen": -9.473308563232422,
      "logps/rejected": -50.56467056274414,
      "loss": 0.1473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5725209712982178,
      "rewards/margins": 2.5725209712982178,
      "rewards/rejected": 0.0,
      "step": 737
    },
    {
      "epoch": 4.122905027932961,
      "grad_norm": 4.728934027507981,
      "learning_rate": 4.6315605991256495e-06,
      "logits/chosen": -3.6122803688049316,
      "logits/rejected": -3.5838589668273926,
      "logps/chosen": -8.12095832824707,
      "logps/rejected": -39.781089782714844,
      "loss": 0.1326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9667471647262573,
      "rewards/margins": 1.9667471647262573,
      "rewards/rejected": 0.0,
      "step": 738
    },
    {
      "epoch": 4.128491620111732,
      "grad_norm": 5.284337986899528,
      "learning_rate": 4.629966505598795e-06,
      "logits/chosen": -3.7463555335998535,
      "logits/rejected": -3.7387025356292725,
      "logps/chosen": -9.790915489196777,
      "logps/rejected": -46.62009048461914,
      "loss": 0.152,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9321821928024292,
      "rewards/margins": 0.9321821928024292,
      "rewards/rejected": 0.0,
      "step": 739
    },
    {
      "epoch": 4.134078212290503,
      "grad_norm": 9.368252977940006,
      "learning_rate": 4.628369246562387e-06,
      "logits/chosen": -3.4574930667877197,
      "logits/rejected": -3.70601487159729,
      "logps/chosen": -20.119129180908203,
      "logps/rejected": -59.305763244628906,
      "loss": 0.1889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.977313995361328,
      "rewards/margins": 2.977313995361328,
      "rewards/rejected": 0.0,
      "step": 740
    },
    {
      "epoch": 4.134078212290503,
      "eval_logits/chosen": -3.4196033477783203,
      "eval_logits/rejected": -3.5256614685058594,
      "eval_logps/chosen": -30.847646713256836,
      "eval_logps/rejected": -45.86442947387695,
      "eval_loss": 1.0261096954345703,
      "eval_rewards/accuracies": 0.4749999940395355,
      "eval_rewards/chosen": -0.07776059210300446,
      "eval_rewards/margins": -0.07776059210300446,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5177,
      "eval_samples_per_second": 12.148,
      "eval_steps_per_second": 0.392,
      "step": 740
    },
    {
      "epoch": 4.139664804469274,
      "grad_norm": 12.964141741872293,
      "learning_rate": 4.6267688243902364e-06,
      "logits/chosen": -3.7445197105407715,
      "logits/rejected": -3.797344446182251,
      "logps/chosen": -11.749675750732422,
      "logps/rejected": -39.702392578125,
      "loss": 0.2496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8981637954711914,
      "rewards/margins": 2.8981637954711914,
      "rewards/rejected": 0.0,
      "step": 741
    },
    {
      "epoch": 4.145251396648045,
      "grad_norm": 3.823503641567035,
      "learning_rate": 4.625165241460856e-06,
      "logits/chosen": -3.6251561641693115,
      "logits/rejected": -3.518676280975342,
      "logps/chosen": -1.2292946577072144,
      "logps/rejected": -87.60292053222656,
      "loss": 0.1442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.212498664855957,
      "rewards/margins": 1.212498664855957,
      "rewards/rejected": 0.0,
      "step": 742
    },
    {
      "epoch": 4.150837988826815,
      "grad_norm": 8.110196963739636,
      "learning_rate": 4.623558500157456e-06,
      "logits/chosen": -3.64036226272583,
      "logits/rejected": -3.5662267208099365,
      "logps/chosen": -10.91348934173584,
      "logps/rejected": -34.58348083496094,
      "loss": 0.2076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4766712188720703,
      "rewards/margins": 2.4766712188720703,
      "rewards/rejected": 0.0,
      "step": 743
    },
    {
      "epoch": 4.156424581005586,
      "grad_norm": 14.426351120657197,
      "learning_rate": 4.621948602867939e-06,
      "logits/chosen": -3.8273398876190186,
      "logits/rejected": -3.8245506286621094,
      "logps/chosen": -2.5250821113586426,
      "logps/rejected": -73.19476318359375,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6244632005691528,
      "rewards/margins": 1.6244632005691528,
      "rewards/rejected": 0.0,
      "step": 744
    },
    {
      "epoch": 4.162011173184357,
      "grad_norm": 3.453796053308548,
      "learning_rate": 4.620335551984901e-06,
      "logits/chosen": -3.438783884048462,
      "logits/rejected": -3.3697266578674316,
      "logps/chosen": -5.788082122802734,
      "logps/rejected": -51.19171905517578,
      "loss": 0.1137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.594414472579956,
      "rewards/margins": 1.594414472579956,
      "rewards/rejected": 0.0,
      "step": 745
    },
    {
      "epoch": 4.167597765363128,
      "grad_norm": 7.020518654414756,
      "learning_rate": 4.61871934990562e-06,
      "logits/chosen": -3.476973056793213,
      "logits/rejected": -3.5621016025543213,
      "logps/chosen": -9.30459976196289,
      "logps/rejected": -45.40360641479492,
      "loss": 0.1432,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.172651767730713,
      "rewards/margins": 2.172651767730713,
      "rewards/rejected": 0.0,
      "step": 746
    },
    {
      "epoch": 4.173184357541899,
      "grad_norm": 4.878776285412827,
      "learning_rate": 4.617099999032063e-06,
      "logits/chosen": -3.8005568981170654,
      "logits/rejected": -3.5242912769317627,
      "logps/chosen": -6.869067192077637,
      "logps/rejected": -63.71710968017578,
      "loss": 0.1579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.97243332862854,
      "rewards/margins": 1.97243332862854,
      "rewards/rejected": 0.0,
      "step": 747
    },
    {
      "epoch": 4.17877094972067,
      "grad_norm": 5.524736370893323,
      "learning_rate": 4.615477501770872e-06,
      "logits/chosen": -3.529799461364746,
      "logits/rejected": -3.658822536468506,
      "logps/chosen": -4.1674065589904785,
      "logps/rejected": -37.839805603027344,
      "loss": 0.1542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1230050325393677,
      "rewards/margins": 1.1230050325393677,
      "rewards/rejected": 0.0,
      "step": 748
    },
    {
      "epoch": 4.184357541899441,
      "grad_norm": 11.443049733910817,
      "learning_rate": 4.613851860533367e-06,
      "logits/chosen": -3.304138660430908,
      "logits/rejected": -3.2540645599365234,
      "logps/chosen": -18.536678314208984,
      "logps/rejected": -42.10388946533203,
      "loss": 0.1928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.461616277694702,
      "rewards/margins": 2.461616277694702,
      "rewards/rejected": 0.0,
      "step": 749
    },
    {
      "epoch": 4.189944134078212,
      "grad_norm": 4.657492354938152,
      "learning_rate": 4.612223077735542e-06,
      "logits/chosen": -3.509897470474243,
      "logits/rejected": -3.7284507751464844,
      "logps/chosen": -37.30125045776367,
      "logps/rejected": -41.02810287475586,
      "loss": 0.1615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4014248847961426,
      "rewards/margins": 3.4014248847961426,
      "rewards/rejected": 0.0,
      "step": 750
    },
    {
      "epoch": 4.195530726256983,
      "grad_norm": 7.56213472823233,
      "learning_rate": 4.6105911557980555e-06,
      "logits/chosen": -3.5114822387695312,
      "logits/rejected": -3.4443068504333496,
      "logps/chosen": -14.989419937133789,
      "logps/rejected": -45.328338623046875,
      "loss": 0.1742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0449581146240234,
      "rewards/margins": 3.0449581146240234,
      "rewards/rejected": 0.0,
      "step": 751
    },
    {
      "epoch": 4.201117318435754,
      "grad_norm": 2.8431753096950456,
      "learning_rate": 4.608956097146237e-06,
      "logits/chosen": -3.231712818145752,
      "logits/rejected": -3.3731861114501953,
      "logps/chosen": -3.266383647918701,
      "logps/rejected": -54.156333923339844,
      "loss": 0.136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7703007459640503,
      "rewards/margins": 1.7703007459640503,
      "rewards/rejected": 0.0,
      "step": 752
    },
    {
      "epoch": 4.206703910614525,
      "grad_norm": 5.17866743089783,
      "learning_rate": 4.607317904210073e-06,
      "logits/chosen": -3.8302419185638428,
      "logits/rejected": -3.782683849334717,
      "logps/chosen": -8.01800537109375,
      "logps/rejected": -39.0274772644043,
      "loss": 0.1596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8854498863220215,
      "rewards/margins": 2.8854498863220215,
      "rewards/rejected": 0.0,
      "step": 753
    },
    {
      "epoch": 4.212290502793296,
      "grad_norm": 5.347113674003841,
      "learning_rate": 4.6056765794242115e-06,
      "logits/chosen": -3.3751325607299805,
      "logits/rejected": -3.314589738845825,
      "logps/chosen": -8.231046676635742,
      "logps/rejected": -44.134403228759766,
      "loss": 0.1793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5858817100524902,
      "rewards/margins": 2.5858817100524902,
      "rewards/rejected": 0.0,
      "step": 754
    },
    {
      "epoch": 4.217877094972067,
      "grad_norm": 3.7284123059136585,
      "learning_rate": 4.604032125227952e-06,
      "logits/chosen": -3.6316702365875244,
      "logits/rejected": -3.776193857192993,
      "logps/chosen": -2.664914608001709,
      "logps/rejected": -48.23897933959961,
      "loss": 0.1421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9719038009643555,
      "rewards/margins": 1.9719038009643555,
      "rewards/rejected": 0.0,
      "step": 755
    },
    {
      "epoch": 4.223463687150838,
      "grad_norm": 3.8913042876124124,
      "learning_rate": 4.602384544065246e-06,
      "logits/chosen": -3.7124440670013428,
      "logits/rejected": -3.7693710327148438,
      "logps/chosen": -11.9215669631958,
      "logps/rejected": -66.02908325195312,
      "loss": 0.1233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.095400810241699,
      "rewards/margins": 2.095400810241699,
      "rewards/rejected": 0.0,
      "step": 756
    },
    {
      "epoch": 4.229050279329609,
      "grad_norm": 4.499199113685603,
      "learning_rate": 4.600733838384694e-06,
      "logits/chosen": -3.4422457218170166,
      "logits/rejected": -3.425696611404419,
      "logps/chosen": -26.179672241210938,
      "logps/rejected": -33.7657470703125,
      "loss": 0.1479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.917656421661377,
      "rewards/margins": 2.917656421661377,
      "rewards/rejected": 0.0,
      "step": 757
    },
    {
      "epoch": 4.23463687150838,
      "grad_norm": 7.814337757611798,
      "learning_rate": 4.5990800106395385e-06,
      "logits/chosen": -3.3880958557128906,
      "logits/rejected": -3.4926555156707764,
      "logps/chosen": -11.32736587524414,
      "logps/rejected": -43.08003234863281,
      "loss": 0.1962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8268505334854126,
      "rewards/margins": 1.8268505334854126,
      "rewards/rejected": 0.0,
      "step": 758
    },
    {
      "epoch": 4.240223463687151,
      "grad_norm": 3.7992532020632344,
      "learning_rate": 4.597423063287661e-06,
      "logits/chosen": -3.885162115097046,
      "logits/rejected": -3.6814022064208984,
      "logps/chosen": -1.9421697854995728,
      "logps/rejected": -36.19578170776367,
      "loss": 0.1496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9025909900665283,
      "rewards/margins": 1.9025909900665283,
      "rewards/rejected": 0.0,
      "step": 759
    },
    {
      "epoch": 4.245810055865922,
      "grad_norm": 6.461561270362227,
      "learning_rate": 4.595762998791581e-06,
      "logits/chosen": -3.7774322032928467,
      "logits/rejected": -3.5304605960845947,
      "logps/chosen": -7.293344497680664,
      "logps/rejected": -47.9549674987793,
      "loss": 0.1635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.099780559539795,
      "rewards/margins": 2.099780559539795,
      "rewards/rejected": 0.0,
      "step": 760
    },
    {
      "epoch": 4.245810055865922,
      "eval_logits/chosen": -3.4183037281036377,
      "eval_logits/rejected": -3.515153169631958,
      "eval_logps/chosen": -30.27382469177246,
      "eval_logps/rejected": -45.69470977783203,
      "eval_loss": 0.9247696399688721,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.02037840522825718,
      "eval_rewards/margins": -0.02037840522825718,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5897,
      "eval_samples_per_second": 12.114,
      "eval_steps_per_second": 0.391,
      "step": 760
    },
    {
      "epoch": 4.251396648044693,
      "grad_norm": 8.999366670198453,
      "learning_rate": 4.5940998196184485e-06,
      "logits/chosen": -3.5138940811157227,
      "logits/rejected": -3.698422431945801,
      "logps/chosen": -4.331122875213623,
      "logps/rejected": -50.36311721801758,
      "loss": 0.185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.359017848968506,
      "rewards/margins": 2.359017848968506,
      "rewards/rejected": 0.0,
      "step": 761
    },
    {
      "epoch": 4.256983240223463,
      "grad_norm": 4.7892818561654815,
      "learning_rate": 4.5924335282400465e-06,
      "logits/chosen": -3.336611270904541,
      "logits/rejected": -3.2645246982574463,
      "logps/chosen": -10.229825973510742,
      "logps/rejected": -42.96570587158203,
      "loss": 0.1425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9447665214538574,
      "rewards/margins": 2.9447665214538574,
      "rewards/rejected": 0.0,
      "step": 762
    },
    {
      "epoch": 4.262569832402234,
      "grad_norm": 5.522358484927956,
      "learning_rate": 4.590764127132779e-06,
      "logits/chosen": -3.542924642562866,
      "logits/rejected": -3.369382858276367,
      "logps/chosen": -12.09218978881836,
      "logps/rejected": -59.889060974121094,
      "loss": 0.1405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.137403726577759,
      "rewards/margins": 2.137403726577759,
      "rewards/rejected": 0.0,
      "step": 763
    },
    {
      "epoch": 4.268156424581005,
      "grad_norm": 6.9282857450826985,
      "learning_rate": 4.589091618777674e-06,
      "logits/chosen": -3.6814801692962646,
      "logits/rejected": -3.302891969680786,
      "logps/chosen": -5.0992045402526855,
      "logps/rejected": -48.20362091064453,
      "loss": 0.1783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6393470764160156,
      "rewards/margins": 1.6393470764160156,
      "rewards/rejected": 0.0,
      "step": 764
    },
    {
      "epoch": 4.273743016759776,
      "grad_norm": 7.308360625966536,
      "learning_rate": 4.587416005660378e-06,
      "logits/chosen": -3.4182393550872803,
      "logits/rejected": -3.68766450881958,
      "logps/chosen": -3.517066717147827,
      "logps/rejected": -86.84505462646484,
      "loss": 0.1804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.926872730255127,
      "rewards/margins": 1.926872730255127,
      "rewards/rejected": 0.0,
      "step": 765
    },
    {
      "epoch": 4.279329608938547,
      "grad_norm": 4.248688434108273,
      "learning_rate": 4.5857372902711475e-06,
      "logits/chosen": -3.92085862159729,
      "logits/rejected": -3.789999008178711,
      "logps/chosen": -6.135182857513428,
      "logps/rejected": -30.363670349121094,
      "loss": 0.134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8278977870941162,
      "rewards/margins": 1.8278977870941162,
      "rewards/rejected": 0.0,
      "step": 766
    },
    {
      "epoch": 4.284916201117318,
      "grad_norm": 11.383759037156098,
      "learning_rate": 4.584055475104855e-06,
      "logits/chosen": -3.588992118835449,
      "logits/rejected": -3.8538219928741455,
      "logps/chosen": -25.169408798217773,
      "logps/rejected": -61.727806091308594,
      "loss": 0.1981,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0431370735168457,
      "rewards/margins": 2.0431370735168457,
      "rewards/rejected": 0.0,
      "step": 767
    },
    {
      "epoch": 4.290502793296089,
      "grad_norm": 5.485864330403245,
      "learning_rate": 4.582370562660976e-06,
      "logits/chosen": -3.5173323154449463,
      "logits/rejected": -3.6817309856414795,
      "logps/chosen": -8.158233642578125,
      "logps/rejected": -100.23316192626953,
      "loss": 0.181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2456960678100586,
      "rewards/margins": 2.2456960678100586,
      "rewards/rejected": 0.0,
      "step": 768
    },
    {
      "epoch": 4.29608938547486,
      "grad_norm": 12.129305507238893,
      "learning_rate": 4.580682555443591e-06,
      "logits/chosen": -3.315359354019165,
      "logits/rejected": -3.421220541000366,
      "logps/chosen": -8.743600845336914,
      "logps/rejected": -48.458984375,
      "loss": 0.3274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0199007987976074,
      "rewards/margins": 2.0199007987976074,
      "rewards/rejected": 0.0,
      "step": 769
    },
    {
      "epoch": 4.301675977653631,
      "grad_norm": 4.476944544770358,
      "learning_rate": 4.5789914559613796e-06,
      "logits/chosen": -3.658980131149292,
      "logits/rejected": -3.5479958057403564,
      "logps/chosen": -7.4240031242370605,
      "logps/rejected": -38.65718078613281,
      "loss": 0.1463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5969409942626953,
      "rewards/margins": 2.5969409942626953,
      "rewards/rejected": 0.0,
      "step": 770
    },
    {
      "epoch": 4.307262569832402,
      "grad_norm": 7.263920263379756,
      "learning_rate": 4.577297266727616e-06,
      "logits/chosen": -3.882490396499634,
      "logits/rejected": -3.552567481994629,
      "logps/chosen": -3.782008647918701,
      "logps/rejected": -33.42122268676758,
      "loss": 0.1777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6532124280929565,
      "rewards/margins": 1.6532124280929565,
      "rewards/rejected": 0.0,
      "step": 771
    },
    {
      "epoch": 4.312849162011173,
      "grad_norm": 8.808923728260966,
      "learning_rate": 4.575599990260168e-06,
      "logits/chosen": -3.5970852375030518,
      "logits/rejected": -3.4698290824890137,
      "logps/chosen": -10.213488578796387,
      "logps/rejected": -53.910011291503906,
      "loss": 0.2005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.334353446960449,
      "rewards/margins": 2.334353446960449,
      "rewards/rejected": 0.0,
      "step": 772
    },
    {
      "epoch": 4.318435754189944,
      "grad_norm": 3.932984590637169,
      "learning_rate": 4.573899629081491e-06,
      "logits/chosen": -3.2475461959838867,
      "logits/rejected": -3.431431531906128,
      "logps/chosen": -9.153251647949219,
      "logps/rejected": -51.40351867675781,
      "loss": 0.1287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7945611476898193,
      "rewards/margins": 1.7945611476898193,
      "rewards/rejected": 0.0,
      "step": 773
    },
    {
      "epoch": 4.324022346368715,
      "grad_norm": 7.440995226555152,
      "learning_rate": 4.572196185718623e-06,
      "logits/chosen": -3.5288379192352295,
      "logits/rejected": -3.5778770446777344,
      "logps/chosen": -3.2063539028167725,
      "logps/rejected": -42.104705810546875,
      "loss": 0.1592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9111814498901367,
      "rewards/margins": 1.9111814498901367,
      "rewards/rejected": 0.0,
      "step": 774
    },
    {
      "epoch": 4.329608938547486,
      "grad_norm": 18.394138467261744,
      "learning_rate": 4.570489662703184e-06,
      "logits/chosen": -3.3323185443878174,
      "logits/rejected": -3.448967456817627,
      "logps/chosen": -3.1711177825927734,
      "logps/rejected": -65.63610076904297,
      "loss": 0.3935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8211991786956787,
      "rewards/margins": 1.8211991786956787,
      "rewards/rejected": 0.0,
      "step": 775
    },
    {
      "epoch": 4.335195530726257,
      "grad_norm": 4.241233454052833,
      "learning_rate": 4.568780062571375e-06,
      "logits/chosen": -3.4868080615997314,
      "logits/rejected": -3.6634633541107178,
      "logps/chosen": -8.581903457641602,
      "logps/rejected": -55.4521369934082,
      "loss": 0.1397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.323792576789856,
      "rewards/margins": 1.323792576789856,
      "rewards/rejected": 0.0,
      "step": 776
    },
    {
      "epoch": 4.340782122905028,
      "grad_norm": 5.827126889445191,
      "learning_rate": 4.567067387863961e-06,
      "logits/chosen": -3.5679357051849365,
      "logits/rejected": -3.341092348098755,
      "logps/chosen": -10.466950416564941,
      "logps/rejected": -43.48613739013672,
      "loss": 0.1532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.653645634651184,
      "rewards/margins": 1.653645634651184,
      "rewards/rejected": 0.0,
      "step": 777
    },
    {
      "epoch": 4.346368715083799,
      "grad_norm": 8.811852179413448,
      "learning_rate": 4.565351641126286e-06,
      "logits/chosen": -3.261965274810791,
      "logits/rejected": -3.0991013050079346,
      "logps/chosen": -6.030784606933594,
      "logps/rejected": -43.567047119140625,
      "loss": 0.2495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6377742290496826,
      "rewards/margins": 2.6377742290496826,
      "rewards/rejected": 0.0,
      "step": 778
    },
    {
      "epoch": 4.351955307262569,
      "grad_norm": 6.6208072381289265,
      "learning_rate": 4.563632824908252e-06,
      "logits/chosen": -3.4095544815063477,
      "logits/rejected": -3.6149168014526367,
      "logps/chosen": -10.059089660644531,
      "logps/rejected": -57.033935546875,
      "loss": 0.1955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.209472417831421,
      "rewards/margins": 2.209472417831421,
      "rewards/rejected": 0.0,
      "step": 779
    },
    {
      "epoch": 4.35754189944134,
      "grad_norm": 4.656957723945557,
      "learning_rate": 4.561910941764327e-06,
      "logits/chosen": -3.8294358253479004,
      "logits/rejected": -3.518510341644287,
      "logps/chosen": -5.713505268096924,
      "logps/rejected": -58.99755859375,
      "loss": 0.1442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3393452167510986,
      "rewards/margins": 1.3393452167510986,
      "rewards/rejected": 0.0,
      "step": 780
    },
    {
      "epoch": 4.35754189944134,
      "eval_logits/chosen": -3.3423850536346436,
      "eval_logits/rejected": -3.4174256324768066,
      "eval_logps/chosen": -29.111072540283203,
      "eval_logps/rejected": -43.13530349731445,
      "eval_loss": 0.8410378694534302,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": 0.09589698910713196,
      "eval_rewards/margins": 0.09589698910713196,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6028,
      "eval_samples_per_second": 12.108,
      "eval_steps_per_second": 0.391,
      "step": 780
    },
    {
      "epoch": 4.363128491620111,
      "grad_norm": 10.748195734240081,
      "learning_rate": 4.560185994253537e-06,
      "logits/chosen": -3.2885923385620117,
      "logits/rejected": -3.24104380607605,
      "logps/chosen": -33.014686584472656,
      "logps/rejected": -49.976776123046875,
      "loss": 0.2981,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.030259132385254,
      "rewards/margins": 2.030259132385254,
      "rewards/rejected": 0.0,
      "step": 781
    },
    {
      "epoch": 4.368715083798882,
      "grad_norm": 3.8080025458368962,
      "learning_rate": 4.558457984939461e-06,
      "logits/chosen": -3.7148869037628174,
      "logits/rejected": -3.623781442642212,
      "logps/chosen": -15.828756332397461,
      "logps/rejected": -42.97254943847656,
      "loss": 0.1423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.876286506652832,
      "rewards/margins": 2.876286506652832,
      "rewards/rejected": 0.0,
      "step": 782
    },
    {
      "epoch": 4.374301675977653,
      "grad_norm": 5.563597817013053,
      "learning_rate": 4.556726916390227e-06,
      "logits/chosen": -3.6977977752685547,
      "logits/rejected": -3.7069873809814453,
      "logps/chosen": -8.037297248840332,
      "logps/rejected": -37.902000427246094,
      "loss": 0.1752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4641661643981934,
      "rewards/margins": 2.4641661643981934,
      "rewards/rejected": 0.0,
      "step": 783
    },
    {
      "epoch": 4.379888268156424,
      "grad_norm": 6.4668859591727665,
      "learning_rate": 4.554992791178512e-06,
      "logits/chosen": -3.426896095275879,
      "logits/rejected": -3.3399391174316406,
      "logps/chosen": -7.846275329589844,
      "logps/rejected": -49.81325149536133,
      "loss": 0.1618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.331403970718384,
      "rewards/margins": 2.331403970718384,
      "rewards/rejected": 0.0,
      "step": 784
    },
    {
      "epoch": 4.385474860335195,
      "grad_norm": 4.974936000876878,
      "learning_rate": 4.553255611881536e-06,
      "logits/chosen": -3.697094440460205,
      "logits/rejected": -3.637458562850952,
      "logps/chosen": -9.601686477661133,
      "logps/rejected": -50.54014587402344,
      "loss": 0.1696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4411652088165283,
      "rewards/margins": 2.4411652088165283,
      "rewards/rejected": 0.0,
      "step": 785
    },
    {
      "epoch": 4.391061452513966,
      "grad_norm": 5.741153731358423,
      "learning_rate": 4.551515381081056e-06,
      "logits/chosen": -3.4347853660583496,
      "logits/rejected": -3.2223236560821533,
      "logps/chosen": -10.963174819946289,
      "logps/rejected": -38.647560119628906,
      "loss": 0.2015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6849664449691772,
      "rewards/margins": 1.6849664449691772,
      "rewards/rejected": 0.0,
      "step": 786
    },
    {
      "epoch": 4.396648044692737,
      "grad_norm": 6.526230008241431,
      "learning_rate": 4.549772101363365e-06,
      "logits/chosen": -3.712618589401245,
      "logits/rejected": -3.5774834156036377,
      "logps/chosen": -15.668438911437988,
      "logps/rejected": -30.03592300415039,
      "loss": 0.1627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.648077964782715,
      "rewards/margins": 3.648077964782715,
      "rewards/rejected": 0.0,
      "step": 787
    },
    {
      "epoch": 4.402234636871508,
      "grad_norm": 3.5056579115396103,
      "learning_rate": 4.548025775319286e-06,
      "logits/chosen": -3.5784480571746826,
      "logits/rejected": -3.541321039199829,
      "logps/chosen": -34.610450744628906,
      "logps/rejected": -40.07965850830078,
      "loss": 0.1045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1514639854431152,
      "rewards/margins": 3.1514639854431152,
      "rewards/rejected": 0.0,
      "step": 788
    },
    {
      "epoch": 4.407821229050279,
      "grad_norm": 4.7542791408475695,
      "learning_rate": 4.546276405544171e-06,
      "logits/chosen": -3.622431755065918,
      "logits/rejected": -3.364588499069214,
      "logps/chosen": -9.141159057617188,
      "logps/rejected": -36.428157806396484,
      "loss": 0.2131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.277698278427124,
      "rewards/margins": 2.277698278427124,
      "rewards/rejected": 0.0,
      "step": 789
    },
    {
      "epoch": 4.41340782122905,
      "grad_norm": 4.627061765239611,
      "learning_rate": 4.544523994637897e-06,
      "logits/chosen": -3.41998028755188,
      "logits/rejected": -3.4370405673980713,
      "logps/chosen": -11.152835845947266,
      "logps/rejected": -40.41335678100586,
      "loss": 0.1614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5879727602005005,
      "rewards/margins": 1.5879727602005005,
      "rewards/rejected": 0.0,
      "step": 790
    },
    {
      "epoch": 4.418994413407821,
      "grad_norm": 3.1691258628602363,
      "learning_rate": 4.542768545204856e-06,
      "logits/chosen": -3.592027425765991,
      "logits/rejected": -3.404817819595337,
      "logps/chosen": -8.004694938659668,
      "logps/rejected": -62.956878662109375,
      "loss": 0.1148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3351407051086426,
      "rewards/margins": 2.3351407051086426,
      "rewards/rejected": 0.0,
      "step": 791
    },
    {
      "epoch": 4.424581005586592,
      "grad_norm": 7.014277172850563,
      "learning_rate": 4.54101005985396e-06,
      "logits/chosen": -3.8215904235839844,
      "logits/rejected": -3.7280097007751465,
      "logps/chosen": -12.775678634643555,
      "logps/rejected": -61.69499969482422,
      "loss": 0.1728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.10404109954834,
      "rewards/margins": 2.10404109954834,
      "rewards/rejected": 0.0,
      "step": 792
    },
    {
      "epoch": 4.430167597765363,
      "grad_norm": 2.8904738974448145,
      "learning_rate": 4.539248541198631e-06,
      "logits/chosen": -3.6949996948242188,
      "logits/rejected": -3.5375735759735107,
      "logps/chosen": -10.347400665283203,
      "logps/rejected": -34.014163970947266,
      "loss": 0.1225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.854975700378418,
      "rewards/margins": 2.854975700378418,
      "rewards/rejected": 0.0,
      "step": 793
    },
    {
      "epoch": 4.435754189944134,
      "grad_norm": 6.322593519347095,
      "learning_rate": 4.5374839918568e-06,
      "logits/chosen": -3.5431032180786133,
      "logits/rejected": -3.4151551723480225,
      "logps/chosen": -24.45657730102539,
      "logps/rejected": -34.406410217285156,
      "loss": 0.1483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.983802080154419,
      "rewards/margins": 1.983802080154419,
      "rewards/rejected": 0.0,
      "step": 794
    },
    {
      "epoch": 4.441340782122905,
      "grad_norm": 43.505915945650976,
      "learning_rate": 4.5357164144509e-06,
      "logits/chosen": -3.7695469856262207,
      "logits/rejected": -3.739135980606079,
      "logps/chosen": -5.70670223236084,
      "logps/rejected": -38.66884994506836,
      "loss": 0.426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0645461082458496,
      "rewards/margins": 2.0645461082458496,
      "rewards/rejected": 0.0,
      "step": 795
    },
    {
      "epoch": 4.446927374301676,
      "grad_norm": 5.363865836038713,
      "learning_rate": 4.533945811607868e-06,
      "logits/chosen": -3.388658046722412,
      "logits/rejected": -3.607560873031616,
      "logps/chosen": -35.0988655090332,
      "logps/rejected": -61.766273498535156,
      "loss": 0.1831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.521124839782715,
      "rewards/margins": 2.521124839782715,
      "rewards/rejected": 0.0,
      "step": 796
    },
    {
      "epoch": 4.452513966480447,
      "grad_norm": 6.237130961498338,
      "learning_rate": 4.532172185959134e-06,
      "logits/chosen": -2.785365104675293,
      "logits/rejected": -2.871819019317627,
      "logps/chosen": -6.866209030151367,
      "logps/rejected": -78.14918518066406,
      "loss": 0.1646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6445467472076416,
      "rewards/margins": 2.6445467472076416,
      "rewards/rejected": 0.0,
      "step": 797
    },
    {
      "epoch": 4.4581005586592175,
      "grad_norm": 4.432780741807198,
      "learning_rate": 4.53039554014062e-06,
      "logits/chosen": -3.5233161449432373,
      "logits/rejected": -3.475404977798462,
      "logps/chosen": -13.901037216186523,
      "logps/rejected": -66.92015075683594,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9501750469207764,
      "rewards/margins": 2.9501750469207764,
      "rewards/rejected": 0.0,
      "step": 798
    },
    {
      "epoch": 4.4636871508379885,
      "grad_norm": 5.051425561016868,
      "learning_rate": 4.528615876792742e-06,
      "logits/chosen": -3.73816180229187,
      "logits/rejected": -2.9326372146606445,
      "logps/chosen": -9.275007247924805,
      "logps/rejected": -55.50544738769531,
      "loss": 0.1658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.86026668548584,
      "rewards/margins": 2.86026668548584,
      "rewards/rejected": 0.0,
      "step": 799
    },
    {
      "epoch": 4.4692737430167595,
      "grad_norm": 6.932986145686692,
      "learning_rate": 4.526833198560393e-06,
      "logits/chosen": -3.4073750972747803,
      "logits/rejected": -3.370490074157715,
      "logps/chosen": -5.512110710144043,
      "logps/rejected": -49.006954193115234,
      "loss": 0.1926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9388720989227295,
      "rewards/margins": 1.9388720989227295,
      "rewards/rejected": 0.0,
      "step": 800
    },
    {
      "epoch": 4.4692737430167595,
      "eval_logits/chosen": -3.432863235473633,
      "eval_logits/rejected": -3.535919189453125,
      "eval_logps/chosen": -31.406314849853516,
      "eval_logps/rejected": -45.291961669921875,
      "eval_loss": 0.9903203248977661,
      "eval_rewards/accuracies": 0.44999998807907104,
      "eval_rewards/chosen": -0.13362719118595123,
      "eval_rewards/margins": -0.13362719118595123,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.4978,
      "eval_samples_per_second": 12.158,
      "eval_steps_per_second": 0.392,
      "step": 800
    },
    {
      "epoch": 4.4748603351955305,
      "grad_norm": 7.958309665336296,
      "learning_rate": 4.5250475080929515e-06,
      "logits/chosen": -3.669896125793457,
      "logits/rejected": -3.989187717437744,
      "logps/chosen": -8.833820343017578,
      "logps/rejected": -58.351680755615234,
      "loss": 0.1497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0519280433654785,
      "rewards/margins": 3.0519280433654785,
      "rewards/rejected": 0.0,
      "step": 801
    },
    {
      "epoch": 4.4804469273743015,
      "grad_norm": 5.083035096309042,
      "learning_rate": 4.523258808044273e-06,
      "logits/chosen": -3.601163625717163,
      "logits/rejected": -3.6458792686462402,
      "logps/chosen": -8.707113265991211,
      "logps/rejected": -50.957298278808594,
      "loss": 0.1557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.092027187347412,
      "rewards/margins": 2.092027187347412,
      "rewards/rejected": 0.0,
      "step": 802
    },
    {
      "epoch": 4.4860335195530725,
      "grad_norm": 12.988916002042368,
      "learning_rate": 4.521467101072682e-06,
      "logits/chosen": -3.5886282920837402,
      "logits/rejected": -3.2359437942504883,
      "logps/chosen": -14.957812309265137,
      "logps/rejected": -44.580238342285156,
      "loss": 0.23,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4060349464416504,
      "rewards/margins": 3.4060349464416504,
      "rewards/rejected": 0.0,
      "step": 803
    },
    {
      "epoch": 4.4916201117318435,
      "grad_norm": 7.155108731211374,
      "learning_rate": 4.519672389840975e-06,
      "logits/chosen": -3.1861355304718018,
      "logits/rejected": -3.448056697845459,
      "logps/chosen": -6.659432411193848,
      "logps/rejected": -66.4361801147461,
      "loss": 0.1991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.636836051940918,
      "rewards/margins": 1.636836051940918,
      "rewards/rejected": 0.0,
      "step": 804
    },
    {
      "epoch": 4.4972067039106145,
      "grad_norm": 7.128284725026907,
      "learning_rate": 4.517874677016413e-06,
      "logits/chosen": -3.3891377449035645,
      "logits/rejected": -3.4213337898254395,
      "logps/chosen": -12.352534294128418,
      "logps/rejected": -47.50074768066406,
      "loss": 0.1844,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.145017147064209,
      "rewards/margins": 1.145017147064209,
      "rewards/rejected": 0.0,
      "step": 805
    },
    {
      "epoch": 4.5027932960893855,
      "grad_norm": 10.05690109594537,
      "learning_rate": 4.516073965270718e-06,
      "logits/chosen": -3.4710004329681396,
      "logits/rejected": -3.6723508834838867,
      "logps/chosen": -5.352405548095703,
      "logps/rejected": -34.9871826171875,
      "loss": 0.18,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.439537525177002,
      "rewards/margins": 2.439537525177002,
      "rewards/rejected": 0.0,
      "step": 806
    },
    {
      "epoch": 4.5083798882681565,
      "grad_norm": 8.069497965469598,
      "learning_rate": 4.5142702572800665e-06,
      "logits/chosen": -3.555736541748047,
      "logits/rejected": -3.683929920196533,
      "logps/chosen": -6.084300518035889,
      "logps/rejected": -59.178619384765625,
      "loss": 0.2383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.214970111846924,
      "rewards/margins": 2.214970111846924,
      "rewards/rejected": 0.0,
      "step": 807
    },
    {
      "epoch": 4.5139664804469275,
      "grad_norm": 4.440638777146202,
      "learning_rate": 4.51246355572509e-06,
      "logits/chosen": -3.3699798583984375,
      "logits/rejected": -3.5721771717071533,
      "logps/chosen": -69.50880432128906,
      "logps/rejected": -37.43304443359375,
      "loss": 0.1587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.501441717147827,
      "rewards/margins": 3.501441717147827,
      "rewards/rejected": 0.0,
      "step": 808
    },
    {
      "epoch": 4.5195530726256985,
      "grad_norm": 4.695582473017856,
      "learning_rate": 4.510653863290871e-06,
      "logits/chosen": -3.0967493057250977,
      "logits/rejected": -3.10191011428833,
      "logps/chosen": -28.943971633911133,
      "logps/rejected": -65.74657440185547,
      "loss": 0.1826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3763203620910645,
      "rewards/margins": 3.3763203620910645,
      "rewards/rejected": 0.0,
      "step": 809
    },
    {
      "epoch": 4.5251396648044695,
      "grad_norm": 8.173316058443149,
      "learning_rate": 4.508841182666933e-06,
      "logits/chosen": -3.4618868827819824,
      "logits/rejected": -3.5124573707580566,
      "logps/chosen": -4.964077472686768,
      "logps/rejected": -32.115234375,
      "loss": 0.1857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3473126888275146,
      "rewards/margins": 2.3473126888275146,
      "rewards/rejected": 0.0,
      "step": 810
    },
    {
      "epoch": 4.5307262569832405,
      "grad_norm": 5.47260214197143,
      "learning_rate": 4.507025516547242e-06,
      "logits/chosen": -3.3460071086883545,
      "logits/rejected": -3.4204041957855225,
      "logps/chosen": -11.278915405273438,
      "logps/rejected": -48.433135986328125,
      "loss": 0.152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9342643022537231,
      "rewards/margins": 1.9342643022537231,
      "rewards/rejected": 0.0,
      "step": 811
    },
    {
      "epoch": 4.5363128491620115,
      "grad_norm": 4.3644698617404245,
      "learning_rate": 4.505206867630203e-06,
      "logits/chosen": -3.326866388320923,
      "logits/rejected": -3.1453001499176025,
      "logps/chosen": -12.149857521057129,
      "logps/rejected": -58.000389099121094,
      "loss": 0.1562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.417417049407959,
      "rewards/margins": 2.417417049407959,
      "rewards/rejected": 0.0,
      "step": 812
    },
    {
      "epoch": 4.5418994413407825,
      "grad_norm": 6.552862956823418,
      "learning_rate": 4.503385238618649e-06,
      "logits/chosen": -3.4385714530944824,
      "logits/rejected": -3.3496274948120117,
      "logps/chosen": -5.9897966384887695,
      "logps/rejected": -35.332515716552734,
      "loss": 0.2124,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.28525972366333,
      "rewards/margins": 1.28525972366333,
      "rewards/rejected": 0.0,
      "step": 813
    },
    {
      "epoch": 4.547486033519553,
      "grad_norm": 11.779006174498347,
      "learning_rate": 4.501560632219849e-06,
      "logits/chosen": -3.6021764278411865,
      "logits/rejected": -3.3206019401550293,
      "logps/chosen": -13.646133422851562,
      "logps/rejected": -52.922691345214844,
      "loss": 0.2347,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5295689105987549,
      "rewards/margins": 0.5295689105987549,
      "rewards/rejected": 0.0,
      "step": 814
    },
    {
      "epoch": 4.553072625698324,
      "grad_norm": 3.3501198467956765,
      "learning_rate": 4.49973305114549e-06,
      "logits/chosen": -3.6991167068481445,
      "logits/rejected": -3.5438170433044434,
      "logps/chosen": -7.237292289733887,
      "logps/rejected": -39.32611846923828,
      "loss": 0.1468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0487377643585205,
      "rewards/margins": 2.0487377643585205,
      "rewards/rejected": 0.0,
      "step": 815
    },
    {
      "epoch": 4.558659217877095,
      "grad_norm": 4.944625824259073,
      "learning_rate": 4.497902498111684e-06,
      "logits/chosen": -3.9190452098846436,
      "logits/rejected": -3.839730978012085,
      "logps/chosen": -13.003854751586914,
      "logps/rejected": -46.78937530517578,
      "loss": 0.1574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1785929203033447,
      "rewards/margins": 2.1785929203033447,
      "rewards/rejected": 0.0,
      "step": 816
    },
    {
      "epoch": 4.564245810055866,
      "grad_norm": 5.441184188996401,
      "learning_rate": 4.496068975838959e-06,
      "logits/chosen": -3.1634275913238525,
      "logits/rejected": -3.2864935398101807,
      "logps/chosen": -6.289309978485107,
      "logps/rejected": -31.61695671081543,
      "loss": 0.1909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2701427936553955,
      "rewards/margins": 1.2701427936553955,
      "rewards/rejected": 0.0,
      "step": 817
    },
    {
      "epoch": 4.569832402234637,
      "grad_norm": 3.63480233775248,
      "learning_rate": 4.494232487052255e-06,
      "logits/chosen": -3.6363184452056885,
      "logits/rejected": -3.8714144229888916,
      "logps/chosen": -4.632479190826416,
      "logps/rejected": -64.41532135009766,
      "loss": 0.1507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.950472116470337,
      "rewards/margins": 1.950472116470337,
      "rewards/rejected": 0.0,
      "step": 818
    },
    {
      "epoch": 4.575418994413408,
      "grad_norm": 5.190532883628688,
      "learning_rate": 4.492393034480922e-06,
      "logits/chosen": -3.814868450164795,
      "logits/rejected": -3.7208340167999268,
      "logps/chosen": -12.437049865722656,
      "logps/rejected": -45.61674499511719,
      "loss": 0.1681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.759939432144165,
      "rewards/margins": 2.759939432144165,
      "rewards/rejected": 0.0,
      "step": 819
    },
    {
      "epoch": 4.581005586592179,
      "grad_norm": 5.5583698766491905,
      "learning_rate": 4.490550620858714e-06,
      "logits/chosen": -3.303574800491333,
      "logits/rejected": -3.3813412189483643,
      "logps/chosen": -8.311309814453125,
      "logps/rejected": -34.451534271240234,
      "loss": 0.1986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6539678573608398,
      "rewards/margins": 1.6539678573608398,
      "rewards/rejected": 0.0,
      "step": 820
    },
    {
      "epoch": 4.581005586592179,
      "eval_logits/chosen": -3.3460261821746826,
      "eval_logits/rejected": -3.4311435222625732,
      "eval_logps/chosen": -30.220592498779297,
      "eval_logps/rejected": -44.55207061767578,
      "eval_loss": 0.8617421984672546,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.015055114403367043,
      "eval_rewards/margins": -0.015055114403367043,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.504,
      "eval_samples_per_second": 12.155,
      "eval_steps_per_second": 0.392,
      "step": 820
    },
    {
      "epoch": 4.58659217877095,
      "grad_norm": 4.567225765795936,
      "learning_rate": 4.488705248923786e-06,
      "logits/chosen": -3.6829471588134766,
      "logits/rejected": -3.6627907752990723,
      "logps/chosen": -11.213857650756836,
      "logps/rejected": -46.315696716308594,
      "loss": 0.1328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.047987461090088,
      "rewards/margins": 2.047987461090088,
      "rewards/rejected": 0.0,
      "step": 821
    },
    {
      "epoch": 4.592178770949721,
      "grad_norm": 8.580964486209572,
      "learning_rate": 4.486856921418689e-06,
      "logits/chosen": -3.56483793258667,
      "logits/rejected": -3.5969200134277344,
      "logps/chosen": -11.652937889099121,
      "logps/rejected": -42.954586029052734,
      "loss": 0.168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.419696807861328,
      "rewards/margins": 2.419696807861328,
      "rewards/rejected": 0.0,
      "step": 822
    },
    {
      "epoch": 4.597765363128492,
      "grad_norm": 3.347404132208324,
      "learning_rate": 4.485005641090367e-06,
      "logits/chosen": -3.4424848556518555,
      "logits/rejected": -3.6411924362182617,
      "logps/chosen": -20.754655838012695,
      "logps/rejected": -46.96409606933594,
      "loss": 0.1369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8786420822143555,
      "rewards/margins": 2.8786420822143555,
      "rewards/rejected": 0.0,
      "step": 823
    },
    {
      "epoch": 4.603351955307263,
      "grad_norm": 4.493600772373729,
      "learning_rate": 4.483151410690152e-06,
      "logits/chosen": -3.577258825302124,
      "logits/rejected": -3.5579521656036377,
      "logps/chosen": -5.536375045776367,
      "logps/rejected": -35.210899353027344,
      "loss": 0.155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.498645782470703,
      "rewards/margins": 2.498645782470703,
      "rewards/rejected": 0.0,
      "step": 824
    },
    {
      "epoch": 4.608938547486034,
      "grad_norm": 3.7285682483055287,
      "learning_rate": 4.481294232973761e-06,
      "logits/chosen": -3.4677071571350098,
      "logits/rejected": -3.2879984378814697,
      "logps/chosen": -4.209889888763428,
      "logps/rejected": -84.07865905761719,
      "loss": 0.1394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5696327686309814,
      "rewards/margins": 1.5696327686309814,
      "rewards/rejected": 0.0,
      "step": 825
    },
    {
      "epoch": 4.614525139664805,
      "grad_norm": 4.966865848785732,
      "learning_rate": 4.479434110701291e-06,
      "logits/chosen": -3.5029399394989014,
      "logits/rejected": -3.5557947158813477,
      "logps/chosen": -11.800997734069824,
      "logps/rejected": -24.311660766601562,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1881508827209473,
      "rewards/margins": 2.1881508827209473,
      "rewards/rejected": 0.0,
      "step": 826
    },
    {
      "epoch": 4.620111731843576,
      "grad_norm": 8.760852997846886,
      "learning_rate": 4.477571046637214e-06,
      "logits/chosen": -3.4234578609466553,
      "logits/rejected": -3.555323362350464,
      "logps/chosen": -3.2018136978149414,
      "logps/rejected": -34.516571044921875,
      "loss": 0.164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3222732543945312,
      "rewards/margins": 2.3222732543945312,
      "rewards/rejected": 0.0,
      "step": 827
    },
    {
      "epoch": 4.625698324022347,
      "grad_norm": 5.168920367675221,
      "learning_rate": 4.475705043550375e-06,
      "logits/chosen": -3.662780523300171,
      "logits/rejected": -3.5774598121643066,
      "logps/chosen": -4.625075817108154,
      "logps/rejected": -35.35499954223633,
      "loss": 0.1732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3628840446472168,
      "rewards/margins": 1.3628840446472168,
      "rewards/rejected": 0.0,
      "step": 828
    },
    {
      "epoch": 4.631284916201118,
      "grad_norm": 4.36899462465956,
      "learning_rate": 4.473836104213988e-06,
      "logits/chosen": -3.7704780101776123,
      "logits/rejected": -3.7042551040649414,
      "logps/chosen": -5.994783401489258,
      "logps/rejected": -54.699058532714844,
      "loss": 0.1697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.163841724395752,
      "rewards/margins": 2.163841724395752,
      "rewards/rejected": 0.0,
      "step": 829
    },
    {
      "epoch": 4.636871508379889,
      "grad_norm": 4.515335843900256,
      "learning_rate": 4.47196423140563e-06,
      "logits/chosen": -3.7195310592651367,
      "logits/rejected": -3.8611786365509033,
      "logps/chosen": -8.832235336303711,
      "logps/rejected": -57.376651763916016,
      "loss": 0.1438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3616466522216797,
      "rewards/margins": 3.3616466522216797,
      "rewards/rejected": 0.0,
      "step": 830
    },
    {
      "epoch": 4.64245810055866,
      "grad_norm": 12.972189245794127,
      "learning_rate": 4.470089427907235e-06,
      "logits/chosen": -3.6764965057373047,
      "logits/rejected": -3.538334846496582,
      "logps/chosen": -12.100198745727539,
      "logps/rejected": -43.89472579956055,
      "loss": 0.231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.143765926361084,
      "rewards/margins": 3.143765926361084,
      "rewards/rejected": 0.0,
      "step": 831
    },
    {
      "epoch": 4.648044692737431,
      "grad_norm": 5.866091708028377,
      "learning_rate": 4.468211696505097e-06,
      "logits/chosen": -2.896703004837036,
      "logits/rejected": -2.9673259258270264,
      "logps/chosen": -25.107187271118164,
      "logps/rejected": -64.75505828857422,
      "loss": 0.1454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5357792377471924,
      "rewards/margins": 2.5357792377471924,
      "rewards/rejected": 0.0,
      "step": 832
    },
    {
      "epoch": 4.653631284916202,
      "grad_norm": 5.273111556132747,
      "learning_rate": 4.466331039989859e-06,
      "logits/chosen": -3.54569935798645,
      "logits/rejected": -3.6018974781036377,
      "logps/chosen": -5.556939125061035,
      "logps/rejected": -78.42101287841797,
      "loss": 0.1814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8788775205612183,
      "rewards/margins": 1.8788775205612183,
      "rewards/rejected": 0.0,
      "step": 833
    },
    {
      "epoch": 4.659217877094972,
      "grad_norm": 9.665405911713666,
      "learning_rate": 4.4644474611565115e-06,
      "logits/chosen": -3.6431727409362793,
      "logits/rejected": -3.677957773208618,
      "logps/chosen": -23.10557746887207,
      "logps/rejected": -40.454139709472656,
      "loss": 0.1987,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1673779487609863,
      "rewards/margins": 2.1673779487609863,
      "rewards/rejected": 0.0,
      "step": 834
    },
    {
      "epoch": 4.664804469273743,
      "grad_norm": 8.03077215510493,
      "learning_rate": 4.4625609628043864e-06,
      "logits/chosen": -3.9515976905822754,
      "logits/rejected": -3.5435686111450195,
      "logps/chosen": -7.970238208770752,
      "logps/rejected": -69.06492614746094,
      "loss": 0.2323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6625804901123047,
      "rewards/margins": 2.6625804901123047,
      "rewards/rejected": 0.0,
      "step": 835
    },
    {
      "epoch": 4.670391061452514,
      "grad_norm": 6.109097037928799,
      "learning_rate": 4.460671547737159e-06,
      "logits/chosen": -3.470027446746826,
      "logits/rejected": -3.3134069442749023,
      "logps/chosen": -12.160486221313477,
      "logps/rejected": -53.2240104675293,
      "loss": 0.1517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3445682525634766,
      "rewards/margins": 2.3445682525634766,
      "rewards/rejected": 0.0,
      "step": 836
    },
    {
      "epoch": 4.675977653631285,
      "grad_norm": 4.121899024334273,
      "learning_rate": 4.458779218762835e-06,
      "logits/chosen": -3.6297037601470947,
      "logits/rejected": -3.716716766357422,
      "logps/chosen": -5.029935836791992,
      "logps/rejected": -69.54499816894531,
      "loss": 0.1377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6580475568771362,
      "rewards/margins": 1.6580475568771362,
      "rewards/rejected": 0.0,
      "step": 837
    },
    {
      "epoch": 4.681564245810056,
      "grad_norm": 7.641425726277085,
      "learning_rate": 4.456883978693751e-06,
      "logits/chosen": -3.914440393447876,
      "logits/rejected": -3.8704893589019775,
      "logps/chosen": -9.39675521850586,
      "logps/rejected": -51.1124267578125,
      "loss": 0.1663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3109235763549805,
      "rewards/margins": 2.3109235763549805,
      "rewards/rejected": 0.0,
      "step": 838
    },
    {
      "epoch": 4.687150837988827,
      "grad_norm": 7.022666078692328,
      "learning_rate": 4.454985830346574e-06,
      "logits/chosen": -3.3482158184051514,
      "logits/rejected": -3.388627767562866,
      "logps/chosen": -13.688318252563477,
      "logps/rejected": -36.16783142089844,
      "loss": 0.1716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.6636836528778076,
      "rewards/margins": 1.6636836528778076,
      "rewards/rejected": 0.0,
      "step": 839
    },
    {
      "epoch": 4.692737430167598,
      "grad_norm": 5.700171060973575,
      "learning_rate": 4.453084776542289e-06,
      "logits/chosen": -3.4032411575317383,
      "logits/rejected": -3.335221290588379,
      "logps/chosen": -3.7911863327026367,
      "logps/rejected": -39.794166564941406,
      "loss": 0.2175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7186732292175293,
      "rewards/margins": 1.7186732292175293,
      "rewards/rejected": 0.0,
      "step": 840
    },
    {
      "epoch": 4.692737430167598,
      "eval_logits/chosen": -3.3765406608581543,
      "eval_logits/rejected": -3.4661331176757812,
      "eval_logps/chosen": -30.733261108398438,
      "eval_logps/rejected": -46.72361373901367,
      "eval_loss": 0.9028280973434448,
      "eval_rewards/accuracies": 0.44999998807907104,
      "eval_rewards/chosen": -0.06632177531719208,
      "eval_rewards/margins": -0.06632177531719208,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5039,
      "eval_samples_per_second": 12.155,
      "eval_steps_per_second": 0.392,
      "step": 840
    },
    {
      "epoch": 4.698324022346369,
      "grad_norm": 5.510964519351278,
      "learning_rate": 4.4511808201062e-06,
      "logits/chosen": -3.6178505420684814,
      "logits/rejected": -3.4158897399902344,
      "logps/chosen": -11.858049392700195,
      "logps/rejected": -30.43700408935547,
      "loss": 0.1505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5604734420776367,
      "rewards/margins": 2.5604734420776367,
      "rewards/rejected": 0.0,
      "step": 841
    },
    {
      "epoch": 4.70391061452514,
      "grad_norm": 5.78265470941794,
      "learning_rate": 4.449273963867927e-06,
      "logits/chosen": -3.3635616302490234,
      "logits/rejected": -3.323319911956787,
      "logps/chosen": -17.927745819091797,
      "logps/rejected": -40.528167724609375,
      "loss": 0.1652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.8268284797668457,
      "rewards/margins": 2.8268284797668457,
      "rewards/rejected": 0.0,
      "step": 842
    },
    {
      "epoch": 4.709497206703911,
      "grad_norm": 5.3383264172135325,
      "learning_rate": 4.447364210661397e-06,
      "logits/chosen": -3.36578631401062,
      "logits/rejected": -3.6942832469940186,
      "logps/chosen": -24.739791870117188,
      "logps/rejected": -57.483360290527344,
      "loss": 0.1797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4605016708374023,
      "rewards/margins": 3.4605016708374023,
      "rewards/rejected": 0.0,
      "step": 843
    },
    {
      "epoch": 4.715083798882682,
      "grad_norm": 4.90242444562919,
      "learning_rate": 4.445451563324843e-06,
      "logits/chosen": -3.481358289718628,
      "logits/rejected": -3.3932955265045166,
      "logps/chosen": -6.308778285980225,
      "logps/rejected": -34.19693374633789,
      "loss": 0.1597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0635993480682373,
      "rewards/margins": 2.0635993480682373,
      "rewards/rejected": 0.0,
      "step": 844
    },
    {
      "epoch": 4.720670391061453,
      "grad_norm": 4.279186220055138,
      "learning_rate": 4.4435360247008e-06,
      "logits/chosen": -3.6067802906036377,
      "logits/rejected": -3.4772329330444336,
      "logps/chosen": -8.091681480407715,
      "logps/rejected": -39.62975311279297,
      "loss": 0.1633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8796370029449463,
      "rewards/margins": 1.8796370029449463,
      "rewards/rejected": 0.0,
      "step": 845
    },
    {
      "epoch": 4.726256983240224,
      "grad_norm": 3.6805951113561117,
      "learning_rate": 4.441617597636099e-06,
      "logits/chosen": -3.535287380218506,
      "logits/rejected": -3.555997371673584,
      "logps/chosen": -6.684229373931885,
      "logps/rejected": -45.94219207763672,
      "loss": 0.119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4643118381500244,
      "rewards/margins": 2.4643118381500244,
      "rewards/rejected": 0.0,
      "step": 846
    },
    {
      "epoch": 4.731843575418995,
      "grad_norm": 5.226489436900743,
      "learning_rate": 4.439696284981865e-06,
      "logits/chosen": -3.400815486907959,
      "logits/rejected": -3.2702043056488037,
      "logps/chosen": -9.221874237060547,
      "logps/rejected": -42.89576721191406,
      "loss": 0.2089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.686913013458252,
      "rewards/margins": 2.686913013458252,
      "rewards/rejected": 0.0,
      "step": 847
    },
    {
      "epoch": 4.737430167597766,
      "grad_norm": 3.4669277427250424,
      "learning_rate": 4.437772089593511e-06,
      "logits/chosen": -3.9172112941741943,
      "logits/rejected": -3.947497844696045,
      "logps/chosen": -3.505849838256836,
      "logps/rejected": -37.838966369628906,
      "loss": 0.1593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1415975093841553,
      "rewards/margins": 2.1415975093841553,
      "rewards/rejected": 0.0,
      "step": 848
    },
    {
      "epoch": 4.743016759776537,
      "grad_norm": 6.596576409178242,
      "learning_rate": 4.4358450143307325e-06,
      "logits/chosen": -3.516836643218994,
      "logits/rejected": -3.2625770568847656,
      "logps/chosen": -22.528322219848633,
      "logps/rejected": -89.48059844970703,
      "loss": 0.1782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.277338743209839,
      "rewards/margins": 2.277338743209839,
      "rewards/rejected": 0.0,
      "step": 849
    },
    {
      "epoch": 4.748603351955307,
      "grad_norm": 4.8444319122096715,
      "learning_rate": 4.433915062057507e-06,
      "logits/chosen": -3.200207471847534,
      "logits/rejected": -3.399554491043091,
      "logps/chosen": -12.228487014770508,
      "logps/rejected": -47.353797912597656,
      "loss": 0.1556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9310407638549805,
      "rewards/margins": 2.9310407638549805,
      "rewards/rejected": 0.0,
      "step": 850
    },
    {
      "epoch": 4.754189944134078,
      "grad_norm": 4.394716446232864,
      "learning_rate": 4.43198223564209e-06,
      "logits/chosen": -3.7386152744293213,
      "logits/rejected": -3.6863863468170166,
      "logps/chosen": -5.444771766662598,
      "logps/rejected": -29.527118682861328,
      "loss": 0.1392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2304978370666504,
      "rewards/margins": 2.2304978370666504,
      "rewards/rejected": 0.0,
      "step": 851
    },
    {
      "epoch": 4.759776536312849,
      "grad_norm": 9.001135787254931,
      "learning_rate": 4.430046537957002e-06,
      "logits/chosen": -3.2451446056365967,
      "logits/rejected": -3.4211487770080566,
      "logps/chosen": -4.604584693908691,
      "logps/rejected": -91.31523895263672,
      "loss": 0.2192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6775479316711426,
      "rewards/margins": 1.6775479316711426,
      "rewards/rejected": 0.0,
      "step": 852
    },
    {
      "epoch": 4.76536312849162,
      "grad_norm": 5.574141163227111,
      "learning_rate": 4.4281079718790375e-06,
      "logits/chosen": -3.5288302898406982,
      "logits/rejected": -3.2170960903167725,
      "logps/chosen": -9.316011428833008,
      "logps/rejected": -85.23391723632812,
      "loss": 0.1488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.698702812194824,
      "rewards/margins": 2.698702812194824,
      "rewards/rejected": 0.0,
      "step": 853
    },
    {
      "epoch": 4.770949720670391,
      "grad_norm": 6.008558990613018,
      "learning_rate": 4.4261665402892484e-06,
      "logits/chosen": -3.516894817352295,
      "logits/rejected": -3.7632832527160645,
      "logps/chosen": -8.175775527954102,
      "logps/rejected": -27.49606704711914,
      "loss": 0.1851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0042643547058105,
      "rewards/margins": 2.0042643547058105,
      "rewards/rejected": 0.0,
      "step": 854
    },
    {
      "epoch": 4.776536312849162,
      "grad_norm": 9.37256577585607,
      "learning_rate": 4.42422224607295e-06,
      "logits/chosen": -3.6370151042938232,
      "logits/rejected": -3.792327404022217,
      "logps/chosen": -10.259377479553223,
      "logps/rejected": -47.191741943359375,
      "loss": 0.1959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1575818061828613,
      "rewards/margins": 3.1575818061828613,
      "rewards/rejected": 0.0,
      "step": 855
    },
    {
      "epoch": 4.782122905027933,
      "grad_norm": 3.931709755367258,
      "learning_rate": 4.422275092119708e-06,
      "logits/chosen": -3.8366658687591553,
      "logits/rejected": -3.7816245555877686,
      "logps/chosen": -23.79213523864746,
      "logps/rejected": -52.951904296875,
      "loss": 0.1378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.704868793487549,
      "rewards/margins": 2.704868793487549,
      "rewards/rejected": 0.0,
      "step": 856
    },
    {
      "epoch": 4.787709497206704,
      "grad_norm": 5.116089232200666,
      "learning_rate": 4.42032508132334e-06,
      "logits/chosen": -3.8075203895568848,
      "logits/rejected": -3.7658116817474365,
      "logps/chosen": -4.483695983886719,
      "logps/rejected": -54.25300216674805,
      "loss": 0.1565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.293574810028076,
      "rewards/margins": 2.293574810028076,
      "rewards/rejected": 0.0,
      "step": 857
    },
    {
      "epoch": 4.793296089385475,
      "grad_norm": 13.070802074523856,
      "learning_rate": 4.418372216581911e-06,
      "logits/chosen": -3.4802792072296143,
      "logits/rejected": -3.667097806930542,
      "logps/chosen": -24.265382766723633,
      "logps/rejected": -47.303497314453125,
      "loss": 0.2022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.381880760192871,
      "rewards/margins": 3.381880760192871,
      "rewards/rejected": 0.0,
      "step": 858
    },
    {
      "epoch": 4.798882681564246,
      "grad_norm": 4.629744987286383,
      "learning_rate": 4.4164165007977245e-06,
      "logits/chosen": -3.5322015285491943,
      "logits/rejected": -3.645059585571289,
      "logps/chosen": -5.371437072753906,
      "logps/rejected": -61.910911560058594,
      "loss": 0.1709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5554817914962769,
      "rewards/margins": 1.5554817914962769,
      "rewards/rejected": 0.0,
      "step": 859
    },
    {
      "epoch": 4.804469273743017,
      "grad_norm": 7.056361594696115,
      "learning_rate": 4.4144579368773226e-06,
      "logits/chosen": -3.837646484375,
      "logits/rejected": -3.8447318077087402,
      "logps/chosen": -7.276632785797119,
      "logps/rejected": -46.73405456542969,
      "loss": 0.1863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.620959997177124,
      "rewards/margins": 1.620959997177124,
      "rewards/rejected": 0.0,
      "step": 860
    },
    {
      "epoch": 4.804469273743017,
      "eval_logits/chosen": -3.437762498855591,
      "eval_logits/rejected": -3.5381481647491455,
      "eval_logps/chosen": -31.66622543334961,
      "eval_logps/rejected": -47.83777618408203,
      "eval_loss": 0.9416462779045105,
      "eval_rewards/accuracies": 0.375,
      "eval_rewards/chosen": -0.15961864590644836,
      "eval_rewards/margins": -0.15961864590644836,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5222,
      "eval_samples_per_second": 12.146,
      "eval_steps_per_second": 0.392,
      "step": 860
    },
    {
      "epoch": 4.810055865921788,
      "grad_norm": 4.435370585272868,
      "learning_rate": 4.41249652773148e-06,
      "logits/chosen": -3.3896336555480957,
      "logits/rejected": -3.565138816833496,
      "logps/chosen": -24.89761734008789,
      "logps/rejected": -58.513336181640625,
      "loss": 0.1499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8059606552124023,
      "rewards/margins": 2.8059606552124023,
      "rewards/rejected": 0.0,
      "step": 861
    },
    {
      "epoch": 4.815642458100559,
      "grad_norm": 5.153013793625424,
      "learning_rate": 4.4105322762752e-06,
      "logits/chosen": -3.338104486465454,
      "logits/rejected": -3.539729595184326,
      "logps/chosen": -6.965713024139404,
      "logps/rejected": -68.38004302978516,
      "loss": 0.1335,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.754666805267334,
      "rewards/margins": 1.754666805267334,
      "rewards/rejected": 0.0,
      "step": 862
    },
    {
      "epoch": 4.82122905027933,
      "grad_norm": 9.541498455274224,
      "learning_rate": 4.408565185427711e-06,
      "logits/chosen": -3.6362481117248535,
      "logits/rejected": -3.598694324493408,
      "logps/chosen": -6.158782958984375,
      "logps/rejected": -22.962934494018555,
      "loss": 0.2182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1622750759124756,
      "rewards/margins": 1.1622750759124756,
      "rewards/rejected": 0.0,
      "step": 863
    },
    {
      "epoch": 4.826815642458101,
      "grad_norm": 6.5001442510348575,
      "learning_rate": 4.4065952581124595e-06,
      "logits/chosen": -3.8102495670318604,
      "logits/rejected": -3.911731481552124,
      "logps/chosen": -12.48984432220459,
      "logps/rejected": -47.314208984375,
      "loss": 0.1835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4647347927093506,
      "rewards/margins": 3.4647347927093506,
      "rewards/rejected": 0.0,
      "step": 864
    },
    {
      "epoch": 4.832402234636872,
      "grad_norm": 3.7244263300149942,
      "learning_rate": 4.404622497257108e-06,
      "logits/chosen": -3.5598182678222656,
      "logits/rejected": -3.4809019565582275,
      "logps/chosen": -6.935900688171387,
      "logps/rejected": -65.215087890625,
      "loss": 0.1287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.572115421295166,
      "rewards/margins": 2.572115421295166,
      "rewards/rejected": 0.0,
      "step": 865
    },
    {
      "epoch": 4.837988826815643,
      "grad_norm": 3.8511021694845633,
      "learning_rate": 4.40264690579353e-06,
      "logits/chosen": -3.544790506362915,
      "logits/rejected": -3.6027307510375977,
      "logps/chosen": -4.8058247566223145,
      "logps/rejected": -58.449363708496094,
      "loss": 0.1337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4311633110046387,
      "rewards/margins": 1.4311633110046387,
      "rewards/rejected": 0.0,
      "step": 866
    },
    {
      "epoch": 4.843575418994414,
      "grad_norm": 4.657724770215925,
      "learning_rate": 4.4006684866578095e-06,
      "logits/chosen": -3.6006569862365723,
      "logits/rejected": -3.523280620574951,
      "logps/chosen": -11.990618705749512,
      "logps/rejected": -55.612396240234375,
      "loss": 0.1553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6077113151550293,
      "rewards/margins": 2.6077113151550293,
      "rewards/rejected": 0.0,
      "step": 867
    },
    {
      "epoch": 4.849162011173185,
      "grad_norm": 5.203852964596437,
      "learning_rate": 4.398687242790226e-06,
      "logits/chosen": -3.777958631515503,
      "logits/rejected": -3.6164255142211914,
      "logps/chosen": -8.537973403930664,
      "logps/rejected": -49.03691101074219,
      "loss": 0.16,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0001015663146973,
      "rewards/margins": 2.0001015663146973,
      "rewards/rejected": 0.0,
      "step": 868
    },
    {
      "epoch": 4.854748603351956,
      "grad_norm": 6.039263544252343,
      "learning_rate": 4.396703177135262e-06,
      "logits/chosen": -3.534933567047119,
      "logits/rejected": -3.6180620193481445,
      "logps/chosen": -13.429910659790039,
      "logps/rejected": -28.88335418701172,
      "loss": 0.193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.7732949256896973,
      "rewards/margins": 1.7732949256896973,
      "rewards/rejected": 0.0,
      "step": 869
    },
    {
      "epoch": 4.860335195530726,
      "grad_norm": 6.245138459401441,
      "learning_rate": 4.394716292641592e-06,
      "logits/chosen": -3.791564702987671,
      "logits/rejected": -3.772801399230957,
      "logps/chosen": -12.22524642944336,
      "logps/rejected": -34.59894943237305,
      "loss": 0.1513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0332536697387695,
      "rewards/margins": 2.0332536697387695,
      "rewards/rejected": 0.0,
      "step": 870
    },
    {
      "epoch": 4.865921787709497,
      "grad_norm": 5.085727680007054,
      "learning_rate": 4.392726592262081e-06,
      "logits/chosen": -3.357034921646118,
      "logits/rejected": -3.5604248046875,
      "logps/chosen": -6.254604339599609,
      "logps/rejected": -54.51686477661133,
      "loss": 0.2133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0267314910888672,
      "rewards/margins": 1.0267314910888672,
      "rewards/rejected": 0.0,
      "step": 871
    },
    {
      "epoch": 4.871508379888268,
      "grad_norm": 5.468860849344994,
      "learning_rate": 4.390734078953778e-06,
      "logits/chosen": -3.5024092197418213,
      "logits/rejected": -3.5227179527282715,
      "logps/chosen": -5.069463729858398,
      "logps/rejected": -36.81544494628906,
      "loss": 0.1694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3863284587860107,
      "rewards/margins": 2.3863284587860107,
      "rewards/rejected": 0.0,
      "step": 872
    },
    {
      "epoch": 4.877094972067039,
      "grad_norm": 3.516055317069953,
      "learning_rate": 4.388738755677912e-06,
      "logits/chosen": -3.3165283203125,
      "logits/rejected": -3.3143227100372314,
      "logps/chosen": -4.466382026672363,
      "logps/rejected": -44.37221145629883,
      "loss": 0.1417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5486836433410645,
      "rewards/margins": 2.5486836433410645,
      "rewards/rejected": 0.0,
      "step": 873
    },
    {
      "epoch": 4.88268156424581,
      "grad_norm": 3.2198156721558684,
      "learning_rate": 4.386740625399889e-06,
      "logits/chosen": -3.324317216873169,
      "logits/rejected": -3.129852771759033,
      "logps/chosen": -3.9313220977783203,
      "logps/rejected": -60.4588737487793,
      "loss": 0.1419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7394874095916748,
      "rewards/margins": 1.7394874095916748,
      "rewards/rejected": 0.0,
      "step": 874
    },
    {
      "epoch": 4.888268156424581,
      "grad_norm": 3.974329150059506,
      "learning_rate": 4.3847396910892855e-06,
      "logits/chosen": -3.653531789779663,
      "logits/rejected": -3.4467713832855225,
      "logps/chosen": -2.8808507919311523,
      "logps/rejected": -25.34185791015625,
      "loss": 0.1761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3893282413482666,
      "rewards/margins": 1.3893282413482666,
      "rewards/rejected": 0.0,
      "step": 875
    },
    {
      "epoch": 4.893854748603352,
      "grad_norm": 3.7111350913248256,
      "learning_rate": 4.382735955719849e-06,
      "logits/chosen": -3.36330509185791,
      "logits/rejected": -3.376004219055176,
      "logps/chosen": -33.12337875366211,
      "logps/rejected": -51.55621337890625,
      "loss": 0.1611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6818630695343018,
      "rewards/margins": 3.6818630695343018,
      "rewards/rejected": 0.0,
      "step": 876
    },
    {
      "epoch": 4.899441340782123,
      "grad_norm": 4.401011881064847,
      "learning_rate": 4.380729422269485e-06,
      "logits/chosen": -3.4204511642456055,
      "logits/rejected": -3.346039056777954,
      "logps/chosen": -11.711562156677246,
      "logps/rejected": -35.059974670410156,
      "loss": 0.1959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.533952236175537,
      "rewards/margins": 1.533952236175537,
      "rewards/rejected": 0.0,
      "step": 877
    },
    {
      "epoch": 4.905027932960894,
      "grad_norm": 5.04295443921708,
      "learning_rate": 4.37872009372026e-06,
      "logits/chosen": -3.707806348800659,
      "logits/rejected": -3.489685297012329,
      "logps/chosen": -5.4584808349609375,
      "logps/rejected": -34.37434387207031,
      "loss": 0.1744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7852680683135986,
      "rewards/margins": 1.7852680683135986,
      "rewards/rejected": 0.0,
      "step": 878
    },
    {
      "epoch": 4.910614525139665,
      "grad_norm": 8.52305964210566,
      "learning_rate": 4.376707973058394e-06,
      "logits/chosen": -3.4349093437194824,
      "logits/rejected": -3.6542513370513916,
      "logps/chosen": -4.90480899810791,
      "logps/rejected": -73.95443725585938,
      "loss": 0.1968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4834002256393433,
      "rewards/margins": 1.4834002256393433,
      "rewards/rejected": 0.0,
      "step": 879
    },
    {
      "epoch": 4.916201117318436,
      "grad_norm": 9.985085897365488,
      "learning_rate": 4.3746930632742555e-06,
      "logits/chosen": -3.6476714611053467,
      "logits/rejected": -3.7134151458740234,
      "logps/chosen": -5.312650680541992,
      "logps/rejected": -55.08071517944336,
      "loss": 0.2345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.297299385070801,
      "rewards/margins": 2.297299385070801,
      "rewards/rejected": 0.0,
      "step": 880
    },
    {
      "epoch": 4.916201117318436,
      "eval_logits/chosen": -3.404216766357422,
      "eval_logits/rejected": -3.5062038898468018,
      "eval_logps/chosen": -31.27678871154785,
      "eval_logps/rejected": -46.823570251464844,
      "eval_loss": 0.9496168494224548,
      "eval_rewards/accuracies": 0.4749999940395355,
      "eval_rewards/chosen": -0.12067488580942154,
      "eval_rewards/margins": -0.12067488580942154,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6109,
      "eval_samples_per_second": 12.104,
      "eval_steps_per_second": 0.39,
      "step": 880
    },
    {
      "epoch": 4.921787709497207,
      "grad_norm": 6.505226127553696,
      "learning_rate": 4.372675367362361e-06,
      "logits/chosen": -3.488298177719116,
      "logits/rejected": -3.4317123889923096,
      "logps/chosen": -6.669844627380371,
      "logps/rejected": -49.45069885253906,
      "loss": 0.1531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8048789501190186,
      "rewards/margins": 1.8048789501190186,
      "rewards/rejected": 0.0,
      "step": 881
    },
    {
      "epoch": 4.927374301675978,
      "grad_norm": 8.684939628616497,
      "learning_rate": 4.370654888321366e-06,
      "logits/chosen": -3.576824903488159,
      "logits/rejected": -3.4038543701171875,
      "logps/chosen": -4.42025089263916,
      "logps/rejected": -51.09926986694336,
      "loss": 0.1965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9177937507629395,
      "rewards/margins": 1.9177937507629395,
      "rewards/rejected": 0.0,
      "step": 882
    },
    {
      "epoch": 4.932960893854749,
      "grad_norm": 4.978743508611532,
      "learning_rate": 4.368631629154061e-06,
      "logits/chosen": -3.584071636199951,
      "logits/rejected": -3.5035879611968994,
      "logps/chosen": -11.814064025878906,
      "logps/rejected": -33.49521255493164,
      "loss": 0.1487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.01147723197937,
      "rewards/margins": 3.01147723197937,
      "rewards/rejected": 0.0,
      "step": 883
    },
    {
      "epoch": 4.93854748603352,
      "grad_norm": 3.7779599752115534,
      "learning_rate": 4.36660559286737e-06,
      "logits/chosen": -3.2487151622772217,
      "logits/rejected": -3.309112548828125,
      "logps/chosen": -14.13902759552002,
      "logps/rejected": -51.01299285888672,
      "loss": 0.1383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0246143341064453,
      "rewards/margins": 3.0246143341064453,
      "rewards/rejected": 0.0,
      "step": 884
    },
    {
      "epoch": 4.94413407821229,
      "grad_norm": 4.474110497521271,
      "learning_rate": 4.364576782472344e-06,
      "logits/chosen": -3.1249659061431885,
      "logits/rejected": -3.207642078399658,
      "logps/chosen": -10.470190048217773,
      "logps/rejected": -36.01239776611328,
      "loss": 0.1197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4056949615478516,
      "rewards/margins": 2.4056949615478516,
      "rewards/rejected": 0.0,
      "step": 885
    },
    {
      "epoch": 4.949720670391061,
      "grad_norm": 6.028976482063623,
      "learning_rate": 4.362545200984156e-06,
      "logits/chosen": -3.3269166946411133,
      "logits/rejected": -3.3040404319763184,
      "logps/chosen": -8.620523452758789,
      "logps/rejected": -37.12746810913086,
      "loss": 0.1684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.389794111251831,
      "rewards/margins": 2.389794111251831,
      "rewards/rejected": 0.0,
      "step": 886
    },
    {
      "epoch": 4.955307262569832,
      "grad_norm": 5.385209386892133,
      "learning_rate": 4.360510851422098e-06,
      "logits/chosen": -3.439875364303589,
      "logits/rejected": -3.4952027797698975,
      "logps/chosen": -7.177914619445801,
      "logps/rejected": -66.46358489990234,
      "loss": 0.163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.307950019836426,
      "rewards/margins": 2.307950019836426,
      "rewards/rejected": 0.0,
      "step": 887
    },
    {
      "epoch": 4.960893854748603,
      "grad_norm": 5.744682628279501,
      "learning_rate": 4.358473736809577e-06,
      "logits/chosen": -3.384629487991333,
      "logits/rejected": -3.6453051567077637,
      "logps/chosen": -9.515483856201172,
      "logps/rejected": -67.77054595947266,
      "loss": 0.1584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4659817218780518,
      "rewards/margins": 2.4659817218780518,
      "rewards/rejected": 0.0,
      "step": 888
    },
    {
      "epoch": 4.966480446927374,
      "grad_norm": 3.650450161547359,
      "learning_rate": 4.356433860174107e-06,
      "logits/chosen": -3.33927583694458,
      "logits/rejected": -3.304128408432007,
      "logps/chosen": -10.454925537109375,
      "logps/rejected": -42.55261993408203,
      "loss": 0.1137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5965826511383057,
      "rewards/margins": 2.5965826511383057,
      "rewards/rejected": 0.0,
      "step": 889
    },
    {
      "epoch": 4.972067039106145,
      "grad_norm": 3.0235725447407487,
      "learning_rate": 4.354391224547307e-06,
      "logits/chosen": -3.3841030597686768,
      "logits/rejected": -3.5240397453308105,
      "logps/chosen": -12.962348937988281,
      "logps/rejected": -33.8289794921875,
      "loss": 0.1168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5231566429138184,
      "rewards/margins": 2.5231566429138184,
      "rewards/rejected": 0.0,
      "step": 890
    },
    {
      "epoch": 4.977653631284916,
      "grad_norm": 5.899633555430878,
      "learning_rate": 4.3523458329648985e-06,
      "logits/chosen": -3.710519313812256,
      "logits/rejected": -3.620234489440918,
      "logps/chosen": -9.887506484985352,
      "logps/rejected": -43.831390380859375,
      "loss": 0.2044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.150543212890625,
      "rewards/margins": 2.150543212890625,
      "rewards/rejected": 0.0,
      "step": 891
    },
    {
      "epoch": 4.983240223463687,
      "grad_norm": 4.760001521402704,
      "learning_rate": 4.350297688466697e-06,
      "logits/chosen": -3.89351749420166,
      "logits/rejected": -3.5458366870880127,
      "logps/chosen": -10.359257698059082,
      "logps/rejected": -68.96419525146484,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3567605018615723,
      "rewards/margins": 2.3567605018615723,
      "rewards/rejected": 0.0,
      "step": 892
    },
    {
      "epoch": 4.988826815642458,
      "grad_norm": 5.801474298773948,
      "learning_rate": 4.348246794096611e-06,
      "logits/chosen": -3.3248038291931152,
      "logits/rejected": -3.38541316986084,
      "logps/chosen": -5.250064849853516,
      "logps/rejected": -52.668426513671875,
      "loss": 0.1641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7191441059112549,
      "rewards/margins": 1.7191441059112549,
      "rewards/rejected": 0.0,
      "step": 893
    },
    {
      "epoch": 4.994413407821229,
      "grad_norm": 7.7734171896899005,
      "learning_rate": 4.3461931529026344e-06,
      "logits/chosen": -3.0771517753601074,
      "logits/rejected": -3.0543019771575928,
      "logps/chosen": -31.026487350463867,
      "logps/rejected": -77.2094955444336,
      "loss": 0.1824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6994450092315674,
      "rewards/margins": 2.6994450092315674,
      "rewards/rejected": 0.0,
      "step": 894
    },
    {
      "epoch": 5.0,
      "grad_norm": 3.8172662185822914,
      "learning_rate": 4.3441367679368435e-06,
      "logits/chosen": -3.8555355072021484,
      "logits/rejected": -3.424746036529541,
      "logps/chosen": -5.971285820007324,
      "logps/rejected": -34.30250549316406,
      "loss": 0.1238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9980320930480957,
      "rewards/margins": 2.9980320930480957,
      "rewards/rejected": 0.0,
      "step": 895
    },
    {
      "epoch": 5.005586592178771,
      "grad_norm": 4.033014227666791,
      "learning_rate": 4.342077642255393e-06,
      "logits/chosen": -3.8003339767456055,
      "logits/rejected": -3.6959102153778076,
      "logps/chosen": -8.636877059936523,
      "logps/rejected": -36.94289016723633,
      "loss": 0.1321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7159206867218018,
      "rewards/margins": 2.7159206867218018,
      "rewards/rejected": 0.0,
      "step": 896
    },
    {
      "epoch": 5.011173184357542,
      "grad_norm": 2.293049941517156,
      "learning_rate": 4.340015778918509e-06,
      "logits/chosen": -3.192657470703125,
      "logits/rejected": -3.1753580570220947,
      "logps/chosen": -6.219178199768066,
      "logps/rejected": -66.87149047851562,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.107006788253784,
      "rewards/margins": 3.107006788253784,
      "rewards/rejected": 0.0,
      "step": 897
    },
    {
      "epoch": 5.016759776536313,
      "grad_norm": 4.2254424592664455,
      "learning_rate": 4.337951180990489e-06,
      "logits/chosen": -3.2983436584472656,
      "logits/rejected": -3.101539373397827,
      "logps/chosen": -4.120172023773193,
      "logps/rejected": -25.839111328125,
      "loss": 0.142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0798213481903076,
      "rewards/margins": 2.0798213481903076,
      "rewards/rejected": 0.0,
      "step": 898
    },
    {
      "epoch": 5.022346368715084,
      "grad_norm": 5.374202195036095,
      "learning_rate": 4.335883851539693e-06,
      "logits/chosen": -3.894176721572876,
      "logits/rejected": -3.9372172355651855,
      "logps/chosen": -18.211578369140625,
      "logps/rejected": -64.2633056640625,
      "loss": 0.1365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.111745834350586,
      "rewards/margins": 4.111745834350586,
      "rewards/rejected": 0.0,
      "step": 899
    },
    {
      "epoch": 5.027932960893855,
      "grad_norm": 8.371599594776805,
      "learning_rate": 4.333813793638541e-06,
      "logits/chosen": -3.784296751022339,
      "logits/rejected": -3.6467902660369873,
      "logps/chosen": -8.784801483154297,
      "logps/rejected": -72.01776123046875,
      "loss": 0.1759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.635469675064087,
      "rewards/margins": 1.635469675064087,
      "rewards/rejected": 0.0,
      "step": 900
    },
    {
      "epoch": 5.027932960893855,
      "eval_logits/chosen": -3.4269936084747314,
      "eval_logits/rejected": -3.534172773361206,
      "eval_logps/chosen": -32.638267517089844,
      "eval_logps/rejected": -48.70781326293945,
      "eval_loss": 1.0786598920822144,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.25682270526885986,
      "eval_rewards/margins": -0.25682270526885986,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5326,
      "eval_samples_per_second": 12.141,
      "eval_steps_per_second": 0.392,
      "step": 900
    },
    {
      "epoch": 5.033519553072626,
      "grad_norm": 4.204459283395101,
      "learning_rate": 4.3317410103635074e-06,
      "logits/chosen": -3.7769699096679688,
      "logits/rejected": -3.7580206394195557,
      "logps/chosen": -9.723791122436523,
      "logps/rejected": -53.89561080932617,
      "loss": 0.1343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.135910987854004,
      "rewards/margins": 3.135910987854004,
      "rewards/rejected": 0.0,
      "step": 901
    },
    {
      "epoch": 5.039106145251397,
      "grad_norm": 2.8655467471725875,
      "learning_rate": 4.329665504795116e-06,
      "logits/chosen": -3.8191001415252686,
      "logits/rejected": -3.825617790222168,
      "logps/chosen": -4.597505569458008,
      "logps/rejected": -92.31072998046875,
      "loss": 0.1251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.339341402053833,
      "rewards/margins": 2.339341402053833,
      "rewards/rejected": 0.0,
      "step": 902
    },
    {
      "epoch": 5.044692737430168,
      "grad_norm": 2.963917516458961,
      "learning_rate": 4.32758728001794e-06,
      "logits/chosen": -3.401822805404663,
      "logits/rejected": -3.557884931564331,
      "logps/chosen": -17.088092803955078,
      "logps/rejected": -38.396392822265625,
      "loss": 0.1465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2893524169921875,
      "rewards/margins": 3.2893524169921875,
      "rewards/rejected": 0.0,
      "step": 903
    },
    {
      "epoch": 5.050279329608939,
      "grad_norm": 3.551826928128554,
      "learning_rate": 4.32550633912059e-06,
      "logits/chosen": -3.453226089477539,
      "logits/rejected": -3.4691028594970703,
      "logps/chosen": -7.031257629394531,
      "logps/rejected": -51.04556655883789,
      "loss": 0.1522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8948187828063965,
      "rewards/margins": 1.8948187828063965,
      "rewards/rejected": 0.0,
      "step": 904
    },
    {
      "epoch": 5.055865921787709,
      "grad_norm": 3.130520378616442,
      "learning_rate": 4.3234226851957154e-06,
      "logits/chosen": -3.5532772541046143,
      "logits/rejected": -3.5023303031921387,
      "logps/chosen": -3.0045042037963867,
      "logps/rejected": -58.7540168762207,
      "loss": 0.1484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2155604362487793,
      "rewards/margins": 2.2155604362487793,
      "rewards/rejected": 0.0,
      "step": 905
    },
    {
      "epoch": 5.06145251396648,
      "grad_norm": 2.373362010273827,
      "learning_rate": 4.321336321339996e-06,
      "logits/chosen": -2.800563335418701,
      "logits/rejected": -3.0697693824768066,
      "logps/chosen": -34.652191162109375,
      "logps/rejected": -79.70478057861328,
      "loss": 0.1276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.011918306350708,
      "rewards/margins": 3.011918306350708,
      "rewards/rejected": 0.0,
      "step": 906
    },
    {
      "epoch": 5.067039106145251,
      "grad_norm": 4.9402969627022255,
      "learning_rate": 4.319247250654142e-06,
      "logits/chosen": -3.672640323638916,
      "logits/rejected": -3.510838747024536,
      "logps/chosen": -3.0196714401245117,
      "logps/rejected": -47.47083282470703,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.956772804260254,
      "rewards/margins": 1.956772804260254,
      "rewards/rejected": 0.0,
      "step": 907
    },
    {
      "epoch": 5.072625698324022,
      "grad_norm": 4.80351287126347,
      "learning_rate": 4.317155476242882e-06,
      "logits/chosen": -3.7719714641571045,
      "logits/rejected": -3.8798160552978516,
      "logps/chosen": -7.060386657714844,
      "logps/rejected": -54.43825149536133,
      "loss": 0.1417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2841546535491943,
      "rewards/margins": 2.2841546535491943,
      "rewards/rejected": 0.0,
      "step": 908
    },
    {
      "epoch": 5.078212290502793,
      "grad_norm": 4.7177894186750375,
      "learning_rate": 4.3150610012149675e-06,
      "logits/chosen": -3.8833365440368652,
      "logits/rejected": -3.8171563148498535,
      "logps/chosen": -2.245382308959961,
      "logps/rejected": -58.252750396728516,
      "loss": 0.1324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3930037021636963,
      "rewards/margins": 2.3930037021636963,
      "rewards/rejected": 0.0,
      "step": 909
    },
    {
      "epoch": 5.083798882681564,
      "grad_norm": 3.6820730967909396,
      "learning_rate": 4.312963828683161e-06,
      "logits/chosen": -3.5829851627349854,
      "logits/rejected": -3.561239242553711,
      "logps/chosen": -6.6173577308654785,
      "logps/rejected": -60.7462272644043,
      "loss": 0.1273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6491539478302,
      "rewards/margins": 2.6491539478302,
      "rewards/rejected": 0.0,
      "step": 910
    },
    {
      "epoch": 5.089385474860335,
      "grad_norm": 4.764574568293542,
      "learning_rate": 4.310863961764233e-06,
      "logits/chosen": -3.8031461238861084,
      "logits/rejected": -3.863579511642456,
      "logps/chosen": -8.787483215332031,
      "logps/rejected": -53.07954406738281,
      "loss": 0.1763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.938828468322754,
      "rewards/margins": 2.938828468322754,
      "rewards/rejected": 0.0,
      "step": 911
    },
    {
      "epoch": 5.094972067039106,
      "grad_norm": 4.76362933447919,
      "learning_rate": 4.308761403578961e-06,
      "logits/chosen": -3.6871817111968994,
      "logits/rejected": -3.812117338180542,
      "logps/chosen": -2.877894163131714,
      "logps/rejected": -59.29588317871094,
      "loss": 0.1254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2825021743774414,
      "rewards/margins": 2.2825021743774414,
      "rewards/rejected": 0.0,
      "step": 912
    },
    {
      "epoch": 5.100558659217877,
      "grad_norm": 12.639391371345303,
      "learning_rate": 4.30665615725212e-06,
      "logits/chosen": -3.6951026916503906,
      "logits/rejected": -3.7244160175323486,
      "logps/chosen": -3.751936912536621,
      "logps/rejected": -38.922542572021484,
      "loss": 0.1643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.495639681816101,
      "rewards/margins": 1.495639681816101,
      "rewards/rejected": 0.0,
      "step": 913
    },
    {
      "epoch": 5.106145251396648,
      "grad_norm": 2.9017303063448554,
      "learning_rate": 4.3045482259124814e-06,
      "logits/chosen": -3.4731087684631348,
      "logits/rejected": -3.2711215019226074,
      "logps/chosen": -3.1503729820251465,
      "logps/rejected": -46.007041931152344,
      "loss": 0.1383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7106764316558838,
      "rewards/margins": 1.7106764316558838,
      "rewards/rejected": 0.0,
      "step": 914
    },
    {
      "epoch": 5.111731843575419,
      "grad_norm": 3.8897439594868675,
      "learning_rate": 4.302437612692807e-06,
      "logits/chosen": -3.7562947273254395,
      "logits/rejected": -3.6341090202331543,
      "logps/chosen": -8.420059204101562,
      "logps/rejected": -49.18634796142578,
      "loss": 0.1181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4695091247558594,
      "rewards/margins": 2.4695091247558594,
      "rewards/rejected": 0.0,
      "step": 915
    },
    {
      "epoch": 5.11731843575419,
      "grad_norm": 8.89436863961101,
      "learning_rate": 4.300324320729844e-06,
      "logits/chosen": -3.7080318927764893,
      "logits/rejected": -3.7549662590026855,
      "logps/chosen": -8.799090385437012,
      "logps/rejected": -51.31471252441406,
      "loss": 0.1457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.4784018993377686,
      "rewards/margins": 1.4784018993377686,
      "rewards/rejected": 0.0,
      "step": 916
    },
    {
      "epoch": 5.122905027932961,
      "grad_norm": 6.050062338943923,
      "learning_rate": 4.298208353164319e-06,
      "logits/chosen": -3.426654815673828,
      "logits/rejected": -3.536771059036255,
      "logps/chosen": -3.368356943130493,
      "logps/rejected": -42.920814514160156,
      "loss": 0.1543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.830675721168518,
      "rewards/margins": 1.830675721168518,
      "rewards/rejected": 0.0,
      "step": 917
    },
    {
      "epoch": 5.128491620111732,
      "grad_norm": 4.492624325183401,
      "learning_rate": 4.296089713140939e-06,
      "logits/chosen": -3.657578945159912,
      "logits/rejected": -3.418917417526245,
      "logps/chosen": -5.132463455200195,
      "logps/rejected": -57.18238830566406,
      "loss": 0.144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.42518949508667,
      "rewards/margins": 2.42518949508667,
      "rewards/rejected": 0.0,
      "step": 918
    },
    {
      "epoch": 5.134078212290503,
      "grad_norm": 2.937772286907507,
      "learning_rate": 4.293968403808378e-06,
      "logits/chosen": -3.5139365196228027,
      "logits/rejected": -3.5795538425445557,
      "logps/chosen": -14.700274467468262,
      "logps/rejected": -40.12150573730469,
      "loss": 0.1394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.975299835205078,
      "rewards/margins": 2.975299835205078,
      "rewards/rejected": 0.0,
      "step": 919
    },
    {
      "epoch": 5.139664804469274,
      "grad_norm": 6.99696083167904,
      "learning_rate": 4.291844428319281e-06,
      "logits/chosen": -3.4343931674957275,
      "logits/rejected": -3.470061779022217,
      "logps/chosen": -10.290029525756836,
      "logps/rejected": -62.377166748046875,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8636016845703125,
      "rewards/margins": 1.8636016845703125,
      "rewards/rejected": 0.0,
      "step": 920
    },
    {
      "epoch": 5.139664804469274,
      "eval_logits/chosen": -3.424863338470459,
      "eval_logits/rejected": -3.5335965156555176,
      "eval_logps/chosen": -32.97644805908203,
      "eval_logps/rejected": -49.39509201049805,
      "eval_loss": 1.1253883838653564,
      "eval_rewards/accuracies": 0.375,
      "eval_rewards/chosen": -0.2906404435634613,
      "eval_rewards/margins": -0.2906404435634613,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6093,
      "eval_samples_per_second": 12.105,
      "eval_steps_per_second": 0.39,
      "step": 920
    },
    {
      "epoch": 5.145251396648045,
      "grad_norm": 3.2919462206642045,
      "learning_rate": 4.289717789830255e-06,
      "logits/chosen": -3.695216417312622,
      "logits/rejected": -3.735706329345703,
      "logps/chosen": -3.915534019470215,
      "logps/rejected": -58.35315704345703,
      "loss": 0.1423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.111645221710205,
      "rewards/margins": 2.111645221710205,
      "rewards/rejected": 0.0,
      "step": 921
    },
    {
      "epoch": 5.150837988826815,
      "grad_norm": 4.515727608656844,
      "learning_rate": 4.28758849150186e-06,
      "logits/chosen": -3.6697018146514893,
      "logits/rejected": -3.6387767791748047,
      "logps/chosen": -16.114957809448242,
      "logps/rejected": -34.932247161865234,
      "loss": 0.1195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8689799308776855,
      "rewards/margins": 2.8689799308776855,
      "rewards/rejected": 0.0,
      "step": 922
    },
    {
      "epoch": 5.156424581005586,
      "grad_norm": 4.867962473160499,
      "learning_rate": 4.285456536498617e-06,
      "logits/chosen": -3.7479538917541504,
      "logits/rejected": -3.4373276233673096,
      "logps/chosen": -6.732104301452637,
      "logps/rejected": -54.417510986328125,
      "loss": 0.1553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4049553871154785,
      "rewards/margins": 2.4049553871154785,
      "rewards/rejected": 0.0,
      "step": 923
    },
    {
      "epoch": 5.162011173184357,
      "grad_norm": 4.277647787245267,
      "learning_rate": 4.283321927988986e-06,
      "logits/chosen": -3.5994536876678467,
      "logits/rejected": -3.3147575855255127,
      "logps/chosen": -8.227420806884766,
      "logps/rejected": -31.050642013549805,
      "loss": 0.1392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6680076122283936,
      "rewards/margins": 2.6680076122283936,
      "rewards/rejected": 0.0,
      "step": 924
    },
    {
      "epoch": 5.167597765363128,
      "grad_norm": 4.36136168997802,
      "learning_rate": 4.281184669145379e-06,
      "logits/chosen": -3.6601314544677734,
      "logits/rejected": -3.6816258430480957,
      "logps/chosen": -5.830422401428223,
      "logps/rejected": -58.253299713134766,
      "loss": 0.1562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.267904043197632,
      "rewards/margins": 2.267904043197632,
      "rewards/rejected": 0.0,
      "step": 925
    },
    {
      "epoch": 5.173184357541899,
      "grad_norm": 6.182515895629759,
      "learning_rate": 4.279044763144141e-06,
      "logits/chosen": -3.3486433029174805,
      "logits/rejected": -3.5113658905029297,
      "logps/chosen": -15.952951431274414,
      "logps/rejected": -47.528953552246094,
      "loss": 0.1859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.43922758102417,
      "rewards/margins": 2.43922758102417,
      "rewards/rejected": 0.0,
      "step": 926
    },
    {
      "epoch": 5.17877094972067,
      "grad_norm": 7.755025570134518,
      "learning_rate": 4.276902213165555e-06,
      "logits/chosen": -3.569204807281494,
      "logits/rejected": -3.5272228717803955,
      "logps/chosen": -17.43412208557129,
      "logps/rejected": -39.98564910888672,
      "loss": 0.1234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.375035285949707,
      "rewards/margins": 4.375035285949707,
      "rewards/rejected": 0.0,
      "step": 927
    },
    {
      "epoch": 5.184357541899441,
      "grad_norm": 6.430052279072878,
      "learning_rate": 4.274757022393832e-06,
      "logits/chosen": -3.5899007320404053,
      "logits/rejected": -3.277804374694824,
      "logps/chosen": -6.791771411895752,
      "logps/rejected": -65.72271728515625,
      "loss": 0.2208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9770700931549072,
      "rewards/margins": 1.9770700931549072,
      "rewards/rejected": 0.0,
      "step": 928
    },
    {
      "epoch": 5.189944134078212,
      "grad_norm": 6.665042816942045,
      "learning_rate": 4.2726091940171055e-06,
      "logits/chosen": -3.518657922744751,
      "logits/rejected": -3.538637161254883,
      "logps/chosen": -9.226405143737793,
      "logps/rejected": -44.37838363647461,
      "loss": 0.1607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.820798397064209,
      "rewards/margins": 2.820798397064209,
      "rewards/rejected": 0.0,
      "step": 929
    },
    {
      "epoch": 5.195530726256983,
      "grad_norm": 9.761263444909726,
      "learning_rate": 4.270458731227433e-06,
      "logits/chosen": -3.831392765045166,
      "logits/rejected": -3.686521530151367,
      "logps/chosen": -4.668505668640137,
      "logps/rejected": -43.49795150756836,
      "loss": 0.2176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2203733921051025,
      "rewards/margins": 2.2203733921051025,
      "rewards/rejected": 0.0,
      "step": 930
    },
    {
      "epoch": 5.201117318435754,
      "grad_norm": 3.3422197057838186,
      "learning_rate": 4.268305637220784e-06,
      "logits/chosen": -3.701910972595215,
      "logits/rejected": -3.573518753051758,
      "logps/chosen": -34.03693389892578,
      "logps/rejected": -62.910335540771484,
      "loss": 0.1291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9541425704956055,
      "rewards/margins": 3.9541425704956055,
      "rewards/rejected": 0.0,
      "step": 931
    },
    {
      "epoch": 5.206703910614525,
      "grad_norm": 3.75338582752365,
      "learning_rate": 4.26614991519704e-06,
      "logits/chosen": -3.4035232067108154,
      "logits/rejected": -3.7144432067871094,
      "logps/chosen": -4.831063270568848,
      "logps/rejected": -77.08946228027344,
      "loss": 0.1413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2575607299804688,
      "rewards/margins": 2.2575607299804688,
      "rewards/rejected": 0.0,
      "step": 932
    },
    {
      "epoch": 5.212290502793296,
      "grad_norm": 5.404100108234395,
      "learning_rate": 4.263991568359988e-06,
      "logits/chosen": -3.322030782699585,
      "logits/rejected": -3.4050514698028564,
      "logps/chosen": -16.97873878479004,
      "logps/rejected": -44.15199279785156,
      "loss": 0.1366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8617472648620605,
      "rewards/margins": 2.8617472648620605,
      "rewards/rejected": 0.0,
      "step": 933
    },
    {
      "epoch": 5.217877094972067,
      "grad_norm": 20.86316117183553,
      "learning_rate": 4.261830599917316e-06,
      "logits/chosen": -3.88197660446167,
      "logits/rejected": -3.9721131324768066,
      "logps/chosen": -6.194864749908447,
      "logps/rejected": -46.154945373535156,
      "loss": 0.2181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2934279441833496,
      "rewards/margins": 2.2934279441833496,
      "rewards/rejected": 0.0,
      "step": 934
    },
    {
      "epoch": 5.223463687150838,
      "grad_norm": 5.3215849363490655,
      "learning_rate": 4.259667013080608e-06,
      "logits/chosen": -3.345524311065674,
      "logits/rejected": -3.3525664806365967,
      "logps/chosen": -3.5546278953552246,
      "logps/rejected": -54.35467529296875,
      "loss": 0.1806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4731441736221313,
      "rewards/margins": 1.4731441736221313,
      "rewards/rejected": 0.0,
      "step": 935
    },
    {
      "epoch": 5.229050279329609,
      "grad_norm": 9.853592837828447,
      "learning_rate": 4.257500811065338e-06,
      "logits/chosen": -3.6436991691589355,
      "logits/rejected": -3.390190839767456,
      "logps/chosen": -4.562302589416504,
      "logps/rejected": -47.11621856689453,
      "loss": 0.1808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.291530132293701,
      "rewards/margins": 2.291530132293701,
      "rewards/rejected": 0.0,
      "step": 936
    },
    {
      "epoch": 5.23463687150838,
      "grad_norm": 3.538899381483107,
      "learning_rate": 4.255331997090869e-06,
      "logits/chosen": -3.144172191619873,
      "logits/rejected": -3.1454854011535645,
      "logps/chosen": -18.256122589111328,
      "logps/rejected": -47.49192810058594,
      "loss": 0.1307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7728633880615234,
      "rewards/margins": 3.7728633880615234,
      "rewards/rejected": 0.0,
      "step": 937
    },
    {
      "epoch": 5.240223463687151,
      "grad_norm": 5.307340645873255,
      "learning_rate": 4.253160574380446e-06,
      "logits/chosen": -3.5458250045776367,
      "logits/rejected": -3.6001358032226562,
      "logps/chosen": -7.572875499725342,
      "logps/rejected": -44.51588821411133,
      "loss": 0.1604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0244286060333252,
      "rewards/margins": 1.0244286060333252,
      "rewards/rejected": 0.0,
      "step": 938
    },
    {
      "epoch": 5.245810055865922,
      "grad_norm": 4.457159861537102,
      "learning_rate": 4.250986546161188e-06,
      "logits/chosen": -3.3411195278167725,
      "logits/rejected": -3.251831293106079,
      "logps/chosen": -6.225868225097656,
      "logps/rejected": -40.984458923339844,
      "loss": 0.1523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.333200216293335,
      "rewards/margins": 2.333200216293335,
      "rewards/rejected": 0.0,
      "step": 939
    },
    {
      "epoch": 5.251396648044693,
      "grad_norm": 4.117919400872911,
      "learning_rate": 4.2488099156640895e-06,
      "logits/chosen": -3.7482452392578125,
      "logits/rejected": -3.781810760498047,
      "logps/chosen": -8.97645092010498,
      "logps/rejected": -42.83266067504883,
      "loss": 0.1316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8771677017211914,
      "rewards/margins": 2.8771677017211914,
      "rewards/rejected": 0.0,
      "step": 940
    },
    {
      "epoch": 5.251396648044693,
      "eval_logits/chosen": -3.3110263347625732,
      "eval_logits/rejected": -3.386641025543213,
      "eval_logps/chosen": -30.651927947998047,
      "eval_logps/rejected": -45.66022872924805,
      "eval_loss": 0.9537739157676697,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.05818865820765495,
      "eval_rewards/margins": -0.05818865820765495,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5518,
      "eval_samples_per_second": 12.132,
      "eval_steps_per_second": 0.391,
      "step": 940
    },
    {
      "epoch": 5.256983240223463,
      "grad_norm": 8.44340099202969,
      "learning_rate": 4.246630686124009e-06,
      "logits/chosen": -3.6742520332336426,
      "logits/rejected": -3.4408175945281982,
      "logps/chosen": -6.993295669555664,
      "logps/rejected": -30.901168823242188,
      "loss": 0.2327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6908655166625977,
      "rewards/margins": 2.6908655166625977,
      "rewards/rejected": 0.0,
      "step": 941
    },
    {
      "epoch": 5.262569832402234,
      "grad_norm": 11.16442420318577,
      "learning_rate": 4.244448860779672e-06,
      "logits/chosen": -3.6985764503479004,
      "logits/rejected": -3.728121042251587,
      "logps/chosen": -2.659174919128418,
      "logps/rejected": -42.57928466796875,
      "loss": 0.1911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2087342739105225,
      "rewards/margins": 2.2087342739105225,
      "rewards/rejected": 0.0,
      "step": 942
    },
    {
      "epoch": 5.268156424581005,
      "grad_norm": 12.769366800631678,
      "learning_rate": 4.242264442873658e-06,
      "logits/chosen": -3.3823795318603516,
      "logits/rejected": -3.363919258117676,
      "logps/chosen": -6.052245140075684,
      "logps/rejected": -39.94448471069336,
      "loss": 0.2165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7505486011505127,
      "rewards/margins": 1.7505486011505127,
      "rewards/rejected": 0.0,
      "step": 943
    },
    {
      "epoch": 5.273743016759776,
      "grad_norm": 4.239220086348074,
      "learning_rate": 4.240077435652401e-06,
      "logits/chosen": -3.1605324745178223,
      "logits/rejected": -3.1842024326324463,
      "logps/chosen": -9.839599609375,
      "logps/rejected": -21.826736450195312,
      "loss": 0.1698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.27082896232605,
      "rewards/margins": 2.27082896232605,
      "rewards/rejected": 0.0,
      "step": 944
    },
    {
      "epoch": 5.279329608938547,
      "grad_norm": 4.122987961118258,
      "learning_rate": 4.237887842366185e-06,
      "logits/chosen": -3.4826982021331787,
      "logits/rejected": -3.254274606704712,
      "logps/chosen": -3.570706605911255,
      "logps/rejected": -73.85865783691406,
      "loss": 0.1675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9856119155883789,
      "rewards/margins": 0.9856119155883789,
      "rewards/rejected": 0.0,
      "step": 945
    },
    {
      "epoch": 5.284916201117318,
      "grad_norm": 2.250266611321258,
      "learning_rate": 4.235695666269134e-06,
      "logits/chosen": -3.4874536991119385,
      "logits/rejected": -3.502157688140869,
      "logps/chosen": -9.616594314575195,
      "logps/rejected": -30.86969566345215,
      "loss": 0.1177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.79178524017334,
      "rewards/margins": 2.79178524017334,
      "rewards/rejected": 0.0,
      "step": 946
    },
    {
      "epoch": 5.290502793296089,
      "grad_norm": 4.922276070906417,
      "learning_rate": 4.233500910619212e-06,
      "logits/chosen": -3.54256534576416,
      "logits/rejected": -3.1916322708129883,
      "logps/chosen": -4.853883266448975,
      "logps/rejected": -67.94511413574219,
      "loss": 0.174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0899378061294556,
      "rewards/margins": 1.0899378061294556,
      "rewards/rejected": 0.0,
      "step": 947
    },
    {
      "epoch": 5.29608938547486,
      "grad_norm": 3.101944811411969,
      "learning_rate": 4.231303578678217e-06,
      "logits/chosen": -3.09749174118042,
      "logits/rejected": -3.2194621562957764,
      "logps/chosen": -3.990600347518921,
      "logps/rejected": -62.617835998535156,
      "loss": 0.1323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.004678964614868,
      "rewards/margins": 2.004678964614868,
      "rewards/rejected": 0.0,
      "step": 948
    },
    {
      "epoch": 5.301675977653631,
      "grad_norm": 5.718064493245637,
      "learning_rate": 4.229103673711777e-06,
      "logits/chosen": -3.07692813873291,
      "logits/rejected": -2.9394137859344482,
      "logps/chosen": -4.41028356552124,
      "logps/rejected": -72.95929718017578,
      "loss": 0.1448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.228459119796753,
      "rewards/margins": 2.228459119796753,
      "rewards/rejected": 0.0,
      "step": 949
    },
    {
      "epoch": 5.307262569832402,
      "grad_norm": 2.1017869332441124,
      "learning_rate": 4.226901198989341e-06,
      "logits/chosen": -3.4811246395111084,
      "logits/rejected": -3.3890392780303955,
      "logps/chosen": -11.445686340332031,
      "logps/rejected": -41.04914093017578,
      "loss": 0.1077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4251041412353516,
      "rewards/margins": 2.4251041412353516,
      "rewards/rejected": 0.0,
      "step": 950
    },
    {
      "epoch": 5.312849162011173,
      "grad_norm": 6.979778588363219,
      "learning_rate": 4.224696157784178e-06,
      "logits/chosen": -3.603886127471924,
      "logits/rejected": -3.340958595275879,
      "logps/chosen": -5.50705623626709,
      "logps/rejected": -51.666961669921875,
      "loss": 0.1462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4785726070404053,
      "rewards/margins": 2.4785726070404053,
      "rewards/rejected": 0.0,
      "step": 951
    },
    {
      "epoch": 5.318435754189944,
      "grad_norm": 9.776206152244592,
      "learning_rate": 4.222488553373374e-06,
      "logits/chosen": -3.4698362350463867,
      "logits/rejected": -3.4753644466400146,
      "logps/chosen": -19.014272689819336,
      "logps/rejected": -58.044986724853516,
      "loss": 0.1743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.646552085876465,
      "rewards/margins": 3.646552085876465,
      "rewards/rejected": 0.0,
      "step": 952
    },
    {
      "epoch": 5.324022346368715,
      "grad_norm": 2.1726393941054285,
      "learning_rate": 4.22027838903782e-06,
      "logits/chosen": -3.3708794116973877,
      "logits/rejected": -3.5871469974517822,
      "logps/chosen": -4.2408857345581055,
      "logps/rejected": -65.92176818847656,
      "loss": 0.1059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4622855186462402,
      "rewards/margins": 2.4622855186462402,
      "rewards/rejected": 0.0,
      "step": 953
    },
    {
      "epoch": 5.329608938547486,
      "grad_norm": 3.545815096138634,
      "learning_rate": 4.218065668062216e-06,
      "logits/chosen": -3.748992919921875,
      "logits/rejected": -3.7388246059417725,
      "logps/chosen": -8.062171936035156,
      "logps/rejected": -57.39476776123047,
      "loss": 0.1247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7800285816192627,
      "rewards/margins": 1.7800285816192627,
      "rewards/rejected": 0.0,
      "step": 954
    },
    {
      "epoch": 5.335195530726257,
      "grad_norm": 9.42214029649806,
      "learning_rate": 4.215850393735058e-06,
      "logits/chosen": -3.839170455932617,
      "logits/rejected": -3.8366780281066895,
      "logps/chosen": -28.332326889038086,
      "logps/rejected": -43.469947814941406,
      "loss": 0.1297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.450079917907715,
      "rewards/margins": 4.450079917907715,
      "rewards/rejected": 0.0,
      "step": 955
    },
    {
      "epoch": 5.340782122905028,
      "grad_norm": 3.0720008229881826,
      "learning_rate": 4.213632569348639e-06,
      "logits/chosen": -3.535125255584717,
      "logits/rejected": -3.4822850227355957,
      "logps/chosen": -6.787631511688232,
      "logps/rejected": -43.688575744628906,
      "loss": 0.1355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5374698638916016,
      "rewards/margins": 2.5374698638916016,
      "rewards/rejected": 0.0,
      "step": 956
    },
    {
      "epoch": 5.346368715083799,
      "grad_norm": 4.494870625036312,
      "learning_rate": 4.211412198199041e-06,
      "logits/chosen": -3.618846893310547,
      "logits/rejected": -3.535999298095703,
      "logps/chosen": -6.492527961730957,
      "logps/rejected": -52.021080017089844,
      "loss": 0.1083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2043561935424805,
      "rewards/margins": 2.2043561935424805,
      "rewards/rejected": 0.0,
      "step": 957
    },
    {
      "epoch": 5.351955307262569,
      "grad_norm": 7.485361920990615,
      "learning_rate": 4.209189283586131e-06,
      "logits/chosen": -3.8497495651245117,
      "logits/rejected": -3.82089900970459,
      "logps/chosen": -10.607149124145508,
      "logps/rejected": -46.030006408691406,
      "loss": 0.1888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6418509483337402,
      "rewards/margins": 2.6418509483337402,
      "rewards/rejected": 0.0,
      "step": 958
    },
    {
      "epoch": 5.35754189944134,
      "grad_norm": 3.585581988441107,
      "learning_rate": 4.206963828813555e-06,
      "logits/chosen": -3.619285821914673,
      "logits/rejected": -3.5842599868774414,
      "logps/chosen": -4.1832685470581055,
      "logps/rejected": -56.89471435546875,
      "loss": 0.1196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2457611560821533,
      "rewards/margins": 2.2457611560821533,
      "rewards/rejected": 0.0,
      "step": 959
    },
    {
      "epoch": 5.363128491620111,
      "grad_norm": 3.698180225756441,
      "learning_rate": 4.204735837188735e-06,
      "logits/chosen": -3.524261951446533,
      "logits/rejected": -3.4705047607421875,
      "logps/chosen": -4.208440780639648,
      "logps/rejected": -49.59043884277344,
      "loss": 0.1398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2154362201690674,
      "rewards/margins": 3.2154362201690674,
      "rewards/rejected": 0.0,
      "step": 960
    },
    {
      "epoch": 5.363128491620111,
      "eval_logits/chosen": -3.3240363597869873,
      "eval_logits/rejected": -3.4067063331604004,
      "eval_logps/chosen": -33.255680084228516,
      "eval_logps/rejected": -48.20379638671875,
      "eval_loss": 1.1064077615737915,
      "eval_rewards/accuracies": 0.42500001192092896,
      "eval_rewards/chosen": -0.3185637295246124,
      "eval_rewards/margins": -0.3185637295246124,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6476,
      "eval_samples_per_second": 12.087,
      "eval_steps_per_second": 0.39,
      "step": 960
    },
    {
      "epoch": 5.368715083798882,
      "grad_norm": 3.656779673907986,
      "learning_rate": 4.2025053120228654e-06,
      "logits/chosen": -3.326375961303711,
      "logits/rejected": -3.226975440979004,
      "logps/chosen": -12.265459060668945,
      "logps/rejected": -47.578514099121094,
      "loss": 0.156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.352531909942627,
      "rewards/margins": 3.352531909942627,
      "rewards/rejected": 0.0,
      "step": 961
    },
    {
      "epoch": 5.374301675977653,
      "grad_norm": 4.809674681113121,
      "learning_rate": 4.200272256630902e-06,
      "logits/chosen": -3.6934399604797363,
      "logits/rejected": -3.5985448360443115,
      "logps/chosen": -7.9778151512146,
      "logps/rejected": -39.421104431152344,
      "loss": 0.1398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7998226881027222,
      "rewards/margins": 1.7998226881027222,
      "rewards/rejected": 0.0,
      "step": 962
    },
    {
      "epoch": 5.379888268156424,
      "grad_norm": 2.73595867110771,
      "learning_rate": 4.1980366743315636e-06,
      "logits/chosen": -3.5109663009643555,
      "logits/rejected": -3.420870780944824,
      "logps/chosen": -5.674378395080566,
      "logps/rejected": -47.55903625488281,
      "loss": 0.1084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6255154609680176,
      "rewards/margins": 2.6255154609680176,
      "rewards/rejected": 0.0,
      "step": 963
    },
    {
      "epoch": 5.385474860335195,
      "grad_norm": 5.861091657517266,
      "learning_rate": 4.195798568447323e-06,
      "logits/chosen": -3.561384439468384,
      "logits/rejected": -3.2259035110473633,
      "logps/chosen": -12.507508277893066,
      "logps/rejected": -61.17552947998047,
      "loss": 0.1949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4067444801330566,
      "rewards/margins": 2.4067444801330566,
      "rewards/rejected": 0.0,
      "step": 964
    },
    {
      "epoch": 5.391061452513966,
      "grad_norm": 5.177146072534838,
      "learning_rate": 4.193557942304404e-06,
      "logits/chosen": -3.6712286472320557,
      "logits/rejected": -3.5625998973846436,
      "logps/chosen": -3.276979684829712,
      "logps/rejected": -94.75445556640625,
      "loss": 0.1737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1561698913574219,
      "rewards/margins": 1.1561698913574219,
      "rewards/rejected": 0.0,
      "step": 965
    },
    {
      "epoch": 5.396648044692737,
      "grad_norm": 5.242191624736138,
      "learning_rate": 4.191314799232777e-06,
      "logits/chosen": -3.7593326568603516,
      "logits/rejected": -3.521401882171631,
      "logps/chosen": -4.455077648162842,
      "logps/rejected": -37.304744720458984,
      "loss": 0.1584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3510775566101074,
      "rewards/margins": 2.3510775566101074,
      "rewards/rejected": 0.0,
      "step": 966
    },
    {
      "epoch": 5.402234636871508,
      "grad_norm": 2.641643560567007,
      "learning_rate": 4.189069142566152e-06,
      "logits/chosen": -3.5930635929107666,
      "logits/rejected": -3.4844088554382324,
      "logps/chosen": -1.7989535331726074,
      "logps/rejected": -33.01203918457031,
      "loss": 0.1173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4563655853271484,
      "rewards/margins": 1.4563655853271484,
      "rewards/rejected": 0.0,
      "step": 967
    },
    {
      "epoch": 5.407821229050279,
      "grad_norm": 6.637644696961425,
      "learning_rate": 4.186820975641975e-06,
      "logits/chosen": -3.2550129890441895,
      "logits/rejected": -3.2350900173187256,
      "logps/chosen": -7.21677303314209,
      "logps/rejected": -51.3714714050293,
      "loss": 0.1503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2997395992279053,
      "rewards/margins": 1.2997395992279053,
      "rewards/rejected": 0.0,
      "step": 968
    },
    {
      "epoch": 5.41340782122905,
      "grad_norm": 6.362768936898133,
      "learning_rate": 4.1845703018014205e-06,
      "logits/chosen": -3.883927583694458,
      "logits/rejected": -3.7432942390441895,
      "logps/chosen": -9.327871322631836,
      "logps/rejected": -40.85734176635742,
      "loss": 0.1925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2538366317749023,
      "rewards/margins": 2.2538366317749023,
      "rewards/rejected": 0.0,
      "step": 969
    },
    {
      "epoch": 5.418994413407821,
      "grad_norm": 4.939846204820397,
      "learning_rate": 4.1823171243893925e-06,
      "logits/chosen": -3.4432690143585205,
      "logits/rejected": -3.4598608016967773,
      "logps/chosen": -6.514553070068359,
      "logps/rejected": -53.05991744995117,
      "loss": 0.1458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0755748748779297,
      "rewards/margins": 2.0755748748779297,
      "rewards/rejected": 0.0,
      "step": 970
    },
    {
      "epoch": 5.424581005586592,
      "grad_norm": 6.264957170110832,
      "learning_rate": 4.180061446754513e-06,
      "logits/chosen": -3.714150905609131,
      "logits/rejected": -3.598496675491333,
      "logps/chosen": -3.8850553035736084,
      "logps/rejected": -49.10525131225586,
      "loss": 0.1712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.322482109069824,
      "rewards/margins": 2.322482109069824,
      "rewards/rejected": 0.0,
      "step": 971
    },
    {
      "epoch": 5.430167597765363,
      "grad_norm": 2.8623672642274878,
      "learning_rate": 4.177803272249121e-06,
      "logits/chosen": -3.4865658283233643,
      "logits/rejected": -3.4785873889923096,
      "logps/chosen": -6.820018291473389,
      "logps/rejected": -44.95675277709961,
      "loss": 0.14,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.225419044494629,
      "rewards/margins": 2.225419044494629,
      "rewards/rejected": 0.0,
      "step": 972
    },
    {
      "epoch": 5.435754189944134,
      "grad_norm": 3.6139101023038602,
      "learning_rate": 4.1755426042292665e-06,
      "logits/chosen": -3.298283338546753,
      "logits/rejected": -3.3344979286193848,
      "logps/chosen": -4.391506195068359,
      "logps/rejected": -72.88529205322266,
      "loss": 0.1399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2621567249298096,
      "rewards/margins": 2.2621567249298096,
      "rewards/rejected": 0.0,
      "step": 973
    },
    {
      "epoch": 5.441340782122905,
      "grad_norm": 5.522000187275701,
      "learning_rate": 4.1732794460547045e-06,
      "logits/chosen": -3.52689790725708,
      "logits/rejected": -3.2689177989959717,
      "logps/chosen": -4.198764324188232,
      "logps/rejected": -36.7877082824707,
      "loss": 0.1498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.33713436126709,
      "rewards/margins": 2.33713436126709,
      "rewards/rejected": 0.0,
      "step": 974
    },
    {
      "epoch": 5.446927374301676,
      "grad_norm": 6.162193668597155,
      "learning_rate": 4.17101380108889e-06,
      "logits/chosen": -3.091390609741211,
      "logits/rejected": -2.9872539043426514,
      "logps/chosen": -12.440938949584961,
      "logps/rejected": -43.68043899536133,
      "loss": 0.1626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.550236940383911,
      "rewards/margins": 3.550236940383911,
      "rewards/rejected": 0.0,
      "step": 975
    },
    {
      "epoch": 5.452513966480447,
      "grad_norm": 2.8621236506836185,
      "learning_rate": 4.168745672698977e-06,
      "logits/chosen": -3.357341766357422,
      "logits/rejected": -3.219266176223755,
      "logps/chosen": -12.705924034118652,
      "logps/rejected": -55.439579010009766,
      "loss": 0.1214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.331462860107422,
      "rewards/margins": 2.331462860107422,
      "rewards/rejected": 0.0,
      "step": 976
    },
    {
      "epoch": 5.4581005586592175,
      "grad_norm": 4.829265450452771,
      "learning_rate": 4.166475064255806e-06,
      "logits/chosen": -3.9336180686950684,
      "logits/rejected": -3.7325339317321777,
      "logps/chosen": -2.5502398014068604,
      "logps/rejected": -56.49595642089844,
      "loss": 0.1456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9823269844055176,
      "rewards/margins": 1.9823269844055176,
      "rewards/rejected": 0.0,
      "step": 977
    },
    {
      "epoch": 5.4636871508379885,
      "grad_norm": 3.6963793151995477,
      "learning_rate": 4.164201979133907e-06,
      "logits/chosen": -3.4831862449645996,
      "logits/rejected": -3.5308079719543457,
      "logps/chosen": -12.079692840576172,
      "logps/rejected": -44.04035949707031,
      "loss": 0.1271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4513697624206543,
      "rewards/margins": 2.4513697624206543,
      "rewards/rejected": 0.0,
      "step": 978
    },
    {
      "epoch": 5.4692737430167595,
      "grad_norm": 3.6957321259086933,
      "learning_rate": 4.161926420711489e-06,
      "logits/chosen": -3.245147943496704,
      "logits/rejected": -3.419978380203247,
      "logps/chosen": -3.491861343383789,
      "logps/rejected": -67.43717193603516,
      "loss": 0.1551,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9346001744270325,
      "rewards/margins": 0.9346001744270325,
      "rewards/rejected": 0.0,
      "step": 979
    },
    {
      "epoch": 5.4748603351955305,
      "grad_norm": 7.998156186790284,
      "learning_rate": 4.1596483923704365e-06,
      "logits/chosen": -3.490359306335449,
      "logits/rejected": -3.6039059162139893,
      "logps/chosen": -14.837820053100586,
      "logps/rejected": -42.57013702392578,
      "loss": 0.1866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.446465253829956,
      "rewards/margins": 2.446465253829956,
      "rewards/rejected": 0.0,
      "step": 980
    },
    {
      "epoch": 5.4748603351955305,
      "eval_logits/chosen": -3.3516640663146973,
      "eval_logits/rejected": -3.4440560340881348,
      "eval_logps/chosen": -33.12404251098633,
      "eval_logps/rejected": -47.36741638183594,
      "eval_loss": 1.07804274559021,
      "eval_rewards/accuracies": 0.375,
      "eval_rewards/chosen": -0.3054000735282898,
      "eval_rewards/margins": -0.3054000735282898,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.493,
      "eval_samples_per_second": 12.16,
      "eval_steps_per_second": 0.392,
      "step": 980
    },
    {
      "epoch": 5.4804469273743015,
      "grad_norm": 6.119027340047425,
      "learning_rate": 4.157367897496306e-06,
      "logits/chosen": -3.605476140975952,
      "logits/rejected": -3.612088680267334,
      "logps/chosen": -34.079734802246094,
      "logps/rejected": -45.64122009277344,
      "loss": 0.2131,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0694313049316406,
      "rewards/margins": 2.0694313049316406,
      "rewards/rejected": 0.0,
      "step": 981
    },
    {
      "epoch": 5.4860335195530725,
      "grad_norm": 4.308172716840501,
      "learning_rate": 4.155084939478319e-06,
      "logits/chosen": -3.396641254425049,
      "logits/rejected": -3.112525463104248,
      "logps/chosen": -5.759881019592285,
      "logps/rejected": -44.805259704589844,
      "loss": 0.1296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9098210334777832,
      "rewards/margins": 1.9098210334777832,
      "rewards/rejected": 0.0,
      "step": 982
    },
    {
      "epoch": 5.4916201117318435,
      "grad_norm": 4.739028552250493,
      "learning_rate": 4.152799521709357e-06,
      "logits/chosen": -3.6685569286346436,
      "logits/rejected": -3.413083791732788,
      "logps/chosen": -11.82789134979248,
      "logps/rejected": -60.066497802734375,
      "loss": 0.1604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2141571044921875,
      "rewards/margins": 2.2141571044921875,
      "rewards/rejected": 0.0,
      "step": 983
    },
    {
      "epoch": 5.4972067039106145,
      "grad_norm": 8.02903706023069,
      "learning_rate": 4.150511647585958e-06,
      "logits/chosen": -3.3474433422088623,
      "logits/rejected": -3.319369316101074,
      "logps/chosen": -8.371082305908203,
      "logps/rejected": -44.15367126464844,
      "loss": 0.1709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.667130947113037,
      "rewards/margins": 2.667130947113037,
      "rewards/rejected": 0.0,
      "step": 984
    },
    {
      "epoch": 5.5027932960893855,
      "grad_norm": 7.2967830922377,
      "learning_rate": 4.148221320508311e-06,
      "logits/chosen": -3.3001487255096436,
      "logits/rejected": -3.2697408199310303,
      "logps/chosen": -3.3884310722351074,
      "logps/rejected": -55.51695251464844,
      "loss": 0.2177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.217940092086792,
      "rewards/margins": 2.217940092086792,
      "rewards/rejected": 0.0,
      "step": 985
    },
    {
      "epoch": 5.5083798882681565,
      "grad_norm": 2.5971089520661357,
      "learning_rate": 4.14592854388025e-06,
      "logits/chosen": -3.5557010173797607,
      "logits/rejected": -3.6522791385650635,
      "logps/chosen": -11.057498931884766,
      "logps/rejected": -54.885231018066406,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.70231294631958,
      "rewards/margins": 2.70231294631958,
      "rewards/rejected": 0.0,
      "step": 986
    },
    {
      "epoch": 5.5139664804469275,
      "grad_norm": 3.0509413604788826,
      "learning_rate": 4.143633321109248e-06,
      "logits/chosen": -3.5671207904815674,
      "logits/rejected": -3.6111178398132324,
      "logps/chosen": -7.595380783081055,
      "logps/rejected": -39.405189514160156,
      "loss": 0.1425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.55010986328125,
      "rewards/margins": 2.55010986328125,
      "rewards/rejected": 0.0,
      "step": 987
    },
    {
      "epoch": 5.5195530726256985,
      "grad_norm": 3.4228364832319005,
      "learning_rate": 4.141335655606416e-06,
      "logits/chosen": -3.2247073650360107,
      "logits/rejected": -3.2609124183654785,
      "logps/chosen": -11.51943588256836,
      "logps/rejected": -46.08111572265625,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8312346935272217,
      "rewards/margins": 2.8312346935272217,
      "rewards/rejected": 0.0,
      "step": 988
    },
    {
      "epoch": 5.5251396648044695,
      "grad_norm": 5.567529758208262,
      "learning_rate": 4.139035550786495e-06,
      "logits/chosen": -3.344508409500122,
      "logits/rejected": -3.248570442199707,
      "logps/chosen": -6.8366289138793945,
      "logps/rejected": -31.39914321899414,
      "loss": 0.1647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9763725996017456,
      "rewards/margins": 1.9763725996017456,
      "rewards/rejected": 0.0,
      "step": 989
    },
    {
      "epoch": 5.5307262569832405,
      "grad_norm": 5.642760733446209,
      "learning_rate": 4.136733010067848e-06,
      "logits/chosen": -3.6930699348449707,
      "logits/rejected": -3.6946728229522705,
      "logps/chosen": -6.427001953125,
      "logps/rejected": -41.33892822265625,
      "loss": 0.1466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8331751823425293,
      "rewards/margins": 2.8331751823425293,
      "rewards/rejected": 0.0,
      "step": 990
    },
    {
      "epoch": 5.5363128491620115,
      "grad_norm": 2.9108627650677477,
      "learning_rate": 4.134428036872462e-06,
      "logits/chosen": -3.284043073654175,
      "logits/rejected": -3.409698247909546,
      "logps/chosen": -8.456692695617676,
      "logps/rejected": -104.9984130859375,
      "loss": 0.1085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1040992736816406,
      "rewards/margins": 3.1040992736816406,
      "rewards/rejected": 0.0,
      "step": 991
    },
    {
      "epoch": 5.5418994413407825,
      "grad_norm": 2.091212554403077,
      "learning_rate": 4.132120634625938e-06,
      "logits/chosen": -3.6821956634521484,
      "logits/rejected": -3.762444257736206,
      "logps/chosen": -27.80360221862793,
      "logps/rejected": -32.493194580078125,
      "loss": 0.0917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.985988140106201,
      "rewards/margins": 3.985988140106201,
      "rewards/rejected": 0.0,
      "step": 992
    },
    {
      "epoch": 5.547486033519553,
      "grad_norm": 4.953055403306408,
      "learning_rate": 4.129810806757486e-06,
      "logits/chosen": -3.32187819480896,
      "logits/rejected": -3.1670303344726562,
      "logps/chosen": -14.970754623413086,
      "logps/rejected": -66.94774627685547,
      "loss": 0.1975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.7494018077850342,
      "rewards/margins": 1.7494018077850342,
      "rewards/rejected": 0.0,
      "step": 993
    },
    {
      "epoch": 5.553072625698324,
      "grad_norm": 4.381188870297735,
      "learning_rate": 4.127498556699921e-06,
      "logits/chosen": -3.846681594848633,
      "logits/rejected": -3.4768431186676025,
      "logps/chosen": -9.74203109741211,
      "logps/rejected": -45.22877502441406,
      "loss": 0.1399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.528388500213623,
      "rewards/margins": 2.528388500213623,
      "rewards/rejected": 0.0,
      "step": 994
    },
    {
      "epoch": 5.558659217877095,
      "grad_norm": 4.031000246362124,
      "learning_rate": 4.12518388788966e-06,
      "logits/chosen": -3.2938575744628906,
      "logits/rejected": -3.290419816970825,
      "logps/chosen": -4.420078277587891,
      "logps/rejected": -90.68693542480469,
      "loss": 0.1376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8018118143081665,
      "rewards/margins": 1.8018118143081665,
      "rewards/rejected": 0.0,
      "step": 995
    },
    {
      "epoch": 5.564245810055866,
      "grad_norm": 4.014745087827815,
      "learning_rate": 4.122866803766712e-06,
      "logits/chosen": -3.46515154838562,
      "logits/rejected": -3.4277071952819824,
      "logps/chosen": -12.724730491638184,
      "logps/rejected": -42.92158126831055,
      "loss": 0.1327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.825291633605957,
      "rewards/margins": 2.825291633605957,
      "rewards/rejected": 0.0,
      "step": 996
    },
    {
      "epoch": 5.569832402234637,
      "grad_norm": 3.4123063288240805,
      "learning_rate": 4.120547307774676e-06,
      "logits/chosen": -3.398344039916992,
      "logits/rejected": -3.5014967918395996,
      "logps/chosen": -3.306814193725586,
      "logps/rejected": -66.5895004272461,
      "loss": 0.0986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3137049674987793,
      "rewards/margins": 2.3137049674987793,
      "rewards/rejected": 0.0,
      "step": 997
    },
    {
      "epoch": 5.575418994413408,
      "grad_norm": 8.160435095457217,
      "learning_rate": 4.118225403360738e-06,
      "logits/chosen": -3.4880783557891846,
      "logits/rejected": -3.2608509063720703,
      "logps/chosen": -3.787039041519165,
      "logps/rejected": -43.52656173706055,
      "loss": 0.1581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.914976716041565,
      "rewards/margins": 1.914976716041565,
      "rewards/rejected": 0.0,
      "step": 998
    },
    {
      "epoch": 5.581005586592179,
      "grad_norm": 4.859629760949177,
      "learning_rate": 4.11590109397566e-06,
      "logits/chosen": -3.698920726776123,
      "logits/rejected": -3.419184923171997,
      "logps/chosen": -5.219764709472656,
      "logps/rejected": -35.102882385253906,
      "loss": 0.1674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.465695858001709,
      "rewards/margins": 2.465695858001709,
      "rewards/rejected": 0.0,
      "step": 999
    },
    {
      "epoch": 5.58659217877095,
      "grad_norm": 3.3482710918303256,
      "learning_rate": 4.113574383073781e-06,
      "logits/chosen": -3.604121685028076,
      "logits/rejected": -3.7930967807769775,
      "logps/chosen": -14.195884704589844,
      "logps/rejected": -33.524559020996094,
      "loss": 0.1289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2101869583129883,
      "rewards/margins": 3.2101869583129883,
      "rewards/rejected": 0.0,
      "step": 1000
    },
    {
      "epoch": 5.58659217877095,
      "eval_logits/chosen": -3.366478681564331,
      "eval_logits/rejected": -3.4610702991485596,
      "eval_logps/chosen": -32.408451080322266,
      "eval_logps/rejected": -47.66121292114258,
      "eval_loss": 1.061872959136963,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.23384089767932892,
      "eval_rewards/margins": -0.23384089767932892,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5136,
      "eval_samples_per_second": 12.15,
      "eval_steps_per_second": 0.392,
      "step": 1000
    },
    {
      "epoch": 5.592178770949721,
      "grad_norm": 4.864004615512814,
      "learning_rate": 4.111245274113006e-06,
      "logits/chosen": -3.6828019618988037,
      "logits/rejected": -3.5086019039154053,
      "logps/chosen": -5.096928596496582,
      "logps/rejected": -52.530120849609375,
      "loss": 0.1635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3885371685028076,
      "rewards/margins": 2.3885371685028076,
      "rewards/rejected": 0.0,
      "step": 1001
    },
    {
      "epoch": 5.597765363128492,
      "grad_norm": 3.224951145902805,
      "learning_rate": 4.108913770554807e-06,
      "logits/chosen": -3.305185317993164,
      "logits/rejected": -3.2799484729766846,
      "logps/chosen": -2.6047656536102295,
      "logps/rejected": -62.64128112792969,
      "loss": 0.1291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.603882908821106,
      "rewards/margins": 1.603882908821106,
      "rewards/rejected": 0.0,
      "step": 1002
    },
    {
      "epoch": 5.603351955307263,
      "grad_norm": 8.105231501618922,
      "learning_rate": 4.106579875864212e-06,
      "logits/chosen": -3.164358139038086,
      "logits/rejected": -3.1976406574249268,
      "logps/chosen": -19.977449417114258,
      "logps/rejected": -50.416690826416016,
      "loss": 0.129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.870574712753296,
      "rewards/margins": 2.870574712753296,
      "rewards/rejected": 0.0,
      "step": 1003
    },
    {
      "epoch": 5.608938547486034,
      "grad_norm": 4.265248419430463,
      "learning_rate": 4.104243593509807e-06,
      "logits/chosen": -3.3601162433624268,
      "logits/rejected": -3.2196850776672363,
      "logps/chosen": -3.6597352027893066,
      "logps/rejected": -41.860496520996094,
      "loss": 0.1524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2862625122070312,
      "rewards/margins": 1.2862625122070312,
      "rewards/rejected": 0.0,
      "step": 1004
    },
    {
      "epoch": 5.614525139664805,
      "grad_norm": 3.7435460517962427,
      "learning_rate": 4.10190492696372e-06,
      "logits/chosen": -3.528552770614624,
      "logits/rejected": -3.4290125370025635,
      "logps/chosen": -19.781707763671875,
      "logps/rejected": -80.69671630859375,
      "loss": 0.1164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.387997627258301,
      "rewards/margins": 2.387997627258301,
      "rewards/rejected": 0.0,
      "step": 1005
    },
    {
      "epoch": 5.620111731843576,
      "grad_norm": 4.710831507158189,
      "learning_rate": 4.099563879701629e-06,
      "logits/chosen": -3.282404899597168,
      "logits/rejected": -3.245091676712036,
      "logps/chosen": -5.696804046630859,
      "logps/rejected": -50.02850341796875,
      "loss": 0.1531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2988271713256836,
      "rewards/margins": 2.2988271713256836,
      "rewards/rejected": 0.0,
      "step": 1006
    },
    {
      "epoch": 5.625698324022347,
      "grad_norm": 3.8293266662127143,
      "learning_rate": 4.097220455202746e-06,
      "logits/chosen": -3.6073317527770996,
      "logits/rejected": -3.524784803390503,
      "logps/chosen": -5.248610019683838,
      "logps/rejected": -55.40289306640625,
      "loss": 0.1486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.442505359649658,
      "rewards/margins": 2.442505359649658,
      "rewards/rejected": 0.0,
      "step": 1007
    },
    {
      "epoch": 5.631284916201118,
      "grad_norm": 4.2707109777907855,
      "learning_rate": 4.094874656949817e-06,
      "logits/chosen": -3.4131546020507812,
      "logits/rejected": -3.6516430377960205,
      "logps/chosen": -4.395986557006836,
      "logps/rejected": -126.12362670898438,
      "loss": 0.1696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8860347270965576,
      "rewards/margins": 1.8860347270965576,
      "rewards/rejected": 0.0,
      "step": 1008
    },
    {
      "epoch": 5.636871508379889,
      "grad_norm": 3.263495415339057,
      "learning_rate": 4.092526488429116e-06,
      "logits/chosen": -3.5473246574401855,
      "logits/rejected": -3.3235366344451904,
      "logps/chosen": -5.21908712387085,
      "logps/rejected": -49.69849395751953,
      "loss": 0.1535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6283087730407715,
      "rewards/margins": 2.6283087730407715,
      "rewards/rejected": 0.0,
      "step": 1009
    },
    {
      "epoch": 5.64245810055866,
      "grad_norm": 11.058143121036087,
      "learning_rate": 4.090175953130439e-06,
      "logits/chosen": -4.076676368713379,
      "logits/rejected": -3.895958423614502,
      "logps/chosen": -11.839064598083496,
      "logps/rejected": -31.289981842041016,
      "loss": 0.2843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3675286769866943,
      "rewards/margins": 2.3675286769866943,
      "rewards/rejected": 0.0,
      "step": 1010
    },
    {
      "epoch": 5.648044692737431,
      "grad_norm": 3.1430239035908536,
      "learning_rate": 4.087823054547103e-06,
      "logits/chosen": -3.720221757888794,
      "logits/rejected": -3.5819404125213623,
      "logps/chosen": -12.541898727416992,
      "logps/rejected": -45.08509826660156,
      "loss": 0.1127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.13726806640625,
      "rewards/margins": 3.13726806640625,
      "rewards/rejected": 0.0,
      "step": 1011
    },
    {
      "epoch": 5.653631284916202,
      "grad_norm": 6.703673312260912,
      "learning_rate": 4.08546779617593e-06,
      "logits/chosen": -3.578301429748535,
      "logits/rejected": -3.5152969360351562,
      "logps/chosen": -5.297768592834473,
      "logps/rejected": -73.86652374267578,
      "loss": 0.1775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6423366069793701,
      "rewards/margins": 1.6423366069793701,
      "rewards/rejected": 0.0,
      "step": 1012
    },
    {
      "epoch": 5.659217877094972,
      "grad_norm": 4.619197571452185,
      "learning_rate": 4.083110181517257e-06,
      "logits/chosen": -3.6401684284210205,
      "logits/rejected": -3.504210948944092,
      "logps/chosen": -9.954153060913086,
      "logps/rejected": -27.57164764404297,
      "loss": 0.1504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6421775817871094,
      "rewards/margins": 2.6421775817871094,
      "rewards/rejected": 0.0,
      "step": 1013
    },
    {
      "epoch": 5.664804469273743,
      "grad_norm": 5.101889122467853,
      "learning_rate": 4.080750214074917e-06,
      "logits/chosen": -3.553893804550171,
      "logits/rejected": -3.784003734588623,
      "logps/chosen": -2.6529793739318848,
      "logps/rejected": -43.93096160888672,
      "loss": 0.1332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.454346179962158,
      "rewards/margins": 2.454346179962158,
      "rewards/rejected": 0.0,
      "step": 1014
    },
    {
      "epoch": 5.670391061452514,
      "grad_norm": 11.451906877935608,
      "learning_rate": 4.078387897356243e-06,
      "logits/chosen": -3.7262604236602783,
      "logits/rejected": -3.850750684738159,
      "logps/chosen": -9.218001365661621,
      "logps/rejected": -74.10078430175781,
      "loss": 0.1836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.483046770095825,
      "rewards/margins": 2.483046770095825,
      "rewards/rejected": 0.0,
      "step": 1015
    },
    {
      "epoch": 5.675977653631285,
      "grad_norm": 5.098101385795546,
      "learning_rate": 4.0760232348720575e-06,
      "logits/chosen": -3.245898962020874,
      "logits/rejected": -3.3993403911590576,
      "logps/chosen": -7.976612567901611,
      "logps/rejected": -81.99029541015625,
      "loss": 0.1428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.7710864543914795,
      "rewards/margins": 1.7710864543914795,
      "rewards/rejected": 0.0,
      "step": 1016
    },
    {
      "epoch": 5.681564245810056,
      "grad_norm": 6.977218186686033,
      "learning_rate": 4.073656230136671e-06,
      "logits/chosen": -3.7827043533325195,
      "logits/rejected": -3.6966960430145264,
      "logps/chosen": -8.915771484375,
      "logps/rejected": -38.110076904296875,
      "loss": 0.1621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9319510459899902,
      "rewards/margins": 2.9319510459899902,
      "rewards/rejected": 0.0,
      "step": 1017
    },
    {
      "epoch": 5.687150837988827,
      "grad_norm": 8.555366573891881,
      "learning_rate": 4.071286886667873e-06,
      "logits/chosen": -3.3278136253356934,
      "logits/rejected": -3.283686876296997,
      "logps/chosen": -4.663308143615723,
      "logps/rejected": -36.57158279418945,
      "loss": 0.1562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5245814323425293,
      "rewards/margins": 2.5245814323425293,
      "rewards/rejected": 0.0,
      "step": 1018
    },
    {
      "epoch": 5.692737430167598,
      "grad_norm": 4.421165178216936,
      "learning_rate": 4.068915207986931e-06,
      "logits/chosen": -3.6146764755249023,
      "logits/rejected": -3.5973522663116455,
      "logps/chosen": -2.8483362197875977,
      "logps/rejected": -45.48207092285156,
      "loss": 0.1311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6373906135559082,
      "rewards/margins": 1.6373906135559082,
      "rewards/rejected": 0.0,
      "step": 1019
    },
    {
      "epoch": 5.698324022346369,
      "grad_norm": 8.896500362992548,
      "learning_rate": 4.066541197618582e-06,
      "logits/chosen": -3.7063636779785156,
      "logits/rejected": -3.6952147483825684,
      "logps/chosen": -3.4385271072387695,
      "logps/rejected": -38.03120040893555,
      "loss": 0.1823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.317680597305298,
      "rewards/margins": 2.317680597305298,
      "rewards/rejected": 0.0,
      "step": 1020
    },
    {
      "epoch": 5.698324022346369,
      "eval_logits/chosen": -3.3550515174865723,
      "eval_logits/rejected": -3.444105625152588,
      "eval_logps/chosen": -31.41312026977539,
      "eval_logps/rejected": -46.07327651977539,
      "eval_loss": 1.0127313137054443,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.13430775701999664,
      "eval_rewards/margins": -0.13430775701999664,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5405,
      "eval_samples_per_second": 12.138,
      "eval_steps_per_second": 0.392,
      "step": 1020
    },
    {
      "epoch": 5.70391061452514,
      "grad_norm": 4.227463641291707,
      "learning_rate": 4.064164859091027e-06,
      "logits/chosen": -3.71685528755188,
      "logits/rejected": -3.6321969032287598,
      "logps/chosen": -7.337883472442627,
      "logps/rejected": -43.35350036621094,
      "loss": 0.1535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8826847076416016,
      "rewards/margins": 1.8826847076416016,
      "rewards/rejected": 0.0,
      "step": 1021
    },
    {
      "epoch": 5.709497206703911,
      "grad_norm": 4.221846489685135,
      "learning_rate": 4.061786195935928e-06,
      "logits/chosen": -3.628584384918213,
      "logits/rejected": -3.642390012741089,
      "logps/chosen": -5.716330528259277,
      "logps/rejected": -53.29872512817383,
      "loss": 0.1664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1366515159606934,
      "rewards/margins": 2.1366515159606934,
      "rewards/rejected": 0.0,
      "step": 1022
    },
    {
      "epoch": 5.715083798882682,
      "grad_norm": 3.336742107283178,
      "learning_rate": 4.059405211688404e-06,
      "logits/chosen": -3.5016579627990723,
      "logits/rejected": -3.4032227993011475,
      "logps/chosen": -7.556942462921143,
      "logps/rejected": -42.7540283203125,
      "loss": 0.1408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4189674854278564,
      "rewards/margins": 3.4189674854278564,
      "rewards/rejected": 0.0,
      "step": 1023
    },
    {
      "epoch": 5.720670391061453,
      "grad_norm": 3.772839380865515,
      "learning_rate": 4.0570219098870206e-06,
      "logits/chosen": -3.494335174560547,
      "logits/rejected": -3.56472110748291,
      "logps/chosen": -6.003293991088867,
      "logps/rejected": -34.832759857177734,
      "loss": 0.1376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.185802459716797,
      "rewards/margins": 2.185802459716797,
      "rewards/rejected": 0.0,
      "step": 1024
    },
    {
      "epoch": 5.726256983240224,
      "grad_norm": 2.6029788440493715,
      "learning_rate": 4.054636294073789e-06,
      "logits/chosen": -3.5397021770477295,
      "logits/rejected": -3.569044351577759,
      "logps/chosen": -8.088047981262207,
      "logps/rejected": -27.180355072021484,
      "loss": 0.1247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.046342372894287,
      "rewards/margins": 3.046342372894287,
      "rewards/rejected": 0.0,
      "step": 1025
    },
    {
      "epoch": 5.731843575418995,
      "grad_norm": 1.9738240979319797,
      "learning_rate": 4.052248367794158e-06,
      "logits/chosen": -3.546809196472168,
      "logits/rejected": -3.483896017074585,
      "logps/chosen": -7.895759582519531,
      "logps/rejected": -50.68574142456055,
      "loss": 0.1154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.300283908843994,
      "rewards/margins": 2.300283908843994,
      "rewards/rejected": 0.0,
      "step": 1026
    },
    {
      "epoch": 5.737430167597766,
      "grad_norm": 3.248287301656075,
      "learning_rate": 4.0498581345970125e-06,
      "logits/chosen": -3.5969362258911133,
      "logits/rejected": -4.0273237228393555,
      "logps/chosen": -4.940871238708496,
      "logps/rejected": -65.68199920654297,
      "loss": 0.1548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.099397659301758,
      "rewards/margins": 2.099397659301758,
      "rewards/rejected": 0.0,
      "step": 1027
    },
    {
      "epoch": 5.743016759776537,
      "grad_norm": 5.872062950803259,
      "learning_rate": 4.0474655980346645e-06,
      "logits/chosen": -3.9450953006744385,
      "logits/rejected": -3.9359230995178223,
      "logps/chosen": -6.9374918937683105,
      "logps/rejected": -33.195770263671875,
      "loss": 0.1732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3533854484558105,
      "rewards/margins": 2.3533854484558105,
      "rewards/rejected": 0.0,
      "step": 1028
    },
    {
      "epoch": 5.748603351955307,
      "grad_norm": 2.6611177978988048,
      "learning_rate": 4.04507076166285e-06,
      "logits/chosen": -3.5976529121398926,
      "logits/rejected": -3.590365171432495,
      "logps/chosen": -12.917041778564453,
      "logps/rejected": -37.04265594482422,
      "loss": 0.1053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6627421379089355,
      "rewards/margins": 3.6627421379089355,
      "rewards/rejected": 0.0,
      "step": 1029
    },
    {
      "epoch": 5.754189944134078,
      "grad_norm": 3.442777279212277,
      "learning_rate": 4.04267362904072e-06,
      "logits/chosen": -3.7778587341308594,
      "logits/rejected": -3.774622678756714,
      "logps/chosen": -6.781364917755127,
      "logps/rejected": -49.09609603881836,
      "loss": 0.1659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.561772346496582,
      "rewards/margins": 2.561772346496582,
      "rewards/rejected": 0.0,
      "step": 1030
    },
    {
      "epoch": 5.759776536312849,
      "grad_norm": 3.8641200713271786,
      "learning_rate": 4.040274203730843e-06,
      "logits/chosen": -3.5317721366882324,
      "logits/rejected": -3.5161561965942383,
      "logps/chosen": -9.29543685913086,
      "logps/rejected": -48.63385772705078,
      "loss": 0.1516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.491314172744751,
      "rewards/margins": 2.491314172744751,
      "rewards/rejected": 0.0,
      "step": 1031
    },
    {
      "epoch": 5.76536312849162,
      "grad_norm": 3.9673990317519747,
      "learning_rate": 4.03787248929919e-06,
      "logits/chosen": -3.761096954345703,
      "logits/rejected": -3.630002498626709,
      "logps/chosen": -1.7554534673690796,
      "logps/rejected": -58.13871765136719,
      "loss": 0.1581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5760011672973633,
      "rewards/margins": 1.5760011672973633,
      "rewards/rejected": 0.0,
      "step": 1032
    },
    {
      "epoch": 5.770949720670391,
      "grad_norm": 6.238780278203844,
      "learning_rate": 4.0354684893151395e-06,
      "logits/chosen": -3.524676561355591,
      "logits/rejected": -3.3707475662231445,
      "logps/chosen": -6.341963291168213,
      "logps/rejected": -38.045814514160156,
      "loss": 0.1727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9453020095825195,
      "rewards/margins": 2.9453020095825195,
      "rewards/rejected": 0.0,
      "step": 1033
    },
    {
      "epoch": 5.776536312849162,
      "grad_norm": 4.007018434358656,
      "learning_rate": 4.033062207351461e-06,
      "logits/chosen": -3.557126522064209,
      "logits/rejected": -3.545461654663086,
      "logps/chosen": -9.435894012451172,
      "logps/rejected": -44.42424774169922,
      "loss": 0.1343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5184974670410156,
      "rewards/margins": 2.5184974670410156,
      "rewards/rejected": 0.0,
      "step": 1034
    },
    {
      "epoch": 5.782122905027933,
      "grad_norm": 3.085285786693182,
      "learning_rate": 4.030653646984321e-06,
      "logits/chosen": -3.6359152793884277,
      "logits/rejected": -3.284045934677124,
      "logps/chosen": -3.9204912185668945,
      "logps/rejected": -52.480812072753906,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.86320960521698,
      "rewards/margins": 1.86320960521698,
      "rewards/rejected": 0.0,
      "step": 1035
    },
    {
      "epoch": 5.787709497206704,
      "grad_norm": 11.033980510298477,
      "learning_rate": 4.028242811793265e-06,
      "logits/chosen": -3.5492234230041504,
      "logits/rejected": -3.5445568561553955,
      "logps/chosen": -21.047380447387695,
      "logps/rejected": -39.977439880371094,
      "loss": 0.2491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.368694543838501,
      "rewards/margins": 3.368694543838501,
      "rewards/rejected": 0.0,
      "step": 1036
    },
    {
      "epoch": 5.793296089385475,
      "grad_norm": 4.98693550445972,
      "learning_rate": 4.025829705361225e-06,
      "logits/chosen": -3.5596015453338623,
      "logits/rejected": -3.6629374027252197,
      "logps/chosen": -6.3745198249816895,
      "logps/rejected": -47.27210235595703,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.575570821762085,
      "rewards/margins": 2.575570821762085,
      "rewards/rejected": 0.0,
      "step": 1037
    },
    {
      "epoch": 5.798882681564246,
      "grad_norm": 11.051785635195845,
      "learning_rate": 4.023414331274509e-06,
      "logits/chosen": -3.405806303024292,
      "logits/rejected": -3.2989957332611084,
      "logps/chosen": -18.82628059387207,
      "logps/rejected": -55.011993408203125,
      "loss": 0.2232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.136418342590332,
      "rewards/margins": 2.136418342590332,
      "rewards/rejected": 0.0,
      "step": 1038
    },
    {
      "epoch": 5.804469273743017,
      "grad_norm": 4.150025068819089,
      "learning_rate": 4.020996693122791e-06,
      "logits/chosen": -3.849193811416626,
      "logits/rejected": -3.7787649631500244,
      "logps/chosen": -5.509788513183594,
      "logps/rejected": -39.20134735107422,
      "loss": 0.1386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.930098056793213,
      "rewards/margins": 2.930098056793213,
      "rewards/rejected": 0.0,
      "step": 1039
    },
    {
      "epoch": 5.810055865921788,
      "grad_norm": 10.819373292300769,
      "learning_rate": 4.018576794499112e-06,
      "logits/chosen": -3.4298171997070312,
      "logits/rejected": -3.3883469104766846,
      "logps/chosen": -15.516948699951172,
      "logps/rejected": -63.428314208984375,
      "loss": 0.1846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1211042404174805,
      "rewards/margins": 3.1211042404174805,
      "rewards/rejected": 0.0,
      "step": 1040
    },
    {
      "epoch": 5.810055865921788,
      "eval_logits/chosen": -3.3817038536071777,
      "eval_logits/rejected": -3.477989673614502,
      "eval_logps/chosen": -33.27691650390625,
      "eval_logps/rejected": -47.83739471435547,
      "eval_loss": 1.0551259517669678,
      "eval_rewards/accuracies": 0.4749999940395355,
      "eval_rewards/chosen": -0.32068735361099243,
      "eval_rewards/margins": -0.32068735361099243,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5527,
      "eval_samples_per_second": 12.132,
      "eval_steps_per_second": 0.391,
      "step": 1040
    },
    {
      "epoch": 5.815642458100559,
      "grad_norm": 3.6936981250526237,
      "learning_rate": 4.016154638999873e-06,
      "logits/chosen": -3.1566364765167236,
      "logits/rejected": -2.979246139526367,
      "logps/chosen": -12.28927993774414,
      "logps/rejected": -43.493892669677734,
      "loss": 0.1339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.047236680984497,
      "rewards/margins": 3.047236680984497,
      "rewards/rejected": 0.0,
      "step": 1041
    },
    {
      "epoch": 5.82122905027933,
      "grad_norm": 3.9650424661015746,
      "learning_rate": 4.013730230224828e-06,
      "logits/chosen": -3.4462103843688965,
      "logits/rejected": -3.3468823432922363,
      "logps/chosen": -9.76961898803711,
      "logps/rejected": -32.5555419921875,
      "loss": 0.1339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.313230514526367,
      "rewards/margins": 2.313230514526367,
      "rewards/rejected": 0.0,
      "step": 1042
    },
    {
      "epoch": 5.826815642458101,
      "grad_norm": 3.9322100934918027,
      "learning_rate": 4.01130357177708e-06,
      "logits/chosen": -3.4733026027679443,
      "logits/rejected": -3.4808950424194336,
      "logps/chosen": -8.831025123596191,
      "logps/rejected": -53.64814758300781,
      "loss": 0.1291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.124236583709717,
      "rewards/margins": 2.124236583709717,
      "rewards/rejected": 0.0,
      "step": 1043
    },
    {
      "epoch": 5.832402234636872,
      "grad_norm": 6.190306507383057,
      "learning_rate": 4.008874667263076e-06,
      "logits/chosen": -3.846050262451172,
      "logits/rejected": -3.814981698989868,
      "logps/chosen": -9.160316467285156,
      "logps/rejected": -59.777488708496094,
      "loss": 0.1652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3866941928863525,
      "rewards/margins": 2.3866941928863525,
      "rewards/rejected": 0.0,
      "step": 1044
    },
    {
      "epoch": 5.837988826815643,
      "grad_norm": 3.575237800905492,
      "learning_rate": 4.006443520292601e-06,
      "logits/chosen": -3.487407922744751,
      "logits/rejected": -3.4807937145233154,
      "logps/chosen": -7.345776557922363,
      "logps/rejected": -65.09141540527344,
      "loss": 0.1416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.631862163543701,
      "rewards/margins": 2.631862163543701,
      "rewards/rejected": 0.0,
      "step": 1045
    },
    {
      "epoch": 5.843575418994414,
      "grad_norm": 3.755351806443463,
      "learning_rate": 4.004010134478772e-06,
      "logits/chosen": -3.311641216278076,
      "logits/rejected": -3.3476481437683105,
      "logps/chosen": -7.606756687164307,
      "logps/rejected": -30.312942504882812,
      "loss": 0.1515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6067628860473633,
      "rewards/margins": 2.6067628860473633,
      "rewards/rejected": 0.0,
      "step": 1046
    },
    {
      "epoch": 5.849162011173185,
      "grad_norm": 3.3017672981680657,
      "learning_rate": 4.001574513438032e-06,
      "logits/chosen": -3.3316097259521484,
      "logits/rejected": -3.205026626586914,
      "logps/chosen": -2.781273365020752,
      "logps/rejected": -60.679595947265625,
      "loss": 0.1461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0731213092803955,
      "rewards/margins": 2.0731213092803955,
      "rewards/rejected": 0.0,
      "step": 1047
    },
    {
      "epoch": 5.854748603351956,
      "grad_norm": 3.646341281630532,
      "learning_rate": 3.999136660790151e-06,
      "logits/chosen": -3.7104759216308594,
      "logits/rejected": -3.616816759109497,
      "logps/chosen": -21.500778198242188,
      "logps/rejected": -55.23127365112305,
      "loss": 0.1103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8235340118408203,
      "rewards/margins": 3.8235340118408203,
      "rewards/rejected": 0.0,
      "step": 1048
    },
    {
      "epoch": 5.860335195530726,
      "grad_norm": 4.2622895528534155,
      "learning_rate": 3.996696580158211e-06,
      "logits/chosen": -3.447251558303833,
      "logits/rejected": -3.4925875663757324,
      "logps/chosen": -10.673684120178223,
      "logps/rejected": -56.24845886230469,
      "loss": 0.1426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.315687656402588,
      "rewards/margins": 3.315687656402588,
      "rewards/rejected": 0.0,
      "step": 1049
    },
    {
      "epoch": 5.865921787709497,
      "grad_norm": 23.60757879290289,
      "learning_rate": 3.994254275168604e-06,
      "logits/chosen": -3.3782122135162354,
      "logits/rejected": -3.393197774887085,
      "logps/chosen": -11.817492485046387,
      "logps/rejected": -38.91559982299805,
      "loss": 0.317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8674440383911133,
      "rewards/margins": 2.8674440383911133,
      "rewards/rejected": 0.0,
      "step": 1050
    },
    {
      "epoch": 5.871508379888268,
      "grad_norm": 3.6087390110853583,
      "learning_rate": 3.9918097494510325e-06,
      "logits/chosen": -3.9880552291870117,
      "logits/rejected": -3.4267828464508057,
      "logps/chosen": -8.60472583770752,
      "logps/rejected": -66.80339813232422,
      "loss": 0.1673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.110149383544922,
      "rewards/margins": 2.110149383544922,
      "rewards/rejected": 0.0,
      "step": 1051
    },
    {
      "epoch": 5.877094972067039,
      "grad_norm": 7.690572250082997,
      "learning_rate": 3.989363006638497e-06,
      "logits/chosen": -3.5242624282836914,
      "logits/rejected": -3.4739761352539062,
      "logps/chosen": -3.4328463077545166,
      "logps/rejected": -41.25275421142578,
      "loss": 0.2141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.573899507522583,
      "rewards/margins": 1.573899507522583,
      "rewards/rejected": 0.0,
      "step": 1052
    },
    {
      "epoch": 5.88268156424581,
      "grad_norm": 3.811969526388538,
      "learning_rate": 3.986914050367292e-06,
      "logits/chosen": -3.6549835205078125,
      "logits/rejected": -3.5313830375671387,
      "logps/chosen": -33.865875244140625,
      "logps/rejected": -50.51472473144531,
      "loss": 0.1415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9261369705200195,
      "rewards/margins": 2.9261369705200195,
      "rewards/rejected": 0.0,
      "step": 1053
    },
    {
      "epoch": 5.888268156424581,
      "grad_norm": 4.1218090572473045,
      "learning_rate": 3.984462884277003e-06,
      "logits/chosen": -3.3299448490142822,
      "logits/rejected": -3.5784590244293213,
      "logps/chosen": -7.39582633972168,
      "logps/rejected": -53.16732406616211,
      "loss": 0.1377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.214876413345337,
      "rewards/margins": 3.214876413345337,
      "rewards/rejected": 0.0,
      "step": 1054
    },
    {
      "epoch": 5.893854748603352,
      "grad_norm": 2.665184005639575,
      "learning_rate": 3.982009512010501e-06,
      "logits/chosen": -3.791231870651245,
      "logits/rejected": -3.6717522144317627,
      "logps/chosen": -4.817201614379883,
      "logps/rejected": -98.94462585449219,
      "loss": 0.0983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.759626865386963,
      "rewards/margins": 2.759626865386963,
      "rewards/rejected": 0.0,
      "step": 1055
    },
    {
      "epoch": 5.899441340782123,
      "grad_norm": 3.186158522575311,
      "learning_rate": 3.979553937213931e-06,
      "logits/chosen": -3.339613914489746,
      "logits/rejected": -3.1712162494659424,
      "logps/chosen": -2.490313768386841,
      "logps/rejected": -90.51199340820312,
      "loss": 0.1128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.893510103225708,
      "rewards/margins": 1.893510103225708,
      "rewards/rejected": 0.0,
      "step": 1056
    },
    {
      "epoch": 5.905027932960894,
      "grad_norm": 4.269819659366416,
      "learning_rate": 3.9770961635367175e-06,
      "logits/chosen": -3.5573792457580566,
      "logits/rejected": -3.4798436164855957,
      "logps/chosen": -25.1809024810791,
      "logps/rejected": -60.40021514892578,
      "loss": 0.136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2069852352142334,
      "rewards/margins": 3.2069852352142334,
      "rewards/rejected": 0.0,
      "step": 1057
    },
    {
      "epoch": 5.910614525139665,
      "grad_norm": 3.685304749079024,
      "learning_rate": 3.9746361946315484e-06,
      "logits/chosen": -3.591132879257202,
      "logits/rejected": -3.4893863201141357,
      "logps/chosen": -11.836698532104492,
      "logps/rejected": -56.23677062988281,
      "loss": 0.1263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.19769549369812,
      "rewards/margins": 3.19769549369812,
      "rewards/rejected": 0.0,
      "step": 1058
    },
    {
      "epoch": 5.916201117318436,
      "grad_norm": 8.841719131738664,
      "learning_rate": 3.972174034154375e-06,
      "logits/chosen": -3.5077831745147705,
      "logits/rejected": -3.556356430053711,
      "logps/chosen": -3.9245083332061768,
      "logps/rejected": -31.670116424560547,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7649017572402954,
      "rewards/margins": 1.7649017572402954,
      "rewards/rejected": 0.0,
      "step": 1059
    },
    {
      "epoch": 5.921787709497207,
      "grad_norm": 3.2555614455658546,
      "learning_rate": 3.969709685764407e-06,
      "logits/chosen": -3.4020347595214844,
      "logits/rejected": -3.5016863346099854,
      "logps/chosen": -8.872907638549805,
      "logps/rejected": -55.13031768798828,
      "loss": 0.1477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.187807083129883,
      "rewards/margins": 2.187807083129883,
      "rewards/rejected": 0.0,
      "step": 1060
    },
    {
      "epoch": 5.921787709497207,
      "eval_logits/chosen": -3.3889708518981934,
      "eval_logits/rejected": -3.48614764213562,
      "eval_logps/chosen": -33.02777862548828,
      "eval_logps/rejected": -47.85812759399414,
      "eval_loss": 1.0503984689712524,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.29577335715293884,
      "eval_rewards/margins": -0.29577335715293884,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.4835,
      "eval_samples_per_second": 12.165,
      "eval_steps_per_second": 0.392,
      "step": 1060
    },
    {
      "epoch": 5.927374301675978,
      "grad_norm": 12.379570320535102,
      "learning_rate": 3.967243153124105e-06,
      "logits/chosen": -3.6995716094970703,
      "logits/rejected": -3.6105897426605225,
      "logps/chosen": -20.662715911865234,
      "logps/rejected": -56.20240020751953,
      "loss": 0.2016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5521750450134277,
      "rewards/margins": 2.5521750450134277,
      "rewards/rejected": 0.0,
      "step": 1061
    },
    {
      "epoch": 5.932960893854749,
      "grad_norm": 3.101387358671692,
      "learning_rate": 3.964774439899177e-06,
      "logits/chosen": -3.6016225814819336,
      "logits/rejected": -3.5821430683135986,
      "logps/chosen": -23.76500701904297,
      "logps/rejected": -58.93753433227539,
      "loss": 0.1037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8057727813720703,
      "rewards/margins": 3.8057727813720703,
      "rewards/rejected": 0.0,
      "step": 1062
    },
    {
      "epoch": 5.93854748603352,
      "grad_norm": 5.332568405786988,
      "learning_rate": 3.962303549758567e-06,
      "logits/chosen": -3.4951725006103516,
      "logits/rejected": -3.4829611778259277,
      "logps/chosen": -15.85065746307373,
      "logps/rejected": -61.10017395019531,
      "loss": 0.1766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2119369506835938,
      "rewards/margins": 3.2119369506835938,
      "rewards/rejected": 0.0,
      "step": 1063
    },
    {
      "epoch": 5.94413407821229,
      "grad_norm": 4.017401421323797,
      "learning_rate": 3.959830486374462e-06,
      "logits/chosen": -3.3531060218811035,
      "logits/rejected": -3.460519790649414,
      "logps/chosen": -2.8554153442382812,
      "logps/rejected": -74.76547241210938,
      "loss": 0.1408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7529418468475342,
      "rewards/margins": 1.7529418468475342,
      "rewards/rejected": 0.0,
      "step": 1064
    },
    {
      "epoch": 5.949720670391061,
      "grad_norm": 5.489233007627254,
      "learning_rate": 3.957355253422273e-06,
      "logits/chosen": -3.0031938552856445,
      "logits/rejected": -3.148664951324463,
      "logps/chosen": -8.71554946899414,
      "logps/rejected": -52.82553482055664,
      "loss": 0.1703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.278548002243042,
      "rewards/margins": 2.278548002243042,
      "rewards/rejected": 0.0,
      "step": 1065
    },
    {
      "epoch": 5.955307262569832,
      "grad_norm": 7.1509093718333645,
      "learning_rate": 3.954877854580635e-06,
      "logits/chosen": -3.9418160915374756,
      "logits/rejected": -3.928135395050049,
      "logps/chosen": -1.6878254413604736,
      "logps/rejected": -56.20683288574219,
      "loss": 0.1799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9391365051269531,
      "rewards/margins": 1.9391365051269531,
      "rewards/rejected": 0.0,
      "step": 1066
    },
    {
      "epoch": 5.960893854748603,
      "grad_norm": 3.2870701101621234,
      "learning_rate": 3.952398293531406e-06,
      "logits/chosen": -3.326227903366089,
      "logits/rejected": -3.244715929031372,
      "logps/chosen": -15.793505668640137,
      "logps/rejected": -45.685401916503906,
      "loss": 0.1386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.635570526123047,
      "rewards/margins": 2.635570526123047,
      "rewards/rejected": 0.0,
      "step": 1067
    },
    {
      "epoch": 5.966480446927374,
      "grad_norm": 2.6865696039641254,
      "learning_rate": 3.949916573959655e-06,
      "logits/chosen": -2.9134507179260254,
      "logits/rejected": -2.9102890491485596,
      "logps/chosen": -4.9715423583984375,
      "logps/rejected": -50.376739501953125,
      "loss": 0.1281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.518960952758789,
      "rewards/margins": 2.518960952758789,
      "rewards/rejected": 0.0,
      "step": 1068
    },
    {
      "epoch": 5.972067039106145,
      "grad_norm": 4.991714077365185,
      "learning_rate": 3.9474326995536595e-06,
      "logits/chosen": -3.8167779445648193,
      "logits/rejected": -3.3905787467956543,
      "logps/chosen": -16.52462387084961,
      "logps/rejected": -54.312744140625,
      "loss": 0.1533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7363076210021973,
      "rewards/margins": 2.7363076210021973,
      "rewards/rejected": 0.0,
      "step": 1069
    },
    {
      "epoch": 5.977653631284916,
      "grad_norm": 6.486860605582879,
      "learning_rate": 3.944946674004898e-06,
      "logits/chosen": -3.7118113040924072,
      "logits/rejected": -3.466515302658081,
      "logps/chosen": -20.06594467163086,
      "logps/rejected": -52.286468505859375,
      "loss": 0.1858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.5914969444274902,
      "rewards/margins": 2.5914969444274902,
      "rewards/rejected": 0.0,
      "step": 1070
    },
    {
      "epoch": 5.983240223463687,
      "grad_norm": 3.3543176552058815,
      "learning_rate": 3.942458501008049e-06,
      "logits/chosen": -3.183274984359741,
      "logits/rejected": -3.2507340908050537,
      "logps/chosen": -8.660908699035645,
      "logps/rejected": -35.41450500488281,
      "loss": 0.1389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9135220050811768,
      "rewards/margins": 1.9135220050811768,
      "rewards/rejected": 0.0,
      "step": 1071
    },
    {
      "epoch": 5.988826815642458,
      "grad_norm": 4.90803235200529,
      "learning_rate": 3.939968184260979e-06,
      "logits/chosen": -3.922919511795044,
      "logits/rejected": -3.8492355346679688,
      "logps/chosen": -3.9412736892700195,
      "logps/rejected": -38.316898345947266,
      "loss": 0.1707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.281545400619507,
      "rewards/margins": 2.281545400619507,
      "rewards/rejected": 0.0,
      "step": 1072
    },
    {
      "epoch": 5.994413407821229,
      "grad_norm": 4.603645775486941,
      "learning_rate": 3.937475727464742e-06,
      "logits/chosen": -3.3629138469696045,
      "logits/rejected": -3.4302260875701904,
      "logps/chosen": -4.195798397064209,
      "logps/rejected": -67.29536437988281,
      "loss": 0.1363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.379310369491577,
      "rewards/margins": 2.379310369491577,
      "rewards/rejected": 0.0,
      "step": 1073
    },
    {
      "epoch": 6.0,
      "grad_norm": 2.472306553874945,
      "learning_rate": 3.934981134323575e-06,
      "logits/chosen": -3.511173725128174,
      "logits/rejected": -3.453517436981201,
      "logps/chosen": -3.160811424255371,
      "logps/rejected": -34.299842834472656,
      "loss": 0.1194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.132629632949829,
      "rewards/margins": 2.132629632949829,
      "rewards/rejected": 0.0,
      "step": 1074
    },
    {
      "epoch": 6.005586592178771,
      "grad_norm": 3.671758313653127,
      "learning_rate": 3.932484408544886e-06,
      "logits/chosen": -3.5809106826782227,
      "logits/rejected": -3.606673002243042,
      "logps/chosen": -5.4994425773620605,
      "logps/rejected": -43.17390441894531,
      "loss": 0.1341,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.299943685531616,
      "rewards/margins": 2.299943685531616,
      "rewards/rejected": 0.0,
      "step": 1075
    },
    {
      "epoch": 6.011173184357542,
      "grad_norm": 2.2372304182737883,
      "learning_rate": 3.929985553839254e-06,
      "logits/chosen": -3.339599847793579,
      "logits/rejected": -3.364102840423584,
      "logps/chosen": -7.34370231628418,
      "logps/rejected": -26.639598846435547,
      "loss": 0.1201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.912445306777954,
      "rewards/margins": 2.912445306777954,
      "rewards/rejected": 0.0,
      "step": 1076
    },
    {
      "epoch": 6.016759776536313,
      "grad_norm": 3.68447165952507,
      "learning_rate": 3.927484573920422e-06,
      "logits/chosen": -3.311675548553467,
      "logits/rejected": -3.426536798477173,
      "logps/chosen": -29.198030471801758,
      "logps/rejected": -80.43093872070312,
      "loss": 0.1539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.001962661743164,
      "rewards/margins": 4.001962661743164,
      "rewards/rejected": 0.0,
      "step": 1077
    },
    {
      "epoch": 6.022346368715084,
      "grad_norm": 5.265443145367432,
      "learning_rate": 3.9249814725052935e-06,
      "logits/chosen": -3.533015012741089,
      "logits/rejected": -3.485283851623535,
      "logps/chosen": -18.018535614013672,
      "logps/rejected": -41.99738693237305,
      "loss": 0.1691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.173248767852783,
      "rewards/margins": 3.173248767852783,
      "rewards/rejected": 0.0,
      "step": 1078
    },
    {
      "epoch": 6.027932960893855,
      "grad_norm": 2.8644473233117664,
      "learning_rate": 3.922476253313921e-06,
      "logits/chosen": -3.5161991119384766,
      "logits/rejected": -3.4835028648376465,
      "logps/chosen": -6.319921493530273,
      "logps/rejected": -64.60212707519531,
      "loss": 0.1132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3243777751922607,
      "rewards/margins": 2.3243777751922607,
      "rewards/rejected": 0.0,
      "step": 1079
    },
    {
      "epoch": 6.033519553072626,
      "grad_norm": 2.126289607328819,
      "learning_rate": 3.919968920069507e-06,
      "logits/chosen": -3.4131832122802734,
      "logits/rejected": -3.5574710369110107,
      "logps/chosen": -9.780200004577637,
      "logps/rejected": -55.17369842529297,
      "loss": 0.0936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1089978218078613,
      "rewards/margins": 3.1089978218078613,
      "rewards/rejected": 0.0,
      "step": 1080
    },
    {
      "epoch": 6.033519553072626,
      "eval_logits/chosen": -3.4080052375793457,
      "eval_logits/rejected": -3.5125632286071777,
      "eval_logps/chosen": -31.859609603881836,
      "eval_logps/rejected": -48.04511260986328,
      "eval_loss": 1.0683165788650513,
      "eval_rewards/accuracies": 0.44999998807907104,
      "eval_rewards/chosen": -0.17895670235157013,
      "eval_rewards/margins": -0.17895670235157013,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5439,
      "eval_samples_per_second": 12.136,
      "eval_steps_per_second": 0.391,
      "step": 1080
    },
    {
      "epoch": 6.039106145251397,
      "grad_norm": 4.135415074426569,
      "learning_rate": 3.917459476498394e-06,
      "logits/chosen": -3.345837116241455,
      "logits/rejected": -3.2194206714630127,
      "logps/chosen": -8.147448539733887,
      "logps/rejected": -48.570533752441406,
      "loss": 0.1375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4418768882751465,
      "rewards/margins": 3.4418768882751465,
      "rewards/rejected": 0.0,
      "step": 1081
    },
    {
      "epoch": 6.044692737430168,
      "grad_norm": 1.6224816906967556,
      "learning_rate": 3.914947926330065e-06,
      "logits/chosen": -3.591627836227417,
      "logits/rejected": -3.6117630004882812,
      "logps/chosen": -7.291751861572266,
      "logps/rejected": -30.084707260131836,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5902791023254395,
      "rewards/margins": 2.5902791023254395,
      "rewards/rejected": 0.0,
      "step": 1082
    },
    {
      "epoch": 6.050279329608939,
      "grad_norm": 2.4361546585935607,
      "learning_rate": 3.9124342732971274e-06,
      "logits/chosen": -3.478733539581299,
      "logits/rejected": -3.467419147491455,
      "logps/chosen": -9.867010116577148,
      "logps/rejected": -36.859161376953125,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.612853527069092,
      "rewards/margins": 2.612853527069092,
      "rewards/rejected": 0.0,
      "step": 1083
    },
    {
      "epoch": 6.055865921787709,
      "grad_norm": 5.653724159254551,
      "learning_rate": 3.90991852113532e-06,
      "logits/chosen": -3.2665274143218994,
      "logits/rejected": -3.309467077255249,
      "logps/chosen": -4.951417922973633,
      "logps/rejected": -41.53954315185547,
      "loss": 0.1561,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2193386554718018,
      "rewards/margins": 2.2193386554718018,
      "rewards/rejected": 0.0,
      "step": 1084
    },
    {
      "epoch": 6.06145251396648,
      "grad_norm": 7.060803105883795,
      "learning_rate": 3.907400673583497e-06,
      "logits/chosen": -3.481679916381836,
      "logits/rejected": -3.5527923107147217,
      "logps/chosen": -10.077159881591797,
      "logps/rejected": -66.84455871582031,
      "loss": 0.179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5542965531349182,
      "rewards/margins": 0.5542965531349182,
      "rewards/rejected": 0.0,
      "step": 1085
    },
    {
      "epoch": 6.067039106145251,
      "grad_norm": 4.091027004934299,
      "learning_rate": 3.904880734383628e-06,
      "logits/chosen": -3.979783296585083,
      "logits/rejected": -3.776376724243164,
      "logps/chosen": -14.95450210571289,
      "logps/rejected": -72.60733795166016,
      "loss": 0.1236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.812631607055664,
      "rewards/margins": 2.812631607055664,
      "rewards/rejected": 0.0,
      "step": 1086
    },
    {
      "epoch": 6.072625698324022,
      "grad_norm": 3.723041861211464,
      "learning_rate": 3.9023587072807935e-06,
      "logits/chosen": -3.3484647274017334,
      "logits/rejected": -3.3756535053253174,
      "logps/chosen": -4.945589065551758,
      "logps/rejected": -74.54521942138672,
      "loss": 0.1284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.803053855895996,
      "rewards/margins": 1.803053855895996,
      "rewards/rejected": 0.0,
      "step": 1087
    },
    {
      "epoch": 6.078212290502793,
      "grad_norm": 2.195211556547796,
      "learning_rate": 3.899834596023172e-06,
      "logits/chosen": -3.572812080383301,
      "logits/rejected": -3.6594221591949463,
      "logps/chosen": -4.0466108322143555,
      "logps/rejected": -51.35978698730469,
      "loss": 0.137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4294657707214355,
      "rewards/margins": 2.4294657707214355,
      "rewards/rejected": 0.0,
      "step": 1088
    },
    {
      "epoch": 6.083798882681564,
      "grad_norm": 3.5194378767803487,
      "learning_rate": 3.897308404362045e-06,
      "logits/chosen": -3.4659204483032227,
      "logits/rejected": -3.4253180027008057,
      "logps/chosen": -9.750953674316406,
      "logps/rejected": -54.8645133972168,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.62894868850708,
      "rewards/margins": 2.62894868850708,
      "rewards/rejected": 0.0,
      "step": 1089
    },
    {
      "epoch": 6.089385474860335,
      "grad_norm": 2.175609615224989,
      "learning_rate": 3.894780136051782e-06,
      "logits/chosen": -3.7271621227264404,
      "logits/rejected": -3.843825340270996,
      "logps/chosen": -6.210095405578613,
      "logps/rejected": -49.437278747558594,
      "loss": 0.1193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8659887313842773,
      "rewards/margins": 2.8659887313842773,
      "rewards/rejected": 0.0,
      "step": 1090
    },
    {
      "epoch": 6.094972067039106,
      "grad_norm": 3.6401970814413,
      "learning_rate": 3.892249794849841e-06,
      "logits/chosen": -3.5870368480682373,
      "logits/rejected": -3.434736728668213,
      "logps/chosen": -3.2073287963867188,
      "logps/rejected": -60.976806640625,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4046406745910645,
      "rewards/margins": 2.4046406745910645,
      "rewards/rejected": 0.0,
      "step": 1091
    },
    {
      "epoch": 6.100558659217877,
      "grad_norm": 5.107211382032497,
      "learning_rate": 3.889717384516757e-06,
      "logits/chosen": -3.2125446796417236,
      "logits/rejected": -3.251129627227783,
      "logps/chosen": -2.703939437866211,
      "logps/rejected": -35.81232452392578,
      "loss": 0.1882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8834260702133179,
      "rewards/margins": 1.8834260702133179,
      "rewards/rejected": 0.0,
      "step": 1092
    },
    {
      "epoch": 6.106145251396648,
      "grad_norm": 4.599019889076352,
      "learning_rate": 3.887182908816145e-06,
      "logits/chosen": -3.4298510551452637,
      "logits/rejected": -3.3342483043670654,
      "logps/chosen": -6.447969436645508,
      "logps/rejected": -49.514652252197266,
      "loss": 0.1239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.282226324081421,
      "rewards/margins": 2.282226324081421,
      "rewards/rejected": 0.0,
      "step": 1093
    },
    {
      "epoch": 6.111731843575419,
      "grad_norm": 3.490377633774673,
      "learning_rate": 3.884646371514687e-06,
      "logits/chosen": -3.6606833934783936,
      "logits/rejected": -3.395477533340454,
      "logps/chosen": -8.873565673828125,
      "logps/rejected": -38.32856750488281,
      "loss": 0.1135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0603084564208984,
      "rewards/margins": 3.0603084564208984,
      "rewards/rejected": 0.0,
      "step": 1094
    },
    {
      "epoch": 6.11731843575419,
      "grad_norm": 2.0518816600449585,
      "learning_rate": 3.8821077763821295e-06,
      "logits/chosen": -3.4894211292266846,
      "logits/rejected": -3.5451316833496094,
      "logps/chosen": -2.7195024490356445,
      "logps/rejected": -51.453575134277344,
      "loss": 0.1261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.116638660430908,
      "rewards/margins": 2.116638660430908,
      "rewards/rejected": 0.0,
      "step": 1095
    },
    {
      "epoch": 6.122905027932961,
      "grad_norm": 4.409081583584405,
      "learning_rate": 3.879567127191275e-06,
      "logits/chosen": -3.698540449142456,
      "logits/rejected": -3.6335692405700684,
      "logps/chosen": -1.3083503246307373,
      "logps/rejected": -35.52330017089844,
      "loss": 0.131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.889843225479126,
      "rewards/margins": 1.889843225479126,
      "rewards/rejected": 0.0,
      "step": 1096
    },
    {
      "epoch": 6.128491620111732,
      "grad_norm": 5.012425633957552,
      "learning_rate": 3.8770244277179825e-06,
      "logits/chosen": -3.4981701374053955,
      "logits/rejected": -3.4854583740234375,
      "logps/chosen": -4.877096176147461,
      "logps/rejected": -32.275245666503906,
      "loss": 0.1593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.598606586456299,
      "rewards/margins": 2.598606586456299,
      "rewards/rejected": 0.0,
      "step": 1097
    },
    {
      "epoch": 6.134078212290503,
      "grad_norm": 9.241938928499737,
      "learning_rate": 3.874479681741157e-06,
      "logits/chosen": -3.563802480697632,
      "logits/rejected": -3.474966526031494,
      "logps/chosen": -3.811931610107422,
      "logps/rejected": -35.111915588378906,
      "loss": 0.1918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.177809715270996,
      "rewards/margins": 2.177809715270996,
      "rewards/rejected": 0.0,
      "step": 1098
    },
    {
      "epoch": 6.139664804469274,
      "grad_norm": 3.655614668445559,
      "learning_rate": 3.871932893042741e-06,
      "logits/chosen": -3.561814785003662,
      "logits/rejected": -3.5365641117095947,
      "logps/chosen": -14.445624351501465,
      "logps/rejected": -48.516563415527344,
      "loss": 0.1412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7371771335601807,
      "rewards/margins": 2.7371771335601807,
      "rewards/rejected": 0.0,
      "step": 1099
    },
    {
      "epoch": 6.145251396648045,
      "grad_norm": 4.767319717522824,
      "learning_rate": 3.86938406540772e-06,
      "logits/chosen": -3.2303597927093506,
      "logits/rejected": -3.4256081581115723,
      "logps/chosen": -4.793224334716797,
      "logps/rejected": -54.86408996582031,
      "loss": 0.1038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.517869472503662,
      "rewards/margins": 2.517869472503662,
      "rewards/rejected": 0.0,
      "step": 1100
    },
    {
      "epoch": 6.145251396648045,
      "eval_logits/chosen": -3.364673137664795,
      "eval_logits/rejected": -3.463541030883789,
      "eval_logps/chosen": -34.52946090698242,
      "eval_logps/rejected": -49.91057205200195,
      "eval_loss": 1.154297113418579,
      "eval_rewards/accuracies": 0.3499999940395355,
      "eval_rewards/chosen": -0.4459419846534729,
      "eval_rewards/margins": -0.4459419846534729,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5545,
      "eval_samples_per_second": 12.131,
      "eval_steps_per_second": 0.391,
      "step": 1100
    },
    {
      "epoch": 6.150837988826815,
      "grad_norm": 2.3739659796644954,
      "learning_rate": 3.866833202624102e-06,
      "logits/chosen": -3.6250486373901367,
      "logits/rejected": -3.494135618209839,
      "logps/chosen": -10.72762680053711,
      "logps/rejected": -47.60243606567383,
      "loss": 0.1294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.418684482574463,
      "rewards/margins": 3.418684482574463,
      "rewards/rejected": 0.0,
      "step": 1101
    },
    {
      "epoch": 6.156424581005586,
      "grad_norm": 4.144901216570002,
      "learning_rate": 3.864280308482926e-06,
      "logits/chosen": -3.2511091232299805,
      "logits/rejected": -3.2849559783935547,
      "logps/chosen": -13.829930305480957,
      "logps/rejected": -36.344112396240234,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1663379669189453,
      "rewards/margins": 3.1663379669189453,
      "rewards/rejected": 0.0,
      "step": 1102
    },
    {
      "epoch": 6.162011173184357,
      "grad_norm": 2.390663660058111,
      "learning_rate": 3.8617253867782464e-06,
      "logits/chosen": -3.3602139949798584,
      "logits/rejected": -3.460151433944702,
      "logps/chosen": -32.3359375,
      "logps/rejected": -64.6092529296875,
      "loss": 0.1314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.608625888824463,
      "rewards/margins": 4.608625888824463,
      "rewards/rejected": 0.0,
      "step": 1103
    },
    {
      "epoch": 6.167597765363128,
      "grad_norm": 3.8780888632344377,
      "learning_rate": 3.859168441307133e-06,
      "logits/chosen": -3.502506971359253,
      "logits/rejected": -3.4652786254882812,
      "logps/chosen": -9.613666534423828,
      "logps/rejected": -52.91593933105469,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.365778684616089,
      "rewards/margins": 3.365778684616089,
      "rewards/rejected": 0.0,
      "step": 1104
    },
    {
      "epoch": 6.173184357541899,
      "grad_norm": 4.713498358678493,
      "learning_rate": 3.856609475869661e-06,
      "logits/chosen": -3.3392884731292725,
      "logits/rejected": -3.2669341564178467,
      "logps/chosen": -8.483282089233398,
      "logps/rejected": -54.555702209472656,
      "loss": 0.1694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0133042335510254,
      "rewards/margins": 3.0133042335510254,
      "rewards/rejected": 0.0,
      "step": 1105
    },
    {
      "epoch": 6.17877094972067,
      "grad_norm": 2.303983274125225,
      "learning_rate": 3.854048494268907e-06,
      "logits/chosen": -3.4796624183654785,
      "logits/rejected": -3.4786858558654785,
      "logps/chosen": -1.7385833263397217,
      "logps/rejected": -45.107505798339844,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8612430095672607,
      "rewards/margins": 1.8612430095672607,
      "rewards/rejected": 0.0,
      "step": 1106
    },
    {
      "epoch": 6.184357541899441,
      "grad_norm": 3.615828355658339,
      "learning_rate": 3.85148550031095e-06,
      "logits/chosen": -3.4872615337371826,
      "logits/rejected": -3.442690372467041,
      "logps/chosen": -15.520853996276855,
      "logps/rejected": -66.6838150024414,
      "loss": 0.1208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6175453662872314,
      "rewards/margins": 3.6175453662872314,
      "rewards/rejected": 0.0,
      "step": 1107
    },
    {
      "epoch": 6.189944134078212,
      "grad_norm": 5.404968024706134,
      "learning_rate": 3.848920497804851e-06,
      "logits/chosen": -3.6468029022216797,
      "logits/rejected": -3.5920569896698,
      "logps/chosen": -3.132289409637451,
      "logps/rejected": -36.69212341308594,
      "loss": 0.1902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.285799503326416,
      "rewards/margins": 2.285799503326416,
      "rewards/rejected": 0.0,
      "step": 1108
    },
    {
      "epoch": 6.195530726256983,
      "grad_norm": 3.341047557873241,
      "learning_rate": 3.846353490562664e-06,
      "logits/chosen": -3.219691514968872,
      "logits/rejected": -3.1501271724700928,
      "logps/chosen": -2.984177589416504,
      "logps/rejected": -42.03504180908203,
      "loss": 0.17,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.784132719039917,
      "rewards/margins": 1.784132719039917,
      "rewards/rejected": 0.0,
      "step": 1109
    },
    {
      "epoch": 6.201117318435754,
      "grad_norm": 2.0244592239662684,
      "learning_rate": 3.843784482399418e-06,
      "logits/chosen": -3.4761579036712646,
      "logits/rejected": -3.562474489212036,
      "logps/chosen": -14.205636978149414,
      "logps/rejected": -60.25324249267578,
      "loss": 0.1326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.6222076416015625,
      "rewards/margins": 4.6222076416015625,
      "rewards/rejected": 0.0,
      "step": 1110
    },
    {
      "epoch": 6.206703910614525,
      "grad_norm": 9.260415480482036,
      "learning_rate": 3.841213477133116e-06,
      "logits/chosen": -3.535754442214966,
      "logits/rejected": -3.6734631061553955,
      "logps/chosen": -11.96302318572998,
      "logps/rejected": -35.890830993652344,
      "loss": 0.166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4247822761535645,
      "rewards/margins": 2.4247822761535645,
      "rewards/rejected": 0.0,
      "step": 1111
    },
    {
      "epoch": 6.212290502793296,
      "grad_norm": 13.560125574064278,
      "learning_rate": 3.838640478584729e-06,
      "logits/chosen": -3.402355432510376,
      "logits/rejected": -3.4340977668762207,
      "logps/chosen": -14.996932983398438,
      "logps/rejected": -67.61185455322266,
      "loss": 0.233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.472057819366455,
      "rewards/margins": 1.472057819366455,
      "rewards/rejected": 0.0,
      "step": 1112
    },
    {
      "epoch": 6.217877094972067,
      "grad_norm": 4.40561921975046,
      "learning_rate": 3.8360654905781935e-06,
      "logits/chosen": -3.4055211544036865,
      "logits/rejected": -3.243532657623291,
      "logps/chosen": -3.4907681941986084,
      "logps/rejected": -50.4786491394043,
      "loss": 0.1396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.985630989074707,
      "rewards/margins": 1.985630989074707,
      "rewards/rejected": 0.0,
      "step": 1113
    },
    {
      "epoch": 6.223463687150838,
      "grad_norm": 2.8704754164248287,
      "learning_rate": 3.833488516940398e-06,
      "logits/chosen": -3.7237071990966797,
      "logits/rejected": -3.5950911045074463,
      "logps/chosen": -9.20042610168457,
      "logps/rejected": -34.447410583496094,
      "loss": 0.1087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7421188354492188,
      "rewards/margins": 2.7421188354492188,
      "rewards/rejected": 0.0,
      "step": 1114
    },
    {
      "epoch": 6.229050279329609,
      "grad_norm": 3.3594220448801075,
      "learning_rate": 3.830909561501184e-06,
      "logits/chosen": -3.314828634262085,
      "logits/rejected": -3.3348238468170166,
      "logps/chosen": -2.8928492069244385,
      "logps/rejected": -99.38795471191406,
      "loss": 0.1338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.296383857727051,
      "rewards/margins": 2.296383857727051,
      "rewards/rejected": 0.0,
      "step": 1115
    },
    {
      "epoch": 6.23463687150838,
      "grad_norm": 6.426712894466631,
      "learning_rate": 3.828328628093341e-06,
      "logits/chosen": -3.3324568271636963,
      "logits/rejected": -3.0970919132232666,
      "logps/chosen": -20.335155487060547,
      "logps/rejected": -55.525390625,
      "loss": 0.1442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3250410556793213,
      "rewards/margins": 3.3250410556793213,
      "rewards/rejected": 0.0,
      "step": 1116
    },
    {
      "epoch": 6.240223463687151,
      "grad_norm": 19.27147509436654,
      "learning_rate": 3.825745720552594e-06,
      "logits/chosen": -3.849287986755371,
      "logits/rejected": -3.9194164276123047,
      "logps/chosen": -3.9354805946350098,
      "logps/rejected": -42.28306198120117,
      "loss": 0.2369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0616750717163086,
      "rewards/margins": 2.0616750717163086,
      "rewards/rejected": 0.0,
      "step": 1117
    },
    {
      "epoch": 6.245810055865922,
      "grad_norm": 4.740993698033832,
      "learning_rate": 3.823160842717602e-06,
      "logits/chosen": -3.2873690128326416,
      "logits/rejected": -3.386643409729004,
      "logps/chosen": -2.4249305725097656,
      "logps/rejected": -35.960975646972656,
      "loss": 0.1852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5011444091796875,
      "rewards/margins": 2.5011444091796875,
      "rewards/rejected": 0.0,
      "step": 1118
    },
    {
      "epoch": 6.251396648044693,
      "grad_norm": 2.3719615371713383,
      "learning_rate": 3.820573998429958e-06,
      "logits/chosen": -3.4640400409698486,
      "logits/rejected": -3.476572275161743,
      "logps/chosen": -1.9098536968231201,
      "logps/rejected": -39.352294921875,
      "loss": 0.1456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5538809299468994,
      "rewards/margins": 1.5538809299468994,
      "rewards/rejected": 0.0,
      "step": 1119
    },
    {
      "epoch": 6.256983240223463,
      "grad_norm": 4.361221181068119,
      "learning_rate": 3.81798519153417e-06,
      "logits/chosen": -3.286963701248169,
      "logits/rejected": -3.338052272796631,
      "logps/chosen": -8.237045288085938,
      "logps/rejected": -36.56188201904297,
      "loss": 0.1463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3231234550476074,
      "rewards/margins": 1.3231234550476074,
      "rewards/rejected": 0.0,
      "step": 1120
    },
    {
      "epoch": 6.256983240223463,
      "eval_logits/chosen": -3.3387978076934814,
      "eval_logits/rejected": -3.42364239692688,
      "eval_logps/chosen": -32.5814208984375,
      "eval_logps/rejected": -47.5413818359375,
      "eval_loss": 1.031313180923462,
      "eval_rewards/accuracies": 0.42500001192092896,
      "eval_rewards/chosen": -0.2511380612850189,
      "eval_rewards/margins": -0.2511380612850189,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5909,
      "eval_samples_per_second": 12.114,
      "eval_steps_per_second": 0.391,
      "step": 1120
    },
    {
      "epoch": 6.262569832402234,
      "grad_norm": 3.8511749675586135,
      "learning_rate": 3.815394425877668e-06,
      "logits/chosen": -3.386605978012085,
      "logits/rejected": -3.371230363845825,
      "logps/chosen": -4.60771369934082,
      "logps/rejected": -38.07048034667969,
      "loss": 0.1313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0355145931243896,
      "rewards/margins": 2.0355145931243896,
      "rewards/rejected": 0.0,
      "step": 1121
    },
    {
      "epoch": 6.268156424581005,
      "grad_norm": 2.7577920369840516,
      "learning_rate": 3.812801705310789e-06,
      "logits/chosen": -3.271294116973877,
      "logits/rejected": -3.271610975265503,
      "logps/chosen": -8.853504180908203,
      "logps/rejected": -42.50678253173828,
      "loss": 0.0948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0448246002197266,
      "rewards/margins": 2.0448246002197266,
      "rewards/rejected": 0.0,
      "step": 1122
    },
    {
      "epoch": 6.273743016759776,
      "grad_norm": 3.29156655324528,
      "learning_rate": 3.810207033686779e-06,
      "logits/chosen": -3.2859952449798584,
      "logits/rejected": -3.382519245147705,
      "logps/chosen": -10.580485343933105,
      "logps/rejected": -62.671993255615234,
      "loss": 0.131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.704019069671631,
      "rewards/margins": 2.704019069671631,
      "rewards/rejected": 0.0,
      "step": 1123
    },
    {
      "epoch": 6.279329608938547,
      "grad_norm": 2.9559272409274446,
      "learning_rate": 3.807610414861782e-06,
      "logits/chosen": -3.420053482055664,
      "logits/rejected": -2.9610395431518555,
      "logps/chosen": -1.819044828414917,
      "logps/rejected": -85.26407623291016,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6277222633361816,
      "rewards/margins": 1.6277222633361816,
      "rewards/rejected": 0.0,
      "step": 1124
    },
    {
      "epoch": 6.284916201117318,
      "grad_norm": 4.81878041529971,
      "learning_rate": 3.8050118526948366e-06,
      "logits/chosen": -3.605724573135376,
      "logits/rejected": -3.6205883026123047,
      "logps/chosen": -3.859933614730835,
      "logps/rejected": -44.35682678222656,
      "loss": 0.1287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.205167293548584,
      "rewards/margins": 2.205167293548584,
      "rewards/rejected": 0.0,
      "step": 1125
    },
    {
      "epoch": 6.290502793296089,
      "grad_norm": 1.7722366805942908,
      "learning_rate": 3.802411351047867e-06,
      "logits/chosen": -3.5238118171691895,
      "logits/rejected": -3.618359327316284,
      "logps/chosen": -7.339072227478027,
      "logps/rejected": -57.9815673828125,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5378689765930176,
      "rewards/margins": 2.5378689765930176,
      "rewards/rejected": 0.0,
      "step": 1126
    },
    {
      "epoch": 6.29608938547486,
      "grad_norm": 3.2570169887006126,
      "learning_rate": 3.7998089137856837e-06,
      "logits/chosen": -3.1298975944519043,
      "logits/rejected": -3.17403507232666,
      "logps/chosen": -5.119269847869873,
      "logps/rejected": -42.54350280761719,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.345257520675659,
      "rewards/margins": 2.345257520675659,
      "rewards/rejected": 0.0,
      "step": 1127
    },
    {
      "epoch": 6.301675977653631,
      "grad_norm": 5.101683436363153,
      "learning_rate": 3.797204544775971e-06,
      "logits/chosen": -3.417712688446045,
      "logits/rejected": -3.492089033126831,
      "logps/chosen": -3.254772663116455,
      "logps/rejected": -37.03923797607422,
      "loss": 0.1489,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.45308256149292,
      "rewards/margins": 2.45308256149292,
      "rewards/rejected": 0.0,
      "step": 1128
    },
    {
      "epoch": 6.307262569832402,
      "grad_norm": 13.418778152149928,
      "learning_rate": 3.7945982478892852e-06,
      "logits/chosen": -3.2679171562194824,
      "logits/rejected": -3.2899487018585205,
      "logps/chosen": -9.121322631835938,
      "logps/rejected": -68.7931900024414,
      "loss": 0.2086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.759267807006836,
      "rewards/margins": 2.759267807006836,
      "rewards/rejected": 0.0,
      "step": 1129
    },
    {
      "epoch": 6.312849162011173,
      "grad_norm": 4.468650688053343,
      "learning_rate": 3.7919900269990473e-06,
      "logits/chosen": -3.585369348526001,
      "logits/rejected": -3.387390375137329,
      "logps/chosen": -7.195544719696045,
      "logps/rejected": -60.2138671875,
      "loss": 0.1452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.081402540206909,
      "rewards/margins": 3.081402540206909,
      "rewards/rejected": 0.0,
      "step": 1130
    },
    {
      "epoch": 6.318435754189944,
      "grad_norm": 6.071139758028256,
      "learning_rate": 3.789379885981539e-06,
      "logits/chosen": -3.5556752681732178,
      "logits/rejected": -3.4618611335754395,
      "logps/chosen": -24.996349334716797,
      "logps/rejected": -39.31341552734375,
      "loss": 0.1805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9355533123016357,
      "rewards/margins": 3.9355533123016357,
      "rewards/rejected": 0.0,
      "step": 1131
    },
    {
      "epoch": 6.324022346368715,
      "grad_norm": 3.056905090235902,
      "learning_rate": 3.7867678287158937e-06,
      "logits/chosen": -3.6165666580200195,
      "logits/rejected": -3.503013849258423,
      "logps/chosen": -6.770464897155762,
      "logps/rejected": -58.67135238647461,
      "loss": 0.1364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.775120258331299,
      "rewards/margins": 2.775120258331299,
      "rewards/rejected": 0.0,
      "step": 1132
    },
    {
      "epoch": 6.329608938547486,
      "grad_norm": 3.4569638365614574,
      "learning_rate": 3.784153859084095e-06,
      "logits/chosen": -3.3558847904205322,
      "logits/rejected": -3.539433002471924,
      "logps/chosen": -3.5147719383239746,
      "logps/rejected": -67.03024291992188,
      "loss": 0.1535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.177488088607788,
      "rewards/margins": 2.177488088607788,
      "rewards/rejected": 0.0,
      "step": 1133
    },
    {
      "epoch": 6.335195530726257,
      "grad_norm": 2.772960541744316,
      "learning_rate": 3.781537980970965e-06,
      "logits/chosen": -3.5715179443359375,
      "logits/rejected": -3.5270485877990723,
      "logps/chosen": -12.589774131774902,
      "logps/rejected": -58.378517150878906,
      "loss": 0.1105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.032893180847168,
      "rewards/margins": 3.032893180847168,
      "rewards/rejected": 0.0,
      "step": 1134
    },
    {
      "epoch": 6.340782122905028,
      "grad_norm": 4.965546953433802,
      "learning_rate": 3.7789201982641673e-06,
      "logits/chosen": -3.2005817890167236,
      "logits/rejected": -3.0245237350463867,
      "logps/chosen": -4.275275230407715,
      "logps/rejected": -29.941753387451172,
      "loss": 0.115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0397372245788574,
      "rewards/margins": 3.0397372245788574,
      "rewards/rejected": 0.0,
      "step": 1135
    },
    {
      "epoch": 6.346368715083799,
      "grad_norm": 1.3807100041515357,
      "learning_rate": 3.776300514854191e-06,
      "logits/chosen": -3.8738651275634766,
      "logits/rejected": -3.6950457096099854,
      "logps/chosen": -8.990592956542969,
      "logps/rejected": -42.57445526123047,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.31115460395813,
      "rewards/margins": 3.31115460395813,
      "rewards/rejected": 0.0,
      "step": 1136
    },
    {
      "epoch": 6.351955307262569,
      "grad_norm": 2.6175648443853254,
      "learning_rate": 3.7736789346343527e-06,
      "logits/chosen": -3.61669659614563,
      "logits/rejected": -3.736576557159424,
      "logps/chosen": -3.232555389404297,
      "logps/rejected": -59.962669372558594,
      "loss": 0.1228,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2469005584716797,
      "rewards/margins": 2.2469005584716797,
      "rewards/rejected": 0.0,
      "step": 1137
    },
    {
      "epoch": 6.35754189944134,
      "grad_norm": 4.796262926709334,
      "learning_rate": 3.771055461500788e-06,
      "logits/chosen": -3.505089282989502,
      "logits/rejected": -3.7549476623535156,
      "logps/chosen": -5.527320384979248,
      "logps/rejected": -55.1052360534668,
      "loss": 0.1544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1100356578826904,
      "rewards/margins": 2.1100356578826904,
      "rewards/rejected": 0.0,
      "step": 1138
    },
    {
      "epoch": 6.363128491620111,
      "grad_norm": 4.543372025405231,
      "learning_rate": 3.768430099352445e-06,
      "logits/chosen": -3.374507427215576,
      "logits/rejected": -3.2958126068115234,
      "logps/chosen": -7.714788436889648,
      "logps/rejected": -36.068058013916016,
      "loss": 0.1449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4415814876556396,
      "rewards/margins": 3.4415814876556396,
      "rewards/rejected": 0.0,
      "step": 1139
    },
    {
      "epoch": 6.368715083798882,
      "grad_norm": 14.472459130574764,
      "learning_rate": 3.7658028520910795e-06,
      "logits/chosen": -3.4908268451690674,
      "logits/rejected": -3.4177329540252686,
      "logps/chosen": -8.596047401428223,
      "logps/rejected": -63.16864013671875,
      "loss": 0.2254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2334482669830322,
      "rewards/margins": 1.2334482669830322,
      "rewards/rejected": 0.0,
      "step": 1140
    },
    {
      "epoch": 6.368715083798882,
      "eval_logits/chosen": -3.3538403511047363,
      "eval_logits/rejected": -3.4476256370544434,
      "eval_logps/chosen": -33.392578125,
      "eval_logps/rejected": -50.405235290527344,
      "eval_loss": 1.1407933235168457,
      "eval_rewards/accuracies": 0.4749999940395355,
      "eval_rewards/chosen": -0.332253634929657,
      "eval_rewards/margins": -0.332253634929657,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6371,
      "eval_samples_per_second": 12.092,
      "eval_steps_per_second": 0.39,
      "step": 1140
    },
    {
      "epoch": 6.374301675977653,
      "grad_norm": 5.235801693813544,
      "learning_rate": 3.7631737236212486e-06,
      "logits/chosen": -3.4019289016723633,
      "logits/rejected": -3.3608782291412354,
      "logps/chosen": -19.110515594482422,
      "logps/rejected": -61.68394088745117,
      "loss": 0.1727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.33971905708313,
      "rewards/margins": 3.33971905708313,
      "rewards/rejected": 0.0,
      "step": 1141
    },
    {
      "epoch": 6.379888268156424,
      "grad_norm": 5.9680200295550385,
      "learning_rate": 3.760542717850305e-06,
      "logits/chosen": -3.3325440883636475,
      "logits/rejected": -3.456383466720581,
      "logps/chosen": -4.079148769378662,
      "logps/rejected": -47.38029861450195,
      "loss": 0.1478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.252251148223877,
      "rewards/margins": 2.252251148223877,
      "rewards/rejected": 0.0,
      "step": 1142
    },
    {
      "epoch": 6.385474860335195,
      "grad_norm": 2.8804353619830314,
      "learning_rate": 3.7579098386883934e-06,
      "logits/chosen": -3.4619967937469482,
      "logits/rejected": -3.4808268547058105,
      "logps/chosen": -10.057774543762207,
      "logps/rejected": -56.40949249267578,
      "loss": 0.1305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.297025442123413,
      "rewards/margins": 2.297025442123413,
      "rewards/rejected": 0.0,
      "step": 1143
    },
    {
      "epoch": 6.391061452513966,
      "grad_norm": 7.360879560415451,
      "learning_rate": 3.7552750900484388e-06,
      "logits/chosen": -3.6145806312561035,
      "logits/rejected": -3.547624349594116,
      "logps/chosen": -8.118948936462402,
      "logps/rejected": -23.1762638092041,
      "loss": 0.1714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7282023429870605,
      "rewards/margins": 2.7282023429870605,
      "rewards/rejected": 0.0,
      "step": 1144
    },
    {
      "epoch": 6.396648044692737,
      "grad_norm": 1.3655932647452427,
      "learning_rate": 3.752638475846149e-06,
      "logits/chosen": -3.674445867538452,
      "logits/rejected": -3.660644054412842,
      "logps/chosen": -2.5895116329193115,
      "logps/rejected": -69.2605209350586,
      "loss": 0.0926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7081055641174316,
      "rewards/margins": 2.7081055641174316,
      "rewards/rejected": 0.0,
      "step": 1145
    },
    {
      "epoch": 6.402234636871508,
      "grad_norm": 25.10512045112549,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": -3.525411367416382,
      "logits/rejected": -3.5484988689422607,
      "logps/chosen": -8.9798583984375,
      "logps/rejected": -46.05303192138672,
      "loss": 0.2574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0807313919067383,
      "rewards/margins": 3.0807313919067383,
      "rewards/rejected": 0.0,
      "step": 1146
    },
    {
      "epoch": 6.407821229050279,
      "grad_norm": 3.4099414241919797,
      "learning_rate": 3.7473596664312396e-06,
      "logits/chosen": -3.6171681880950928,
      "logits/rejected": -3.1184165477752686,
      "logps/chosen": -4.375274658203125,
      "logps/rejected": -33.494239807128906,
      "loss": 0.1431,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5502598285675049,
      "rewards/margins": 1.5502598285675049,
      "rewards/rejected": 0.0,
      "step": 1147
    },
    {
      "epoch": 6.41340782122905,
      "grad_norm": 4.722617006486619,
      "learning_rate": 3.7447174790638717e-06,
      "logits/chosen": -3.561828851699829,
      "logits/rejected": -3.358821392059326,
      "logps/chosen": -10.872705459594727,
      "logps/rejected": -51.69924545288086,
      "loss": 0.1442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5096828937530518,
      "rewards/margins": 2.5096828937530518,
      "rewards/rejected": 0.0,
      "step": 1148
    },
    {
      "epoch": 6.418994413407821,
      "grad_norm": 2.4040782678177286,
      "learning_rate": 3.742073441824658e-06,
      "logits/chosen": -3.6436662673950195,
      "logits/rejected": -3.6746745109558105,
      "logps/chosen": -5.478456020355225,
      "logps/rejected": -46.30843734741211,
      "loss": 0.1179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.039088249206543,
      "rewards/margins": 2.039088249206543,
      "rewards/rejected": 0.0,
      "step": 1149
    },
    {
      "epoch": 6.424581005586592,
      "grad_norm": 2.6037466475715987,
      "learning_rate": 3.7394275586431084e-06,
      "logits/chosen": -3.453171730041504,
      "logits/rejected": -3.5956366062164307,
      "logps/chosen": -19.292266845703125,
      "logps/rejected": -49.14739227294922,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.921170711517334,
      "rewards/margins": 2.921170711517334,
      "rewards/rejected": 0.0,
      "step": 1150
    },
    {
      "epoch": 6.430167597765363,
      "grad_norm": 3.4104063706963745,
      "learning_rate": 3.736779833451477e-06,
      "logits/chosen": -3.5637028217315674,
      "logits/rejected": -3.479008674621582,
      "logps/chosen": -2.964475631713867,
      "logps/rejected": -36.891143798828125,
      "loss": 0.1419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4365653991699219,
      "rewards/margins": 1.4365653991699219,
      "rewards/rejected": 0.0,
      "step": 1151
    },
    {
      "epoch": 6.435754189944134,
      "grad_norm": 2.282462108886095,
      "learning_rate": 3.7341302701847547e-06,
      "logits/chosen": -3.5719363689422607,
      "logits/rejected": -3.5451507568359375,
      "logps/chosen": -5.995074272155762,
      "logps/rejected": -54.29265594482422,
      "loss": 0.1151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7093148231506348,
      "rewards/margins": 2.7093148231506348,
      "rewards/rejected": 0.0,
      "step": 1152
    },
    {
      "epoch": 6.441340782122905,
      "grad_norm": 3.429481417533056,
      "learning_rate": 3.731478872780664e-06,
      "logits/chosen": -3.52285099029541,
      "logits/rejected": -3.64471435546875,
      "logps/chosen": -12.084903717041016,
      "logps/rejected": -56.896148681640625,
      "loss": 0.1356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.590639114379883,
      "rewards/margins": 3.590639114379883,
      "rewards/rejected": 0.0,
      "step": 1153
    },
    {
      "epoch": 6.446927374301676,
      "grad_norm": 2.5884050765431446,
      "learning_rate": 3.7288256451796535e-06,
      "logits/chosen": -3.4532368183135986,
      "logits/rejected": -3.5982980728149414,
      "logps/chosen": -26.86318588256836,
      "logps/rejected": -86.05703735351562,
      "loss": 0.1363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7180776596069336,
      "rewards/margins": 2.7180776596069336,
      "rewards/rejected": 0.0,
      "step": 1154
    },
    {
      "epoch": 6.452513966480447,
      "grad_norm": 1.7258821622565352,
      "learning_rate": 3.7261705913248936e-06,
      "logits/chosen": -3.5971150398254395,
      "logits/rejected": -3.643291473388672,
      "logps/chosen": -8.331355094909668,
      "logps/rejected": -44.22911071777344,
      "loss": 0.1082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5777158737182617,
      "rewards/margins": 2.5777158737182617,
      "rewards/rejected": 0.0,
      "step": 1155
    },
    {
      "epoch": 6.4581005586592175,
      "grad_norm": 4.795913966608766,
      "learning_rate": 3.723513715162265e-06,
      "logits/chosen": -3.2973248958587646,
      "logits/rejected": -3.353360652923584,
      "logps/chosen": -12.106546401977539,
      "logps/rejected": -42.90076446533203,
      "loss": 0.1976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.1821868419647217,
      "rewards/margins": 2.1821868419647217,
      "rewards/rejected": 0.0,
      "step": 1156
    },
    {
      "epoch": 6.4636871508379885,
      "grad_norm": 5.522183857805778,
      "learning_rate": 3.72085502064036e-06,
      "logits/chosen": -3.5433123111724854,
      "logits/rejected": -3.5324819087982178,
      "logps/chosen": -4.075252532958984,
      "logps/rejected": -87.1331787109375,
      "loss": 0.166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0683345794677734,
      "rewards/margins": 2.0683345794677734,
      "rewards/rejected": 0.0,
      "step": 1157
    },
    {
      "epoch": 6.4692737430167595,
      "grad_norm": 4.272286596329329,
      "learning_rate": 3.7181945117104717e-06,
      "logits/chosen": -3.7148938179016113,
      "logits/rejected": -3.652362108230591,
      "logps/chosen": -5.197190761566162,
      "logps/rejected": -51.90229797363281,
      "loss": 0.1437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2557425498962402,
      "rewards/margins": 2.2557425498962402,
      "rewards/rejected": 0.0,
      "step": 1158
    },
    {
      "epoch": 6.4748603351955305,
      "grad_norm": 3.9026672518529235,
      "learning_rate": 3.7155321923265906e-06,
      "logits/chosen": -3.3422443866729736,
      "logits/rejected": -3.524301767349243,
      "logps/chosen": -2.770411252975464,
      "logps/rejected": -67.08094024658203,
      "loss": 0.152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2075414657592773,
      "rewards/margins": 2.2075414657592773,
      "rewards/rejected": 0.0,
      "step": 1159
    },
    {
      "epoch": 6.4804469273743015,
      "grad_norm": 2.500586038686997,
      "learning_rate": 3.7128680664453975e-06,
      "logits/chosen": -3.4098334312438965,
      "logits/rejected": -3.4159913063049316,
      "logps/chosen": -12.889605522155762,
      "logps/rejected": -43.387081146240234,
      "loss": 0.1134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7371985912323,
      "rewards/margins": 2.7371985912323,
      "rewards/rejected": 0.0,
      "step": 1160
    },
    {
      "epoch": 6.4804469273743015,
      "eval_logits/chosen": -3.340679883956909,
      "eval_logits/rejected": -3.4341518878936768,
      "eval_logps/chosen": -32.737239837646484,
      "eval_logps/rejected": -49.40897750854492,
      "eval_loss": 1.1404693126678467,
      "eval_rewards/accuracies": 0.44999998807907104,
      "eval_rewards/chosen": -0.2667195796966553,
      "eval_rewards/margins": -0.2667195796966553,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5696,
      "eval_samples_per_second": 12.124,
      "eval_steps_per_second": 0.391,
      "step": 1160
    },
    {
      "epoch": 6.4860335195530725,
      "grad_norm": 1.8926920573837396,
      "learning_rate": 3.710202138026258e-06,
      "logits/chosen": -3.375234365463257,
      "logits/rejected": -3.4848484992980957,
      "logps/chosen": -3.5468711853027344,
      "logps/rejected": -47.3599967956543,
      "loss": 0.1197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.242820978164673,
      "rewards/margins": 2.242820978164673,
      "rewards/rejected": 0.0,
      "step": 1161
    },
    {
      "epoch": 6.4916201117318435,
      "grad_norm": 2.876984606129492,
      "learning_rate": 3.707534411031216e-06,
      "logits/chosen": -3.6275417804718018,
      "logits/rejected": -3.7729673385620117,
      "logps/chosen": -4.222532272338867,
      "logps/rejected": -58.34333038330078,
      "loss": 0.1233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1685948371887207,
      "rewards/margins": 2.1685948371887207,
      "rewards/rejected": 0.0,
      "step": 1162
    },
    {
      "epoch": 6.4972067039106145,
      "grad_norm": 4.342058377173038,
      "learning_rate": 3.7048648894249885e-06,
      "logits/chosen": -3.431657552719116,
      "logits/rejected": -3.4101974964141846,
      "logps/chosen": -4.504586219787598,
      "logps/rejected": -67.0958251953125,
      "loss": 0.1432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.580522298812866,
      "rewards/margins": 2.580522298812866,
      "rewards/rejected": 0.0,
      "step": 1163
    },
    {
      "epoch": 6.5027932960893855,
      "grad_norm": 2.3210990073228364,
      "learning_rate": 3.7021935771749608e-06,
      "logits/chosen": -3.8184456825256348,
      "logits/rejected": -3.7442638874053955,
      "logps/chosen": -1.3151925802230835,
      "logps/rejected": -46.21575164794922,
      "loss": 0.1254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.929518222808838,
      "rewards/margins": 1.929518222808838,
      "rewards/rejected": 0.0,
      "step": 1164
    },
    {
      "epoch": 6.5083798882681565,
      "grad_norm": 5.873422543189554,
      "learning_rate": 3.699520478251179e-06,
      "logits/chosen": -3.4809532165527344,
      "logits/rejected": -3.436972141265869,
      "logps/chosen": -4.776566505432129,
      "logps/rejected": -57.58720397949219,
      "loss": 0.1502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3925862312316895,
      "rewards/margins": 2.3925862312316895,
      "rewards/rejected": 0.0,
      "step": 1165
    },
    {
      "epoch": 6.5139664804469275,
      "grad_norm": 2.7050990432830337,
      "learning_rate": 3.696845596626343e-06,
      "logits/chosen": -3.4724812507629395,
      "logits/rejected": -3.3344123363494873,
      "logps/chosen": -8.291488647460938,
      "logps/rejected": -36.59842300415039,
      "loss": 0.1529,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7942118644714355,
      "rewards/margins": 2.7942118644714355,
      "rewards/rejected": 0.0,
      "step": 1166
    },
    {
      "epoch": 6.5195530726256985,
      "grad_norm": 7.25174177389504,
      "learning_rate": 3.6941689362758038e-06,
      "logits/chosen": -3.5297858715057373,
      "logits/rejected": -3.3139305114746094,
      "logps/chosen": -23.292606353759766,
      "logps/rejected": -34.276458740234375,
      "loss": 0.2002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.526442050933838,
      "rewards/margins": 3.526442050933838,
      "rewards/rejected": 0.0,
      "step": 1167
    },
    {
      "epoch": 6.5251396648044695,
      "grad_norm": 3.7744271079851686,
      "learning_rate": 3.6914905011775558e-06,
      "logits/chosen": -3.4458601474761963,
      "logits/rejected": -3.527973175048828,
      "logps/chosen": -18.385818481445312,
      "logps/rejected": -68.8880386352539,
      "loss": 0.1451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7839341163635254,
      "rewards/margins": 3.7839341163635254,
      "rewards/rejected": 0.0,
      "step": 1168
    },
    {
      "epoch": 6.5307262569832405,
      "grad_norm": 1.9591027678821236,
      "learning_rate": 3.6888102953122307e-06,
      "logits/chosen": -3.4721152782440186,
      "logits/rejected": -3.3814167976379395,
      "logps/chosen": -12.143474578857422,
      "logps/rejected": -48.143959045410156,
      "loss": 0.107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.543386459350586,
      "rewards/margins": 3.543386459350586,
      "rewards/rejected": 0.0,
      "step": 1169
    },
    {
      "epoch": 6.5363128491620115,
      "grad_norm": 5.010393832700878,
      "learning_rate": 3.68612832266309e-06,
      "logits/chosen": -3.714515447616577,
      "logits/rejected": -3.27245831489563,
      "logps/chosen": -2.4237256050109863,
      "logps/rejected": -64.12448120117188,
      "loss": 0.16,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8898448944091797,
      "rewards/margins": 1.8898448944091797,
      "rewards/rejected": 0.0,
      "step": 1170
    },
    {
      "epoch": 6.5418994413407825,
      "grad_norm": 2.390933554638665,
      "learning_rate": 3.683444587216025e-06,
      "logits/chosen": -3.0771172046661377,
      "logits/rejected": -3.1032464504241943,
      "logps/chosen": -13.83850383758545,
      "logps/rejected": -42.16681671142578,
      "loss": 0.0966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2679176330566406,
      "rewards/margins": 3.2679176330566406,
      "rewards/rejected": 0.0,
      "step": 1171
    },
    {
      "epoch": 6.547486033519553,
      "grad_norm": 5.026399863177568,
      "learning_rate": 3.6807590929595433e-06,
      "logits/chosen": -3.5369913578033447,
      "logits/rejected": -3.311669111251831,
      "logps/chosen": -5.550508499145508,
      "logps/rejected": -42.64193344116211,
      "loss": 0.1575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.074368476867676,
      "rewards/margins": 2.074368476867676,
      "rewards/rejected": 0.0,
      "step": 1172
    },
    {
      "epoch": 6.553072625698324,
      "grad_norm": 2.1858168388118386,
      "learning_rate": 3.6780718438847674e-06,
      "logits/chosen": -3.415032148361206,
      "logits/rejected": -3.323038339614868,
      "logps/chosen": -14.650259971618652,
      "logps/rejected": -66.83538818359375,
      "loss": 0.1064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7769181728363037,
      "rewards/margins": 2.7769181728363037,
      "rewards/rejected": 0.0,
      "step": 1173
    },
    {
      "epoch": 6.558659217877095,
      "grad_norm": 1.7596404546333546,
      "learning_rate": 3.6753828439854285e-06,
      "logits/chosen": -3.48465633392334,
      "logits/rejected": -3.4619460105895996,
      "logps/chosen": -1.6297099590301514,
      "logps/rejected": -46.87957000732422,
      "loss": 0.1123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9911344051361084,
      "rewards/margins": 1.9911344051361084,
      "rewards/rejected": 0.0,
      "step": 1174
    },
    {
      "epoch": 6.564245810055866,
      "grad_norm": 2.374583365805269,
      "learning_rate": 3.6726920972578593e-06,
      "logits/chosen": -3.589745044708252,
      "logits/rejected": -3.694348096847534,
      "logps/chosen": -3.2722935676574707,
      "logps/rejected": -53.31220626831055,
      "loss": 0.0976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.376155376434326,
      "rewards/margins": 2.376155376434326,
      "rewards/rejected": 0.0,
      "step": 1175
    },
    {
      "epoch": 6.569832402234637,
      "grad_norm": 3.0619016237568393,
      "learning_rate": 3.669999607700988e-06,
      "logits/chosen": -3.4073376655578613,
      "logits/rejected": -3.477745771408081,
      "logps/chosen": -6.835681915283203,
      "logps/rejected": -58.6749267578125,
      "loss": 0.1188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7319345474243164,
      "rewards/margins": 2.7319345474243164,
      "rewards/rejected": 0.0,
      "step": 1176
    },
    {
      "epoch": 6.575418994413408,
      "grad_norm": 2.72382341637731,
      "learning_rate": 3.6673053793163348e-06,
      "logits/chosen": -3.692430019378662,
      "logits/rejected": -3.7897369861602783,
      "logps/chosen": -22.361116409301758,
      "logps/rejected": -65.92049407958984,
      "loss": 0.1006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.860748767852783,
      "rewards/margins": 3.860748767852783,
      "rewards/rejected": 0.0,
      "step": 1177
    },
    {
      "epoch": 6.581005586592179,
      "grad_norm": 3.660248707648256,
      "learning_rate": 3.664609416108002e-06,
      "logits/chosen": -3.4990363121032715,
      "logits/rejected": -3.661099672317505,
      "logps/chosen": -5.820465087890625,
      "logps/rejected": -59.443050384521484,
      "loss": 0.1258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2899580001831055,
      "rewards/margins": 2.2899580001831055,
      "rewards/rejected": 0.0,
      "step": 1178
    },
    {
      "epoch": 6.58659217877095,
      "grad_norm": 3.38267940340457,
      "learning_rate": 3.6619117220826704e-06,
      "logits/chosen": -3.3616254329681396,
      "logits/rejected": -3.365900993347168,
      "logps/chosen": -5.127434730529785,
      "logps/rejected": -73.11082458496094,
      "loss": 0.1266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8964943885803223,
      "rewards/margins": 2.8964943885803223,
      "rewards/rejected": 0.0,
      "step": 1179
    },
    {
      "epoch": 6.592178770949721,
      "grad_norm": 3.262193649111385,
      "learning_rate": 3.659212301249596e-06,
      "logits/chosen": -3.5624570846557617,
      "logits/rejected": -3.4759604930877686,
      "logps/chosen": -8.148561477661133,
      "logps/rejected": -59.54766082763672,
      "loss": 0.1196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8140616416931152,
      "rewards/margins": 2.8140616416931152,
      "rewards/rejected": 0.0,
      "step": 1180
    },
    {
      "epoch": 6.592178770949721,
      "eval_logits/chosen": -3.406801700592041,
      "eval_logits/rejected": -3.5115649700164795,
      "eval_logps/chosen": -34.01567840576172,
      "eval_logps/rejected": -53.983421325683594,
      "eval_loss": 1.2593226432800293,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.394563764333725,
      "eval_rewards/margins": -0.394563764333725,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5477,
      "eval_samples_per_second": 12.134,
      "eval_steps_per_second": 0.391,
      "step": 1180
    },
    {
      "epoch": 6.597765363128492,
      "grad_norm": 4.1174277817600675,
      "learning_rate": 3.6565111576205975e-06,
      "logits/chosen": -3.7059664726257324,
      "logits/rejected": -3.7746970653533936,
      "logps/chosen": -14.042475700378418,
      "logps/rejected": -49.44803237915039,
      "loss": 0.1072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7122766971588135,
      "rewards/margins": 2.7122766971588135,
      "rewards/rejected": 0.0,
      "step": 1181
    },
    {
      "epoch": 6.603351955307263,
      "grad_norm": 3.736733009864841,
      "learning_rate": 3.653808295210055e-06,
      "logits/chosen": -3.404310464859009,
      "logits/rejected": -3.6486356258392334,
      "logps/chosen": -2.9585094451904297,
      "logps/rejected": -53.34756851196289,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7569546699523926,
      "rewards/margins": 2.7569546699523926,
      "rewards/rejected": 0.0,
      "step": 1182
    },
    {
      "epoch": 6.608938547486034,
      "grad_norm": 3.5359142521395723,
      "learning_rate": 3.6511037180349035e-06,
      "logits/chosen": -3.8098714351654053,
      "logits/rejected": -3.9320313930511475,
      "logps/chosen": -3.074950695037842,
      "logps/rejected": -98.469970703125,
      "loss": 0.1256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.918553113937378,
      "rewards/margins": 1.918553113937378,
      "rewards/rejected": 0.0,
      "step": 1183
    },
    {
      "epoch": 6.614525139664805,
      "grad_norm": 10.231298298560844,
      "learning_rate": 3.6483974301146267e-06,
      "logits/chosen": -3.464539051055908,
      "logits/rejected": -3.3334813117980957,
      "logps/chosen": -3.9566736221313477,
      "logps/rejected": -46.82851791381836,
      "loss": 0.2061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.43855881690979,
      "rewards/margins": 2.43855881690979,
      "rewards/rejected": 0.0,
      "step": 1184
    },
    {
      "epoch": 6.620111731843576,
      "grad_norm": 2.5544439168206314,
      "learning_rate": 3.6456894354712506e-06,
      "logits/chosen": -3.7284889221191406,
      "logits/rejected": -3.522240400314331,
      "logps/chosen": -5.090880393981934,
      "logps/rejected": -55.86814880371094,
      "loss": 0.116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7483129501342773,
      "rewards/margins": 2.7483129501342773,
      "rewards/rejected": 0.0,
      "step": 1185
    },
    {
      "epoch": 6.625698324022347,
      "grad_norm": 2.4798257658429286,
      "learning_rate": 3.642979738129337e-06,
      "logits/chosen": -3.7306487560272217,
      "logits/rejected": -3.599515438079834,
      "logps/chosen": -8.06637191772461,
      "logps/rejected": -35.64269256591797,
      "loss": 0.0917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5447075366973877,
      "rewards/margins": 3.5447075366973877,
      "rewards/rejected": 0.0,
      "step": 1186
    },
    {
      "epoch": 6.631284916201118,
      "grad_norm": 6.308419749809192,
      "learning_rate": 3.640268342115979e-06,
      "logits/chosen": -3.4670910835266113,
      "logits/rejected": -3.337085485458374,
      "logps/chosen": -3.8201518058776855,
      "logps/rejected": -38.832149505615234,
      "loss": 0.1742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2409934997558594,
      "rewards/margins": 2.2409934997558594,
      "rewards/rejected": 0.0,
      "step": 1187
    },
    {
      "epoch": 6.636871508379889,
      "grad_norm": 4.497263842230765,
      "learning_rate": 3.637555251460794e-06,
      "logits/chosen": -3.5429811477661133,
      "logits/rejected": -3.6211752891540527,
      "logps/chosen": -1.3998100757598877,
      "logps/rejected": -54.94021224975586,
      "loss": 0.1241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.947134256362915,
      "rewards/margins": 1.947134256362915,
      "rewards/rejected": 0.0,
      "step": 1188
    },
    {
      "epoch": 6.64245810055866,
      "grad_norm": 3.6400242914877725,
      "learning_rate": 3.634840470195918e-06,
      "logits/chosen": -3.4005041122436523,
      "logits/rejected": -3.5011863708496094,
      "logps/chosen": -9.133241653442383,
      "logps/rejected": -40.90557098388672,
      "loss": 0.1196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7320034503936768,
      "rewards/margins": 2.7320034503936768,
      "rewards/rejected": 0.0,
      "step": 1189
    },
    {
      "epoch": 6.648044692737431,
      "grad_norm": 4.232607993397541,
      "learning_rate": 3.632124002355999e-06,
      "logits/chosen": -3.5736243724823,
      "logits/rejected": -3.455132246017456,
      "logps/chosen": -13.166725158691406,
      "logps/rejected": -65.958984375,
      "loss": 0.1447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.1939303874969482,
      "rewards/margins": 2.1939303874969482,
      "rewards/rejected": 0.0,
      "step": 1190
    },
    {
      "epoch": 6.653631284916202,
      "grad_norm": 2.1431230287111354,
      "learning_rate": 3.6294058519781927e-06,
      "logits/chosen": -3.1136457920074463,
      "logits/rejected": -3.0513079166412354,
      "logps/chosen": -6.02688455581665,
      "logps/rejected": -50.974937438964844,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.003389358520508,
      "rewards/margins": 3.003389358520508,
      "rewards/rejected": 0.0,
      "step": 1191
    },
    {
      "epoch": 6.659217877094972,
      "grad_norm": 3.7456483341087408,
      "learning_rate": 3.626686023102154e-06,
      "logits/chosen": -3.5788280963897705,
      "logits/rejected": -3.6021556854248047,
      "logps/chosen": -1.8645957708358765,
      "logps/rejected": -24.253360748291016,
      "loss": 0.1603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1817772388458252,
      "rewards/margins": 1.1817772388458252,
      "rewards/rejected": 0.0,
      "step": 1192
    },
    {
      "epoch": 6.664804469273743,
      "grad_norm": 1.8798902558776691,
      "learning_rate": 3.623964519770033e-06,
      "logits/chosen": -3.5519866943359375,
      "logits/rejected": -3.466799259185791,
      "logps/chosen": -7.675619125366211,
      "logps/rejected": -46.995094299316406,
      "loss": 0.1087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8218088150024414,
      "rewards/margins": 2.8218088150024414,
      "rewards/rejected": 0.0,
      "step": 1193
    },
    {
      "epoch": 6.670391061452514,
      "grad_norm": 3.042814570511875,
      "learning_rate": 3.6212413460264685e-06,
      "logits/chosen": -3.2026872634887695,
      "logits/rejected": -3.116597890853882,
      "logps/chosen": -5.730603218078613,
      "logps/rejected": -60.62698745727539,
      "loss": 0.0892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0394463539123535,
      "rewards/margins": 3.0394463539123535,
      "rewards/rejected": 0.0,
      "step": 1194
    },
    {
      "epoch": 6.675977653631285,
      "grad_norm": 3.4694285677076118,
      "learning_rate": 3.618516505918581e-06,
      "logits/chosen": -3.5183441638946533,
      "logits/rejected": -3.439265012741089,
      "logps/chosen": -6.863008499145508,
      "logps/rejected": -61.84767150878906,
      "loss": 0.1629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.090785503387451,
      "rewards/margins": 2.090785503387451,
      "rewards/rejected": 0.0,
      "step": 1195
    },
    {
      "epoch": 6.681564245810056,
      "grad_norm": 2.9008008157977256,
      "learning_rate": 3.6157900034959704e-06,
      "logits/chosen": -3.608008861541748,
      "logits/rejected": -3.685791254043579,
      "logps/chosen": -23.791606903076172,
      "logps/rejected": -48.956993103027344,
      "loss": 0.1279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.667881965637207,
      "rewards/margins": 3.667881965637207,
      "rewards/rejected": 0.0,
      "step": 1196
    },
    {
      "epoch": 6.687150837988827,
      "grad_norm": 2.994604778818588,
      "learning_rate": 3.6130618428107025e-06,
      "logits/chosen": -3.711984395980835,
      "logits/rejected": -3.6275200843811035,
      "logps/chosen": -4.740073204040527,
      "logps/rejected": -47.8201904296875,
      "loss": 0.117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.491058111190796,
      "rewards/margins": 1.491058111190796,
      "rewards/rejected": 0.0,
      "step": 1197
    },
    {
      "epoch": 6.692737430167598,
      "grad_norm": 5.242712030819273,
      "learning_rate": 3.6103320279173102e-06,
      "logits/chosen": -3.4325900077819824,
      "logits/rejected": -3.6539576053619385,
      "logps/chosen": -11.078686714172363,
      "logps/rejected": -58.16801452636719,
      "loss": 0.1486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1246068477630615,
      "rewards/margins": 3.1246068477630615,
      "rewards/rejected": 0.0,
      "step": 1198
    },
    {
      "epoch": 6.698324022346369,
      "grad_norm": 3.409750818447534,
      "learning_rate": 3.607600562872785e-06,
      "logits/chosen": -3.3369717597961426,
      "logits/rejected": -3.4176833629608154,
      "logps/chosen": -1.407241702079773,
      "logps/rejected": -51.28607940673828,
      "loss": 0.1479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.746654987335205,
      "rewards/margins": 1.746654987335205,
      "rewards/rejected": 0.0,
      "step": 1199
    },
    {
      "epoch": 6.70391061452514,
      "grad_norm": 1.5903067438027907,
      "learning_rate": 3.604867451736571e-06,
      "logits/chosen": -3.2746450901031494,
      "logits/rejected": -3.2570905685424805,
      "logps/chosen": -1.549607276916504,
      "logps/rejected": -53.34917068481445,
      "loss": 0.0958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2619409561157227,
      "rewards/margins": 2.2619409561157227,
      "rewards/rejected": 0.0,
      "step": 1200
    },
    {
      "epoch": 6.70391061452514,
      "eval_logits/chosen": -3.358707904815674,
      "eval_logits/rejected": -3.456047534942627,
      "eval_logps/chosen": -34.738197326660156,
      "eval_logps/rejected": -52.904502868652344,
      "eval_loss": 1.2453341484069824,
      "eval_rewards/accuracies": 0.375,
      "eval_rewards/chosen": -0.4668155610561371,
      "eval_rewards/margins": -0.4668155610561371,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6036,
      "eval_samples_per_second": 12.108,
      "eval_steps_per_second": 0.391,
      "step": 1200
    },
    {
      "epoch": 6.709497206703911,
      "grad_norm": 2.718254116964111,
      "learning_rate": 3.602132698570558e-06,
      "logits/chosen": -3.200268030166626,
      "logits/rejected": -3.204192876815796,
      "logps/chosen": -3.4654486179351807,
      "logps/rejected": -49.111724853515625,
      "loss": 0.1407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6467664241790771,
      "rewards/margins": 1.6467664241790771,
      "rewards/rejected": 0.0,
      "step": 1201
    },
    {
      "epoch": 6.715083798882682,
      "grad_norm": 2.899268260427417,
      "learning_rate": 3.5993963074390753e-06,
      "logits/chosen": -3.728611469268799,
      "logits/rejected": -3.753563165664673,
      "logps/chosen": -3.294473648071289,
      "logps/rejected": -47.62419509887695,
      "loss": 0.1161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.145267963409424,
      "rewards/margins": 3.145267963409424,
      "rewards/rejected": 0.0,
      "step": 1202
    },
    {
      "epoch": 6.720670391061453,
      "grad_norm": 2.334809496184261,
      "learning_rate": 3.596658282408888e-06,
      "logits/chosen": -3.4959328174591064,
      "logits/rejected": -3.5434508323669434,
      "logps/chosen": -2.8938140869140625,
      "logps/rejected": -60.49093246459961,
      "loss": 0.119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.190312147140503,
      "rewards/margins": 2.190312147140503,
      "rewards/rejected": 0.0,
      "step": 1203
    },
    {
      "epoch": 6.726256983240224,
      "grad_norm": 6.467806496956333,
      "learning_rate": 3.5939186275491894e-06,
      "logits/chosen": -3.417365789413452,
      "logits/rejected": -3.2698147296905518,
      "logps/chosen": -8.977821350097656,
      "logps/rejected": -68.47224426269531,
      "loss": 0.1946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0479140281677246,
      "rewards/margins": 2.0479140281677246,
      "rewards/rejected": 0.0,
      "step": 1204
    },
    {
      "epoch": 6.731843575418995,
      "grad_norm": 2.9079670859051014,
      "learning_rate": 3.591177346931593e-06,
      "logits/chosen": -3.340362787246704,
      "logits/rejected": -3.6014187335968018,
      "logps/chosen": -0.8695567846298218,
      "logps/rejected": -58.59797286987305,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6913838386535645,
      "rewards/margins": 1.6913838386535645,
      "rewards/rejected": 0.0,
      "step": 1205
    },
    {
      "epoch": 6.737430167597766,
      "grad_norm": 3.445597758503588,
      "learning_rate": 3.588434444630132e-06,
      "logits/chosen": -3.735029935836792,
      "logits/rejected": -3.63885235786438,
      "logps/chosen": -4.516703128814697,
      "logps/rejected": -56.18919372558594,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9885883331298828,
      "rewards/margins": 1.9885883331298828,
      "rewards/rejected": 0.0,
      "step": 1206
    },
    {
      "epoch": 6.743016759776537,
      "grad_norm": 3.988814321835354,
      "learning_rate": 3.585689924721246e-06,
      "logits/chosen": -3.3136649131774902,
      "logits/rejected": -3.261103868484497,
      "logps/chosen": -23.05203628540039,
      "logps/rejected": -51.69981002807617,
      "loss": 0.1435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1570966243743896,
      "rewards/margins": 3.1570966243743896,
      "rewards/rejected": 0.0,
      "step": 1207
    },
    {
      "epoch": 6.748603351955307,
      "grad_norm": 4.326807838915022,
      "learning_rate": 3.582943791283781e-06,
      "logits/chosen": -3.375868082046509,
      "logits/rejected": -3.244872570037842,
      "logps/chosen": -9.186286926269531,
      "logps/rejected": -43.8629150390625,
      "loss": 0.1467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5599589347839355,
      "rewards/margins": 2.5599589347839355,
      "rewards/rejected": 0.0,
      "step": 1208
    },
    {
      "epoch": 6.754189944134078,
      "grad_norm": 2.7155283512324693,
      "learning_rate": 3.580196048398981e-06,
      "logits/chosen": -3.3864502906799316,
      "logits/rejected": -3.3721137046813965,
      "logps/chosen": -4.3876190185546875,
      "logps/rejected": -51.16981506347656,
      "loss": 0.13,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.474050521850586,
      "rewards/margins": 2.474050521850586,
      "rewards/rejected": 0.0,
      "step": 1209
    },
    {
      "epoch": 6.759776536312849,
      "grad_norm": 2.0594640336512153,
      "learning_rate": 3.5774467001504807e-06,
      "logits/chosen": -3.3227529525756836,
      "logits/rejected": -3.2843830585479736,
      "logps/chosen": -13.993382453918457,
      "logps/rejected": -51.7266845703125,
      "loss": 0.1138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.971540927886963,
      "rewards/margins": 3.971540927886963,
      "rewards/rejected": 0.0,
      "step": 1210
    },
    {
      "epoch": 6.76536312849162,
      "grad_norm": 2.4017695576630644,
      "learning_rate": 3.574695750624301e-06,
      "logits/chosen": -3.6090023517608643,
      "logits/rejected": -3.6179182529449463,
      "logps/chosen": -6.177509784698486,
      "logps/rejected": -69.2092514038086,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.532200574874878,
      "rewards/margins": 2.532200574874878,
      "rewards/rejected": 0.0,
      "step": 1211
    },
    {
      "epoch": 6.770949720670391,
      "grad_norm": 3.949092734792092,
      "learning_rate": 3.5719432039088432e-06,
      "logits/chosen": -3.2234604358673096,
      "logits/rejected": -3.1791374683380127,
      "logps/chosen": -1.786184310913086,
      "logps/rejected": -44.634735107421875,
      "loss": 0.1466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1005892753601074,
      "rewards/margins": 2.1005892753601074,
      "rewards/rejected": 0.0,
      "step": 1212
    },
    {
      "epoch": 6.776536312849162,
      "grad_norm": 3.3403627351730565,
      "learning_rate": 3.569189064094882e-06,
      "logits/chosen": -3.2410190105438232,
      "logits/rejected": -3.1844699382781982,
      "logps/chosen": -3.8431761264801025,
      "logps/rejected": -45.300987243652344,
      "loss": 0.1413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9908065795898438,
      "rewards/margins": 0.9908065795898438,
      "rewards/rejected": 0.0,
      "step": 1213
    },
    {
      "epoch": 6.782122905027933,
      "grad_norm": 7.351900222066683,
      "learning_rate": 3.5664333352755586e-06,
      "logits/chosen": -3.657400608062744,
      "logits/rejected": -3.7031350135803223,
      "logps/chosen": -11.068269729614258,
      "logps/rejected": -58.15324401855469,
      "loss": 0.1348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.501215934753418,
      "rewards/margins": 3.501215934753418,
      "rewards/rejected": 0.0,
      "step": 1214
    },
    {
      "epoch": 6.787709497206704,
      "grad_norm": 4.872150970708349,
      "learning_rate": 3.5636760215463778e-06,
      "logits/chosen": -3.475881576538086,
      "logits/rejected": -3.222125291824341,
      "logps/chosen": -5.621091842651367,
      "logps/rejected": -54.80028533935547,
      "loss": 0.1271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7435646057128906,
      "rewards/margins": 2.7435646057128906,
      "rewards/rejected": 0.0,
      "step": 1215
    },
    {
      "epoch": 6.793296089385475,
      "grad_norm": 2.1940370179448023,
      "learning_rate": 3.560917127005199e-06,
      "logits/chosen": -3.644538640975952,
      "logits/rejected": -3.4631941318511963,
      "logps/chosen": -2.5209717750549316,
      "logps/rejected": -38.38048553466797,
      "loss": 0.1359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.080871105194092,
      "rewards/margins": 2.080871105194092,
      "rewards/rejected": 0.0,
      "step": 1216
    },
    {
      "epoch": 6.798882681564246,
      "grad_norm": 3.093454167634596,
      "learning_rate": 3.55815665575223e-06,
      "logits/chosen": -3.8834593296051025,
      "logits/rejected": -3.8816542625427246,
      "logps/chosen": -6.240243434906006,
      "logps/rejected": -59.838165283203125,
      "loss": 0.134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.089115619659424,
      "rewards/margins": 3.089115619659424,
      "rewards/rejected": 0.0,
      "step": 1217
    },
    {
      "epoch": 6.804469273743017,
      "grad_norm": 2.9678800052798406,
      "learning_rate": 3.5553946118900227e-06,
      "logits/chosen": -3.4787113666534424,
      "logits/rejected": -3.589905023574829,
      "logps/chosen": -6.99671745300293,
      "logps/rejected": -67.79549407958984,
      "loss": 0.1324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6433210372924805,
      "rewards/margins": 2.6433210372924805,
      "rewards/rejected": 0.0,
      "step": 1218
    },
    {
      "epoch": 6.810055865921788,
      "grad_norm": 10.457738187062835,
      "learning_rate": 3.5526309995234676e-06,
      "logits/chosen": -3.424924373626709,
      "logits/rejected": -3.3850228786468506,
      "logps/chosen": -19.615924835205078,
      "logps/rejected": -57.679443359375,
      "loss": 0.211,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.7707979679107666,
      "rewards/margins": 0.7707979679107666,
      "rewards/rejected": 0.0,
      "step": 1219
    },
    {
      "epoch": 6.815642458100559,
      "grad_norm": 2.363342788140939,
      "learning_rate": 3.549865822759784e-06,
      "logits/chosen": -3.6532270908355713,
      "logits/rejected": -3.7368929386138916,
      "logps/chosen": -13.092616081237793,
      "logps/rejected": -44.16669464111328,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.978381633758545,
      "rewards/margins": 2.978381633758545,
      "rewards/rejected": 0.0,
      "step": 1220
    },
    {
      "epoch": 6.815642458100559,
      "eval_logits/chosen": -3.358588457107544,
      "eval_logits/rejected": -3.4645233154296875,
      "eval_logps/chosen": -33.250404357910156,
      "eval_logps/rejected": -52.61695098876953,
      "eval_loss": 1.2153328657150269,
      "eval_rewards/accuracies": 0.44999998807907104,
      "eval_rewards/chosen": -0.31803610920906067,
      "eval_rewards/margins": -0.31803610920906067,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5039,
      "eval_samples_per_second": 12.155,
      "eval_steps_per_second": 0.392,
      "step": 1220
    },
    {
      "epoch": 6.82122905027933,
      "grad_norm": 4.136563500079351,
      "learning_rate": 3.5470990857085162e-06,
      "logits/chosen": -3.5219814777374268,
      "logits/rejected": -3.4366648197174072,
      "logps/chosen": -1.2788894176483154,
      "logps/rejected": -48.96375274658203,
      "loss": 0.1521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7272849082946777,
      "rewards/margins": 1.7272849082946777,
      "rewards/rejected": 0.0,
      "step": 1221
    },
    {
      "epoch": 6.826815642458101,
      "grad_norm": 5.259487857332462,
      "learning_rate": 3.5443307924815297e-06,
      "logits/chosen": -3.6544270515441895,
      "logits/rejected": -3.454582452774048,
      "logps/chosen": -8.364638328552246,
      "logps/rejected": -29.392757415771484,
      "loss": 0.1506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8381567001342773,
      "rewards/margins": 2.8381567001342773,
      "rewards/rejected": 0.0,
      "step": 1222
    },
    {
      "epoch": 6.832402234636872,
      "grad_norm": 2.6919622022663643,
      "learning_rate": 3.5415609471930003e-06,
      "logits/chosen": -3.3025543689727783,
      "logits/rejected": -3.429774761199951,
      "logps/chosen": -5.657528400421143,
      "logps/rejected": -72.04032897949219,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.94565486907959,
      "rewards/margins": 2.94565486907959,
      "rewards/rejected": 0.0,
      "step": 1223
    },
    {
      "epoch": 6.837988826815643,
      "grad_norm": 3.513477275625711,
      "learning_rate": 3.538789553959411e-06,
      "logits/chosen": -3.8039350509643555,
      "logits/rejected": -3.6798031330108643,
      "logps/chosen": -4.6705732345581055,
      "logps/rejected": -34.82079315185547,
      "loss": 0.13,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.78250789642334,
      "rewards/margins": 2.78250789642334,
      "rewards/rejected": 0.0,
      "step": 1224
    },
    {
      "epoch": 6.843575418994414,
      "grad_norm": 3.086174283467153,
      "learning_rate": 3.536016616899546e-06,
      "logits/chosen": -3.5758886337280273,
      "logits/rejected": -3.5973963737487793,
      "logps/chosen": -3.5940890312194824,
      "logps/rejected": -78.54000854492188,
      "loss": 0.1326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9924037456512451,
      "rewards/margins": 1.9924037456512451,
      "rewards/rejected": 0.0,
      "step": 1225
    },
    {
      "epoch": 6.849162011173185,
      "grad_norm": 9.447388101162943,
      "learning_rate": 3.533242140134484e-06,
      "logits/chosen": -3.2824013233184814,
      "logits/rejected": -3.3484771251678467,
      "logps/chosen": -5.358222007751465,
      "logps/rejected": -45.078636169433594,
      "loss": 0.2337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.128056049346924,
      "rewards/margins": 3.128056049346924,
      "rewards/rejected": 0.0,
      "step": 1226
    },
    {
      "epoch": 6.854748603351956,
      "grad_norm": 3.243560915824732,
      "learning_rate": 3.530466127787592e-06,
      "logits/chosen": -3.4562830924987793,
      "logits/rejected": -3.2959349155426025,
      "logps/chosen": -2.355396032333374,
      "logps/rejected": -111.21221923828125,
      "loss": 0.1347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1651711463928223,
      "rewards/margins": 2.1651711463928223,
      "rewards/rejected": 0.0,
      "step": 1227
    },
    {
      "epoch": 6.860335195530726,
      "grad_norm": 4.82998353738795,
      "learning_rate": 3.5276885839845177e-06,
      "logits/chosen": -3.5984251499176025,
      "logits/rejected": -3.6833364963531494,
      "logps/chosen": -3.3010337352752686,
      "logps/rejected": -45.88125991821289,
      "loss": 0.1403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.007054328918457,
      "rewards/margins": 2.007054328918457,
      "rewards/rejected": 0.0,
      "step": 1228
    },
    {
      "epoch": 6.865921787709497,
      "grad_norm": 8.30064424438668,
      "learning_rate": 3.5249095128531863e-06,
      "logits/chosen": -3.6454827785491943,
      "logits/rejected": -3.546408176422119,
      "logps/chosen": -1.765796422958374,
      "logps/rejected": -39.02449035644531,
      "loss": 0.1612,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.349612236022949,
      "rewards/margins": 2.349612236022949,
      "rewards/rejected": 0.0,
      "step": 1229
    },
    {
      "epoch": 6.871508379888268,
      "grad_norm": 2.7071535808449214,
      "learning_rate": 3.5221289185237923e-06,
      "logits/chosen": -3.547161102294922,
      "logits/rejected": -3.505394697189331,
      "logps/chosen": -6.130772590637207,
      "logps/rejected": -57.223175048828125,
      "loss": 0.1503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4147212505340576,
      "rewards/margins": 2.4147212505340576,
      "rewards/rejected": 0.0,
      "step": 1230
    },
    {
      "epoch": 6.877094972067039,
      "grad_norm": 2.8679358778216018,
      "learning_rate": 3.5193468051287945e-06,
      "logits/chosen": -3.50045108795166,
      "logits/rejected": -3.2234549522399902,
      "logps/chosen": -7.000777244567871,
      "logps/rejected": -42.16571807861328,
      "loss": 0.1352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.734152317047119,
      "rewards/margins": 2.734152317047119,
      "rewards/rejected": 0.0,
      "step": 1231
    },
    {
      "epoch": 6.88268156424581,
      "grad_norm": 4.891321969006927,
      "learning_rate": 3.5165631768029086e-06,
      "logits/chosen": -3.3600335121154785,
      "logits/rejected": -3.142240524291992,
      "logps/chosen": -10.404138565063477,
      "logps/rejected": -28.696144104003906,
      "loss": 0.1856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0897376537323,
      "rewards/margins": 2.0897376537323,
      "rewards/rejected": 0.0,
      "step": 1232
    },
    {
      "epoch": 6.888268156424581,
      "grad_norm": 3.6988786692262674,
      "learning_rate": 3.5137780376831016e-06,
      "logits/chosen": -3.301203966140747,
      "logits/rejected": -3.3994667530059814,
      "logps/chosen": -2.9594907760620117,
      "logps/rejected": -62.231685638427734,
      "loss": 0.1456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.468902826309204,
      "rewards/margins": 2.468902826309204,
      "rewards/rejected": 0.0,
      "step": 1233
    },
    {
      "epoch": 6.893854748603352,
      "grad_norm": 3.804001114424846,
      "learning_rate": 3.5109913919085866e-06,
      "logits/chosen": -3.4081900119781494,
      "logits/rejected": -3.5142111778259277,
      "logps/chosen": -8.51041316986084,
      "logps/rejected": -53.443511962890625,
      "loss": 0.1209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8947439193725586,
      "rewards/margins": 2.8947439193725586,
      "rewards/rejected": 0.0,
      "step": 1234
    },
    {
      "epoch": 6.899441340782123,
      "grad_norm": 3.3928225572991573,
      "learning_rate": 3.5082032436208156e-06,
      "logits/chosen": -3.3903284072875977,
      "logits/rejected": -3.360440254211426,
      "logps/chosen": -3.2188973426818848,
      "logps/rejected": -54.100547790527344,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7623515129089355,
      "rewards/margins": 2.7623515129089355,
      "rewards/rejected": 0.0,
      "step": 1235
    },
    {
      "epoch": 6.905027932960894,
      "grad_norm": 4.870336524663638,
      "learning_rate": 3.5054135969634734e-06,
      "logits/chosen": -3.483417510986328,
      "logits/rejected": -3.457212209701538,
      "logps/chosen": -3.3400022983551025,
      "logps/rejected": -66.06517028808594,
      "loss": 0.1534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1301727294921875,
      "rewards/margins": 1.1301727294921875,
      "rewards/rejected": 0.0,
      "step": 1236
    },
    {
      "epoch": 6.910614525139665,
      "grad_norm": 1.9238381238156976,
      "learning_rate": 3.5026224560824712e-06,
      "logits/chosen": -3.312150239944458,
      "logits/rejected": -3.438943386077881,
      "logps/chosen": -4.530386447906494,
      "logps/rejected": -46.82548904418945,
      "loss": 0.1091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8358852863311768,
      "rewards/margins": 2.8358852863311768,
      "rewards/rejected": 0.0,
      "step": 1237
    },
    {
      "epoch": 6.916201117318436,
      "grad_norm": 2.069693942580619,
      "learning_rate": 3.499829825125943e-06,
      "logits/chosen": -3.5653109550476074,
      "logits/rejected": -3.516477108001709,
      "logps/chosen": -13.937589645385742,
      "logps/rejected": -46.726646423339844,
      "loss": 0.0978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9357876777648926,
      "rewards/margins": 3.9357876777648926,
      "rewards/rejected": 0.0,
      "step": 1238
    },
    {
      "epoch": 6.921787709497207,
      "grad_norm": 2.95560471528565,
      "learning_rate": 3.497035708244234e-06,
      "logits/chosen": -3.5304312705993652,
      "logits/rejected": -3.8631017208099365,
      "logps/chosen": -3.1259589195251465,
      "logps/rejected": -80.44233703613281,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0105416774749756,
      "rewards/margins": 2.0105416774749756,
      "rewards/rejected": 0.0,
      "step": 1239
    },
    {
      "epoch": 6.927374301675978,
      "grad_norm": 9.263184984622647,
      "learning_rate": 3.494240109589901e-06,
      "logits/chosen": -3.4738199710845947,
      "logits/rejected": -2.9943108558654785,
      "logps/chosen": -2.8257861137390137,
      "logps/rejected": -54.57257843017578,
      "loss": 0.2047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6011359691619873,
      "rewards/margins": 1.6011359691619873,
      "rewards/rejected": 0.0,
      "step": 1240
    },
    {
      "epoch": 6.927374301675978,
      "eval_logits/chosen": -3.3507168292999268,
      "eval_logits/rejected": -3.4483284950256348,
      "eval_logps/chosen": -32.52306365966797,
      "eval_logps/rejected": -50.034523010253906,
      "eval_loss": 1.1460907459259033,
      "eval_rewards/accuracies": 0.42500001192092896,
      "eval_rewards/chosen": -0.24530236423015594,
      "eval_rewards/margins": -0.24530236423015594,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.622,
      "eval_samples_per_second": 12.099,
      "eval_steps_per_second": 0.39,
      "step": 1240
    },
    {
      "epoch": 6.932960893854749,
      "grad_norm": 1.7068885329518981,
      "learning_rate": 3.4914430333176995e-06,
      "logits/chosen": -3.414655923843384,
      "logits/rejected": -3.5276355743408203,
      "logps/chosen": -4.202807903289795,
      "logps/rejected": -52.52000045776367,
      "loss": 0.0877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0082130432128906,
      "rewards/margins": 3.0082130432128906,
      "rewards/rejected": 0.0,
      "step": 1241
    },
    {
      "epoch": 6.93854748603352,
      "grad_norm": 5.253725865288661,
      "learning_rate": 3.4886444835845858e-06,
      "logits/chosen": -3.202444076538086,
      "logits/rejected": -3.2403335571289062,
      "logps/chosen": -4.765389442443848,
      "logps/rejected": -36.497093200683594,
      "loss": 0.1953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2606914043426514,
      "rewards/margins": 2.2606914043426514,
      "rewards/rejected": 0.0,
      "step": 1242
    },
    {
      "epoch": 6.94413407821229,
      "grad_norm": 2.606889857754792,
      "learning_rate": 3.4858444645497003e-06,
      "logits/chosen": -3.4344732761383057,
      "logits/rejected": -3.5828490257263184,
      "logps/chosen": -12.154623985290527,
      "logps/rejected": -33.17033386230469,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5837173461914062,
      "rewards/margins": 2.5837173461914062,
      "rewards/rejected": 0.0,
      "step": 1243
    },
    {
      "epoch": 6.949720670391061,
      "grad_norm": 2.8511635523114847,
      "learning_rate": 3.4830429803743708e-06,
      "logits/chosen": -3.3671274185180664,
      "logits/rejected": -3.4146318435668945,
      "logps/chosen": -10.470701217651367,
      "logps/rejected": -39.74541091918945,
      "loss": 0.1463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6055617332458496,
      "rewards/margins": 3.6055617332458496,
      "rewards/rejected": 0.0,
      "step": 1244
    },
    {
      "epoch": 6.955307262569832,
      "grad_norm": 1.8038679449030588,
      "learning_rate": 3.480240035222103e-06,
      "logits/chosen": -3.531485080718994,
      "logits/rejected": -3.618212938308716,
      "logps/chosen": -6.519667625427246,
      "logps/rejected": -31.651111602783203,
      "loss": 0.0749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.236501693725586,
      "rewards/margins": 3.236501693725586,
      "rewards/rejected": 0.0,
      "step": 1245
    },
    {
      "epoch": 6.960893854748603,
      "grad_norm": 3.884038489569088,
      "learning_rate": 3.477435633258571e-06,
      "logits/chosen": -3.5737574100494385,
      "logits/rejected": -3.50321888923645,
      "logps/chosen": -6.5661725997924805,
      "logps/rejected": -41.12270736694336,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1711878776550293,
      "rewards/margins": 3.1711878776550293,
      "rewards/rejected": 0.0,
      "step": 1246
    },
    {
      "epoch": 6.966480446927374,
      "grad_norm": 2.398757738925785,
      "learning_rate": 3.4746297786516165e-06,
      "logits/chosen": -3.5704967975616455,
      "logits/rejected": -3.5431551933288574,
      "logps/chosen": -4.775882720947266,
      "logps/rejected": -26.65507698059082,
      "loss": 0.1338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.307112216949463,
      "rewards/margins": 2.307112216949463,
      "rewards/rejected": 0.0,
      "step": 1247
    },
    {
      "epoch": 6.972067039106145,
      "grad_norm": 3.3994695236732846,
      "learning_rate": 3.4718224755712393e-06,
      "logits/chosen": -3.7280077934265137,
      "logits/rejected": -3.7147397994995117,
      "logps/chosen": -22.558582305908203,
      "logps/rejected": -67.85498046875,
      "loss": 0.1748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.86710262298584,
      "rewards/margins": 3.86710262298584,
      "rewards/rejected": 0.0,
      "step": 1248
    },
    {
      "epoch": 6.977653631284916,
      "grad_norm": 5.191025345902313,
      "learning_rate": 3.469013728189591e-06,
      "logits/chosen": -3.7348906993865967,
      "logits/rejected": -3.8140337467193604,
      "logps/chosen": -7.7849907875061035,
      "logps/rejected": -42.473960876464844,
      "loss": 0.1381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.686887502670288,
      "rewards/margins": 1.686887502670288,
      "rewards/rejected": 0.0,
      "step": 1249
    },
    {
      "epoch": 6.983240223463687,
      "grad_norm": 3.5319937307753655,
      "learning_rate": 3.46620354068097e-06,
      "logits/chosen": -3.376342296600342,
      "logits/rejected": -3.2004027366638184,
      "logps/chosen": -39.394683837890625,
      "logps/rejected": -50.76462936401367,
      "loss": 0.1359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8050692081451416,
      "rewards/margins": 2.8050692081451416,
      "rewards/rejected": 0.0,
      "step": 1250
    },
    {
      "epoch": 6.988826815642458,
      "grad_norm": 2.3171426399918627,
      "learning_rate": 3.463391917221817e-06,
      "logits/chosen": -3.780707359313965,
      "logits/rejected": -3.5573558807373047,
      "logps/chosen": -1.3296253681182861,
      "logps/rejected": -37.755584716796875,
      "loss": 0.1279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8918288946151733,
      "rewards/margins": 1.8918288946151733,
      "rewards/rejected": 0.0,
      "step": 1251
    },
    {
      "epoch": 6.994413407821229,
      "grad_norm": 1.8031389009410512,
      "learning_rate": 3.4605788619907044e-06,
      "logits/chosen": -3.5467166900634766,
      "logits/rejected": -3.434987783432007,
      "logps/chosen": -32.30268478393555,
      "logps/rejected": -36.251983642578125,
      "loss": 0.1046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6422905921936035,
      "rewards/margins": 2.6422905921936035,
      "rewards/rejected": 0.0,
      "step": 1252
    },
    {
      "epoch": 7.0,
      "grad_norm": 2.3203576519655535,
      "learning_rate": 3.4577643791683324e-06,
      "logits/chosen": -3.5667901039123535,
      "logits/rejected": -3.5666840076446533,
      "logps/chosen": -9.10793685913086,
      "logps/rejected": -30.24134063720703,
      "loss": 0.1304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.705249071121216,
      "rewards/margins": 2.705249071121216,
      "rewards/rejected": 0.0,
      "step": 1253
    },
    {
      "epoch": 7.005586592178771,
      "grad_norm": 1.9589170444326944,
      "learning_rate": 3.454948472937525e-06,
      "logits/chosen": -3.5351386070251465,
      "logits/rejected": -3.460944414138794,
      "logps/chosen": -5.1596999168396,
      "logps/rejected": -44.35716247558594,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4183311462402344,
      "rewards/margins": 3.4183311462402344,
      "rewards/rejected": 0.0,
      "step": 1254
    },
    {
      "epoch": 7.011173184357542,
      "grad_norm": 2.1984685602086866,
      "learning_rate": 3.4521311474832196e-06,
      "logits/chosen": -3.7934203147888184,
      "logits/rejected": -3.983018159866333,
      "logps/chosen": -5.046698570251465,
      "logps/rejected": -60.68435287475586,
      "loss": 0.1141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6671862602233887,
      "rewards/margins": 3.6671862602233887,
      "rewards/rejected": 0.0,
      "step": 1255
    },
    {
      "epoch": 7.016759776536313,
      "grad_norm": 2.467919115820958,
      "learning_rate": 3.449312406992464e-06,
      "logits/chosen": -3.449430227279663,
      "logits/rejected": -3.324262857437134,
      "logps/chosen": -1.5647128820419312,
      "logps/rejected": -93.41938781738281,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.465993881225586,
      "rewards/margins": 2.465993881225586,
      "rewards/rejected": 0.0,
      "step": 1256
    },
    {
      "epoch": 7.022346368715084,
      "grad_norm": 0.9284605140013336,
      "learning_rate": 3.4464922556544074e-06,
      "logits/chosen": -3.7484259605407715,
      "logits/rejected": -3.621194362640381,
      "logps/chosen": -2.5483381748199463,
      "logps/rejected": -56.58686065673828,
      "loss": 0.1036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7000746726989746,
      "rewards/margins": 2.7000746726989746,
      "rewards/rejected": 0.0,
      "step": 1257
    },
    {
      "epoch": 7.027932960893855,
      "grad_norm": 2.3225443069495286,
      "learning_rate": 3.4436706976602987e-06,
      "logits/chosen": -3.4254002571105957,
      "logits/rejected": -3.4771718978881836,
      "logps/chosen": -0.6787981390953064,
      "logps/rejected": -42.476531982421875,
      "loss": 0.136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.772031307220459,
      "rewards/margins": 1.772031307220459,
      "rewards/rejected": 0.0,
      "step": 1258
    },
    {
      "epoch": 7.033519553072626,
      "grad_norm": 2.489694873849441,
      "learning_rate": 3.4408477372034743e-06,
      "logits/chosen": -3.6989920139312744,
      "logits/rejected": -3.7951083183288574,
      "logps/chosen": -2.3584017753601074,
      "logps/rejected": -55.804622650146484,
      "loss": 0.1044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9420567750930786,
      "rewards/margins": 1.9420567750930786,
      "rewards/rejected": 0.0,
      "step": 1259
    },
    {
      "epoch": 7.039106145251397,
      "grad_norm": 5.9712750332737645,
      "learning_rate": 3.438023378479357e-06,
      "logits/chosen": -3.6102144718170166,
      "logits/rejected": -3.707829713821411,
      "logps/chosen": -12.798823356628418,
      "logps/rejected": -67.20636749267578,
      "loss": 0.1328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8480570316314697,
      "rewards/margins": 2.8480570316314697,
      "rewards/rejected": 0.0,
      "step": 1260
    },
    {
      "epoch": 7.039106145251397,
      "eval_logits/chosen": -3.4136123657226562,
      "eval_logits/rejected": -3.533205509185791,
      "eval_logps/chosen": -36.52300262451172,
      "eval_logps/rejected": -55.880470275878906,
      "eval_loss": 1.4105753898620605,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.6452959775924683,
      "eval_rewards/margins": -0.6452959775924683,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5359,
      "eval_samples_per_second": 12.14,
      "eval_steps_per_second": 0.392,
      "step": 1260
    },
    {
      "epoch": 7.044692737430168,
      "grad_norm": 1.490725280972391,
      "learning_rate": 3.435197625685447e-06,
      "logits/chosen": -3.6744790077209473,
      "logits/rejected": -3.7605762481689453,
      "logps/chosen": -8.379554748535156,
      "logps/rejected": -56.64335632324219,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.140006065368652,
      "rewards/margins": 4.140006065368652,
      "rewards/rejected": 0.0,
      "step": 1261
    },
    {
      "epoch": 7.050279329608939,
      "grad_norm": 4.543019033845347,
      "learning_rate": 3.4323704830213144e-06,
      "logits/chosen": -3.384969472885132,
      "logits/rejected": -3.398651123046875,
      "logps/chosen": -1.6206963062286377,
      "logps/rejected": -47.572723388671875,
      "loss": 0.1155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3551454544067383,
      "rewards/margins": 2.3551454544067383,
      "rewards/rejected": 0.0,
      "step": 1262
    },
    {
      "epoch": 7.055865921787709,
      "grad_norm": 6.584046176847299,
      "learning_rate": 3.4295419546885983e-06,
      "logits/chosen": -3.387070417404175,
      "logits/rejected": -3.3898098468780518,
      "logps/chosen": -11.58211898803711,
      "logps/rejected": -45.46614074707031,
      "loss": 0.1471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.942746162414551,
      "rewards/margins": 2.942746162414551,
      "rewards/rejected": 0.0,
      "step": 1263
    },
    {
      "epoch": 7.06145251396648,
      "grad_norm": 2.2072440272688363,
      "learning_rate": 3.426712044890995e-06,
      "logits/chosen": -3.3324785232543945,
      "logits/rejected": -3.6151840686798096,
      "logps/chosen": -4.728226661682129,
      "logps/rejected": -76.41590881347656,
      "loss": 0.1481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.79166841506958,
      "rewards/margins": 2.79166841506958,
      "rewards/rejected": 0.0,
      "step": 1264
    },
    {
      "epoch": 7.067039106145251,
      "grad_norm": 3.8804394099448944,
      "learning_rate": 3.423880757834256e-06,
      "logits/chosen": -3.523740530014038,
      "logits/rejected": -3.337608814239502,
      "logps/chosen": -19.16461181640625,
      "logps/rejected": -64.19200134277344,
      "loss": 0.1515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5011651515960693,
      "rewards/margins": 2.5011651515960693,
      "rewards/rejected": 0.0,
      "step": 1265
    },
    {
      "epoch": 7.072625698324022,
      "grad_norm": 4.879147252070711,
      "learning_rate": 3.421048097726175e-06,
      "logits/chosen": -3.2057077884674072,
      "logits/rejected": -3.50311541557312,
      "logps/chosen": -3.8205618858337402,
      "logps/rejected": -118.35800170898438,
      "loss": 0.1353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.854252815246582,
      "rewards/margins": 1.854252815246582,
      "rewards/rejected": 0.0,
      "step": 1266
    },
    {
      "epoch": 7.078212290502793,
      "grad_norm": 1.9983476123561497,
      "learning_rate": 3.418214068776592e-06,
      "logits/chosen": -3.7289559841156006,
      "logits/rejected": -3.7308921813964844,
      "logps/chosen": -4.540436267852783,
      "logps/rejected": -73.53610229492188,
      "loss": 0.1062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.33681058883667,
      "rewards/margins": 3.33681058883667,
      "rewards/rejected": 0.0,
      "step": 1267
    },
    {
      "epoch": 7.083798882681564,
      "grad_norm": 1.365878597781876,
      "learning_rate": 3.415378675197377e-06,
      "logits/chosen": -3.56766939163208,
      "logits/rejected": -3.5680999755859375,
      "logps/chosen": -0.4147040545940399,
      "logps/rejected": -72.73690795898438,
      "loss": 0.1253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7238593101501465,
      "rewards/margins": 1.7238593101501465,
      "rewards/rejected": 0.0,
      "step": 1268
    },
    {
      "epoch": 7.089385474860335,
      "grad_norm": 2.8088068538385893,
      "learning_rate": 3.4125419212024314e-06,
      "logits/chosen": -3.5472803115844727,
      "logits/rejected": -3.548877716064453,
      "logps/chosen": -3.468842029571533,
      "logps/rejected": -118.79151916503906,
      "loss": 0.149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6061608791351318,
      "rewards/margins": 1.6061608791351318,
      "rewards/rejected": 0.0,
      "step": 1269
    },
    {
      "epoch": 7.094972067039106,
      "grad_norm": 2.117169116015229,
      "learning_rate": 3.4097038110076763e-06,
      "logits/chosen": -3.874539613723755,
      "logits/rejected": -3.692657470703125,
      "logps/chosen": -10.614328384399414,
      "logps/rejected": -30.23505210876465,
      "loss": 0.1203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.836109161376953,
      "rewards/margins": 3.836109161376953,
      "rewards/rejected": 0.0,
      "step": 1270
    },
    {
      "epoch": 7.100558659217877,
      "grad_norm": 3.2205050700650757,
      "learning_rate": 3.4068643488310484e-06,
      "logits/chosen": -3.7756195068359375,
      "logits/rejected": -3.536750555038452,
      "logps/chosen": -4.11277437210083,
      "logps/rejected": -72.65196228027344,
      "loss": 0.0928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8362035751342773,
      "rewards/margins": 2.8362035751342773,
      "rewards/rejected": 0.0,
      "step": 1271
    },
    {
      "epoch": 7.106145251396648,
      "grad_norm": 1.9168388884520404,
      "learning_rate": 3.404023538892494e-06,
      "logits/chosen": -3.443941593170166,
      "logits/rejected": -3.636814594268799,
      "logps/chosen": -8.74711799621582,
      "logps/rejected": -53.02593994140625,
      "loss": 0.0787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.597242832183838,
      "rewards/margins": 2.597242832183838,
      "rewards/rejected": 0.0,
      "step": 1272
    },
    {
      "epoch": 7.111731843575419,
      "grad_norm": 2.764200168266773,
      "learning_rate": 3.401181385413963e-06,
      "logits/chosen": -3.47540545463562,
      "logits/rejected": -3.5345706939697266,
      "logps/chosen": -7.91676139831543,
      "logps/rejected": -53.14167022705078,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2978498935699463,
      "rewards/margins": 3.2978498935699463,
      "rewards/rejected": 0.0,
      "step": 1273
    },
    {
      "epoch": 7.11731843575419,
      "grad_norm": 2.744710354523102,
      "learning_rate": 3.398337892619402e-06,
      "logits/chosen": -3.9526751041412354,
      "logits/rejected": -3.8744547367095947,
      "logps/chosen": -10.078615188598633,
      "logps/rejected": -43.824012756347656,
      "loss": 0.1142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1361474990844727,
      "rewards/margins": 3.1361474990844727,
      "rewards/rejected": 0.0,
      "step": 1274
    },
    {
      "epoch": 7.122905027932961,
      "grad_norm": 3.7305919972311314,
      "learning_rate": 3.3954930647347475e-06,
      "logits/chosen": -3.9182820320129395,
      "logits/rejected": -3.8428125381469727,
      "logps/chosen": -3.3205528259277344,
      "logps/rejected": -43.501747131347656,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.64321231842041,
      "rewards/margins": 2.64321231842041,
      "rewards/rejected": 0.0,
      "step": 1275
    },
    {
      "epoch": 7.128491620111732,
      "grad_norm": 2.236418902214291,
      "learning_rate": 3.3926469059879192e-06,
      "logits/chosen": -3.6683576107025146,
      "logits/rejected": -3.8646063804626465,
      "logps/chosen": -1.3951953649520874,
      "logps/rejected": -61.587581634521484,
      "loss": 0.1165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9078558683395386,
      "rewards/margins": 1.9078558683395386,
      "rewards/rejected": 0.0,
      "step": 1276
    },
    {
      "epoch": 7.134078212290503,
      "grad_norm": 1.9309270516377368,
      "learning_rate": 3.3897994206088182e-06,
      "logits/chosen": -3.321279525756836,
      "logits/rejected": -3.6658573150634766,
      "logps/chosen": -1.5325778722763062,
      "logps/rejected": -54.25023651123047,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.208768129348755,
      "rewards/margins": 2.208768129348755,
      "rewards/rejected": 0.0,
      "step": 1277
    },
    {
      "epoch": 7.139664804469274,
      "grad_norm": 1.987614566937524,
      "learning_rate": 3.386950612829313e-06,
      "logits/chosen": -3.696073293685913,
      "logits/rejected": -3.405224084854126,
      "logps/chosen": -2.394883155822754,
      "logps/rejected": -54.19145202636719,
      "loss": 0.1174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.266507863998413,
      "rewards/margins": 2.266507863998413,
      "rewards/rejected": 0.0,
      "step": 1278
    },
    {
      "epoch": 7.145251396648045,
      "grad_norm": 1.9149338534684308,
      "learning_rate": 3.3841004868832396e-06,
      "logits/chosen": -3.436206102371216,
      "logits/rejected": -3.4030215740203857,
      "logps/chosen": -2.7910642623901367,
      "logps/rejected": -53.86592102050781,
      "loss": 0.1016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.126889705657959,
      "rewards/margins": 3.126889705657959,
      "rewards/rejected": 0.0,
      "step": 1279
    },
    {
      "epoch": 7.150837988826815,
      "grad_norm": 1.9295702725135793,
      "learning_rate": 3.3812490470063935e-06,
      "logits/chosen": -3.2617270946502686,
      "logits/rejected": -3.340834856033325,
      "logps/chosen": -3.7884607315063477,
      "logps/rejected": -44.07638168334961,
      "loss": 0.1235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.024786949157715,
      "rewards/margins": 2.024786949157715,
      "rewards/rejected": 0.0,
      "step": 1280
    },
    {
      "epoch": 7.150837988826815,
      "eval_logits/chosen": -3.438541889190674,
      "eval_logits/rejected": -3.554960250854492,
      "eval_logps/chosen": -34.88954544067383,
      "eval_logps/rejected": -54.88138961791992,
      "eval_loss": 1.2975093126296997,
      "eval_rewards/accuracies": 0.3499999940395355,
      "eval_rewards/chosen": -0.4819501042366028,
      "eval_rewards/margins": -0.4819501042366028,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5333,
      "eval_samples_per_second": 12.141,
      "eval_steps_per_second": 0.392,
      "step": 1280
    },
    {
      "epoch": 7.156424581005586,
      "grad_norm": 25.09121292902562,
      "learning_rate": 3.3783962974365215e-06,
      "logits/chosen": -3.491154432296753,
      "logits/rejected": -3.4791672229766846,
      "logps/chosen": -5.431025505065918,
      "logps/rejected": -48.25931930541992,
      "loss": 0.1949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.647160291671753,
      "rewards/margins": 2.647160291671753,
      "rewards/rejected": 0.0,
      "step": 1281
    },
    {
      "epoch": 7.162011173184357,
      "grad_norm": 2.3447092662582496,
      "learning_rate": 3.3755422424133176e-06,
      "logits/chosen": -3.903700828552246,
      "logits/rejected": -3.891294240951538,
      "logps/chosen": -1.9472589492797852,
      "logps/rejected": -47.849754333496094,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6916754245758057,
      "rewards/margins": 2.6916754245758057,
      "rewards/rejected": 0.0,
      "step": 1282
    },
    {
      "epoch": 7.167597765363128,
      "grad_norm": 2.181722348054458,
      "learning_rate": 3.3726868861784156e-06,
      "logits/chosen": -3.772129774093628,
      "logits/rejected": -3.5751514434814453,
      "logps/chosen": -1.823430061340332,
      "logps/rejected": -75.5226058959961,
      "loss": 0.1212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6394705772399902,
      "rewards/margins": 2.6394705772399902,
      "rewards/rejected": 0.0,
      "step": 1283
    },
    {
      "epoch": 7.173184357541899,
      "grad_norm": 1.745137812133354,
      "learning_rate": 3.3698302329753846e-06,
      "logits/chosen": -2.8099474906921387,
      "logits/rejected": -2.6008574962615967,
      "logps/chosen": -13.320572853088379,
      "logps/rejected": -52.993656158447266,
      "loss": 0.1017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4391660690307617,
      "rewards/margins": 2.4391660690307617,
      "rewards/rejected": 0.0,
      "step": 1284
    },
    {
      "epoch": 7.17877094972067,
      "grad_norm": 2.3339033580253017,
      "learning_rate": 3.3669722870497184e-06,
      "logits/chosen": -3.5359156131744385,
      "logits/rejected": -3.4708092212677,
      "logps/chosen": -10.631938934326172,
      "logps/rejected": -55.145145416259766,
      "loss": 0.1407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.181898832321167,
      "rewards/margins": 3.181898832321167,
      "rewards/rejected": 0.0,
      "step": 1285
    },
    {
      "epoch": 7.184357541899441,
      "grad_norm": 1.7601051441089406,
      "learning_rate": 3.364113052648834e-06,
      "logits/chosen": -3.8395466804504395,
      "logits/rejected": -3.7197999954223633,
      "logps/chosen": -1.8532118797302246,
      "logps/rejected": -75.51272583007812,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3407483100891113,
      "rewards/margins": 2.3407483100891113,
      "rewards/rejected": 0.0,
      "step": 1286
    },
    {
      "epoch": 7.189944134078212,
      "grad_norm": 2.0103078299560226,
      "learning_rate": 3.3612525340220642e-06,
      "logits/chosen": -3.579287052154541,
      "logits/rejected": -3.5987939834594727,
      "logps/chosen": -11.54574203491211,
      "logps/rejected": -51.559410095214844,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.346566200256348,
      "rewards/margins": 4.346566200256348,
      "rewards/rejected": 0.0,
      "step": 1287
    },
    {
      "epoch": 7.195530726256983,
      "grad_norm": 4.583592079075617,
      "learning_rate": 3.358390735420648e-06,
      "logits/chosen": -3.592393398284912,
      "logits/rejected": -3.6800501346588135,
      "logps/chosen": -3.8706672191619873,
      "logps/rejected": -71.81122589111328,
      "loss": 0.1455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.539069652557373,
      "rewards/margins": 1.539069652557373,
      "rewards/rejected": 0.0,
      "step": 1288
    },
    {
      "epoch": 7.201117318435754,
      "grad_norm": 2.816492081829169,
      "learning_rate": 3.355527661097728e-06,
      "logits/chosen": -3.875272274017334,
      "logits/rejected": -3.708327054977417,
      "logps/chosen": -5.344283103942871,
      "logps/rejected": -44.72226333618164,
      "loss": 0.0951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6454520225524902,
      "rewards/margins": 3.6454520225524902,
      "rewards/rejected": 0.0,
      "step": 1289
    },
    {
      "epoch": 7.206703910614525,
      "grad_norm": 1.7471544140201396,
      "learning_rate": 3.352663315308342e-06,
      "logits/chosen": -3.2877049446105957,
      "logits/rejected": -3.684209108352661,
      "logps/chosen": -1.2729530334472656,
      "logps/rejected": -54.57737731933594,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.156771183013916,
      "rewards/margins": 2.156771183013916,
      "rewards/rejected": 0.0,
      "step": 1290
    },
    {
      "epoch": 7.212290502793296,
      "grad_norm": 2.612867844883714,
      "learning_rate": 3.3497977023094186e-06,
      "logits/chosen": -3.8285257816314697,
      "logits/rejected": -3.7119204998016357,
      "logps/chosen": -7.7488250732421875,
      "logps/rejected": -37.15199279785156,
      "loss": 0.1202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.841129779815674,
      "rewards/margins": 2.841129779815674,
      "rewards/rejected": 0.0,
      "step": 1291
    },
    {
      "epoch": 7.217877094972067,
      "grad_norm": 3.135571694845437,
      "learning_rate": 3.3469308263597682e-06,
      "logits/chosen": -3.6590006351470947,
      "logits/rejected": -3.813450813293457,
      "logps/chosen": -3.7733592987060547,
      "logps/rejected": -84.60116577148438,
      "loss": 0.1127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9001359939575195,
      "rewards/margins": 2.9001359939575195,
      "rewards/rejected": 0.0,
      "step": 1292
    },
    {
      "epoch": 7.223463687150838,
      "grad_norm": 11.634424897611199,
      "learning_rate": 3.3440626917200802e-06,
      "logits/chosen": -3.5266711711883545,
      "logits/rejected": -3.4917802810668945,
      "logps/chosen": -42.46406173706055,
      "logps/rejected": -39.562286376953125,
      "loss": 0.1421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.3626251220703125,
      "rewards/margins": 4.3626251220703125,
      "rewards/rejected": 0.0,
      "step": 1293
    },
    {
      "epoch": 7.229050279329609,
      "grad_norm": 1.64621953082124,
      "learning_rate": 3.3411933026529137e-06,
      "logits/chosen": -3.496872663497925,
      "logits/rejected": -3.444789171218872,
      "logps/chosen": -3.21152663230896,
      "logps/rejected": -57.4774169921875,
      "loss": 0.1089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.67983341217041,
      "rewards/margins": 2.67983341217041,
      "rewards/rejected": 0.0,
      "step": 1294
    },
    {
      "epoch": 7.23463687150838,
      "grad_norm": 4.834586146601968,
      "learning_rate": 3.3383226634226902e-06,
      "logits/chosen": -3.4799888134002686,
      "logits/rejected": -3.342108726501465,
      "logps/chosen": -4.725401401519775,
      "logps/rejected": -38.707550048828125,
      "loss": 0.1432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.993532657623291,
      "rewards/margins": 2.993532657623291,
      "rewards/rejected": 0.0,
      "step": 1295
    },
    {
      "epoch": 7.240223463687151,
      "grad_norm": 3.4985780886094475,
      "learning_rate": 3.335450778295692e-06,
      "logits/chosen": -3.4759716987609863,
      "logits/rejected": -3.4246718883514404,
      "logps/chosen": -0.8729925155639648,
      "logps/rejected": -45.821964263916016,
      "loss": 0.14,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9603979587554932,
      "rewards/margins": 1.9603979587554932,
      "rewards/rejected": 0.0,
      "step": 1296
    },
    {
      "epoch": 7.245810055865922,
      "grad_norm": 2.2984706678896267,
      "learning_rate": 3.332577651540052e-06,
      "logits/chosen": -3.4497859477996826,
      "logits/rejected": -3.647308349609375,
      "logps/chosen": -5.817111492156982,
      "logps/rejected": -61.148799896240234,
      "loss": 0.1002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.828690528869629,
      "rewards/margins": 3.828690528869629,
      "rewards/rejected": 0.0,
      "step": 1297
    },
    {
      "epoch": 7.251396648044693,
      "grad_norm": 3.105387485589185,
      "learning_rate": 3.3297032874257474e-06,
      "logits/chosen": -3.560116767883301,
      "logits/rejected": -3.472620964050293,
      "logps/chosen": -3.3382365703582764,
      "logps/rejected": -70.62516784667969,
      "loss": 0.1081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2756729125976562,
      "rewards/margins": 3.2756729125976562,
      "rewards/rejected": 0.0,
      "step": 1298
    },
    {
      "epoch": 7.256983240223463,
      "grad_norm": 4.694930850118088,
      "learning_rate": 3.3268276902245962e-06,
      "logits/chosen": -2.9434919357299805,
      "logits/rejected": -2.8958568572998047,
      "logps/chosen": -20.13140106201172,
      "logps/rejected": -68.50651550292969,
      "loss": 0.1338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.717489719390869,
      "rewards/margins": 2.717489719390869,
      "rewards/rejected": 0.0,
      "step": 1299
    },
    {
      "epoch": 7.262569832402234,
      "grad_norm": 1.9129830372425076,
      "learning_rate": 3.323950864210247e-06,
      "logits/chosen": -3.648099660873413,
      "logits/rejected": -3.63379168510437,
      "logps/chosen": -14.29564380645752,
      "logps/rejected": -39.975807189941406,
      "loss": 0.0887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3505196571350098,
      "rewards/margins": 3.3505196571350098,
      "rewards/rejected": 0.0,
      "step": 1300
    },
    {
      "epoch": 7.262569832402234,
      "eval_logits/chosen": -3.4270756244659424,
      "eval_logits/rejected": -3.537365436553955,
      "eval_logps/chosen": -35.47867965698242,
      "eval_logps/rejected": -54.09196090698242,
      "eval_loss": 1.359712839126587,
      "eval_rewards/accuracies": 0.375,
      "eval_rewards/chosen": -0.540864109992981,
      "eval_rewards/margins": -0.540864109992981,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5213,
      "eval_samples_per_second": 12.147,
      "eval_steps_per_second": 0.392,
      "step": 1300
    },
    {
      "epoch": 7.268156424581005,
      "grad_norm": 2.6038022163366965,
      "learning_rate": 3.3210728136581753e-06,
      "logits/chosen": -3.5671274662017822,
      "logits/rejected": -3.500967025756836,
      "logps/chosen": -12.686714172363281,
      "logps/rejected": -61.52901077270508,
      "loss": 0.1041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6852035522460938,
      "rewards/margins": 3.6852035522460938,
      "rewards/rejected": 0.0,
      "step": 1301
    },
    {
      "epoch": 7.273743016759776,
      "grad_norm": 4.542708789890042,
      "learning_rate": 3.318193542845678e-06,
      "logits/chosen": -3.7196500301361084,
      "logits/rejected": -3.5919597148895264,
      "logps/chosen": -15.581475257873535,
      "logps/rejected": -47.76002502441406,
      "loss": 0.1379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.201329231262207,
      "rewards/margins": 3.201329231262207,
      "rewards/rejected": 0.0,
      "step": 1302
    },
    {
      "epoch": 7.279329608938547,
      "grad_norm": 5.834513540144929,
      "learning_rate": 3.3153130560518636e-06,
      "logits/chosen": -3.4174282550811768,
      "logits/rejected": -3.4716150760650635,
      "logps/chosen": -3.2023725509643555,
      "logps/rejected": -118.94593048095703,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9737348556518555,
      "rewards/margins": 2.9737348556518555,
      "rewards/rejected": 0.0,
      "step": 1303
    },
    {
      "epoch": 7.284916201117318,
      "grad_norm": 3.021385647937655,
      "learning_rate": 3.3124313575576488e-06,
      "logits/chosen": -3.5643279552459717,
      "logits/rejected": -3.5438339710235596,
      "logps/chosen": -4.0167412757873535,
      "logps/rejected": -68.73052978515625,
      "loss": 0.1491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0249686241149902,
      "rewards/margins": 3.0249686241149902,
      "rewards/rejected": 0.0,
      "step": 1304
    },
    {
      "epoch": 7.290502793296089,
      "grad_norm": 1.9920225850606932,
      "learning_rate": 3.3095484516457503e-06,
      "logits/chosen": -3.504487991333008,
      "logits/rejected": -3.5063960552215576,
      "logps/chosen": -4.034384727478027,
      "logps/rejected": -71.32831573486328,
      "loss": 0.1199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.706968307495117,
      "rewards/margins": 2.706968307495117,
      "rewards/rejected": 0.0,
      "step": 1305
    },
    {
      "epoch": 7.29608938547486,
      "grad_norm": 6.388765586725635,
      "learning_rate": 3.3066643426006807e-06,
      "logits/chosen": -3.5584261417388916,
      "logits/rejected": -3.370828866958618,
      "logps/chosen": -7.73097562789917,
      "logps/rejected": -32.45014190673828,
      "loss": 0.1846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6032400131225586,
      "rewards/margins": 2.6032400131225586,
      "rewards/rejected": 0.0,
      "step": 1306
    },
    {
      "epoch": 7.301675977653631,
      "grad_norm": 2.1529082871892617,
      "learning_rate": 3.3037790347087383e-06,
      "logits/chosen": -3.5948336124420166,
      "logits/rejected": -3.5868046283721924,
      "logps/chosen": -7.112545967102051,
      "logps/rejected": -43.93928146362305,
      "loss": 0.1118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6886940002441406,
      "rewards/margins": 2.6886940002441406,
      "rewards/rejected": 0.0,
      "step": 1307
    },
    {
      "epoch": 7.307262569832402,
      "grad_norm": 6.983737288368889,
      "learning_rate": 3.3008925322580054e-06,
      "logits/chosen": -3.562041997909546,
      "logits/rejected": -3.561347246170044,
      "logps/chosen": -2.5870792865753174,
      "logps/rejected": -58.47859191894531,
      "loss": 0.1899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5761067867279053,
      "rewards/margins": 1.5761067867279053,
      "rewards/rejected": 0.0,
      "step": 1308
    },
    {
      "epoch": 7.312849162011173,
      "grad_norm": 5.1387644431747885,
      "learning_rate": 3.298004839538338e-06,
      "logits/chosen": -3.4319448471069336,
      "logits/rejected": -3.5235471725463867,
      "logps/chosen": -12.227655410766602,
      "logps/rejected": -69.55813598632812,
      "loss": 0.1251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.232229471206665,
      "rewards/margins": 2.232229471206665,
      "rewards/rejected": 0.0,
      "step": 1309
    },
    {
      "epoch": 7.318435754189944,
      "grad_norm": 2.5869353176875225,
      "learning_rate": 3.2951159608413624e-06,
      "logits/chosen": -3.5709710121154785,
      "logits/rejected": -3.7116217613220215,
      "logps/chosen": -7.830829620361328,
      "logps/rejected": -63.003623962402344,
      "loss": 0.1055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.455850124359131,
      "rewards/margins": 3.455850124359131,
      "rewards/rejected": 0.0,
      "step": 1310
    },
    {
      "epoch": 7.324022346368715,
      "grad_norm": 1.9008870493968077,
      "learning_rate": 3.292225900460467e-06,
      "logits/chosen": -3.424057960510254,
      "logits/rejected": -3.4913198947906494,
      "logps/chosen": -1.0120909214019775,
      "logps/rejected": -103.53536987304688,
      "loss": 0.1079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.708504319190979,
      "rewards/margins": 1.708504319190979,
      "rewards/rejected": 0.0,
      "step": 1311
    },
    {
      "epoch": 7.329608938547486,
      "grad_norm": 2.668498913302955,
      "learning_rate": 3.2893346626907945e-06,
      "logits/chosen": -3.786965847015381,
      "logits/rejected": -3.7534713745117188,
      "logps/chosen": -10.34956169128418,
      "logps/rejected": -56.24884033203125,
      "loss": 0.131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.336085081100464,
      "rewards/margins": 3.336085081100464,
      "rewards/rejected": 0.0,
      "step": 1312
    },
    {
      "epoch": 7.335195530726257,
      "grad_norm": 2.7663456885729323,
      "learning_rate": 3.2864422518292415e-06,
      "logits/chosen": -2.905658006668091,
      "logits/rejected": -2.9094552993774414,
      "logps/chosen": -60.27330780029297,
      "logps/rejected": -56.19474792480469,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.232359886169434,
      "rewards/margins": 4.232359886169434,
      "rewards/rejected": 0.0,
      "step": 1313
    },
    {
      "epoch": 7.340782122905028,
      "grad_norm": 1.6766870063363537,
      "learning_rate": 3.2835486721744447e-06,
      "logits/chosen": -3.280801296234131,
      "logits/rejected": -3.4090359210968018,
      "logps/chosen": -2.473923444747925,
      "logps/rejected": -36.2934684753418,
      "loss": 0.1143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5461585521698,
      "rewards/margins": 2.5461585521698,
      "rewards/rejected": 0.0,
      "step": 1314
    },
    {
      "epoch": 7.346368715083799,
      "grad_norm": 1.7864414481476465,
      "learning_rate": 3.280653928026778e-06,
      "logits/chosen": -3.785238742828369,
      "logits/rejected": -3.753920078277588,
      "logps/chosen": -5.65324068069458,
      "logps/rejected": -69.96549987792969,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5529117584228516,
      "rewards/margins": 3.5529117584228516,
      "rewards/rejected": 0.0,
      "step": 1315
    },
    {
      "epoch": 7.351955307262569,
      "grad_norm": 5.18503145355805,
      "learning_rate": 3.2777580236883477e-06,
      "logits/chosen": -3.5962677001953125,
      "logits/rejected": -3.7623705863952637,
      "logps/chosen": -1.867814064025879,
      "logps/rejected": -53.98804473876953,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.720536231994629,
      "rewards/margins": 2.720536231994629,
      "rewards/rejected": 0.0,
      "step": 1316
    },
    {
      "epoch": 7.35754189944134,
      "grad_norm": 2.8268635524636045,
      "learning_rate": 3.274860963462983e-06,
      "logits/chosen": -3.8937861919403076,
      "logits/rejected": -4.019940376281738,
      "logps/chosen": -13.062024116516113,
      "logps/rejected": -64.17465209960938,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7857155799865723,
      "rewards/margins": 2.7857155799865723,
      "rewards/rejected": 0.0,
      "step": 1317
    },
    {
      "epoch": 7.363128491620111,
      "grad_norm": 1.8942124981887123,
      "learning_rate": 3.2719627516562314e-06,
      "logits/chosen": -3.7813923358917236,
      "logits/rejected": -3.7676987648010254,
      "logps/chosen": -3.9013547897338867,
      "logps/rejected": -35.44066619873047,
      "loss": 0.1006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.432565212249756,
      "rewards/margins": 2.432565212249756,
      "rewards/rejected": 0.0,
      "step": 1318
    },
    {
      "epoch": 7.368715083798882,
      "grad_norm": 4.755461552801464,
      "learning_rate": 3.269063392575352e-06,
      "logits/chosen": -3.642350912094116,
      "logits/rejected": -3.6819519996643066,
      "logps/chosen": -6.701589584350586,
      "logps/rejected": -40.62275695800781,
      "loss": 0.1509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.078422546386719,
      "rewards/margins": 4.078422546386719,
      "rewards/rejected": 0.0,
      "step": 1319
    },
    {
      "epoch": 7.374301675977653,
      "grad_norm": 1.3579079463145505,
      "learning_rate": 3.2661628905293084e-06,
      "logits/chosen": -3.7299013137817383,
      "logits/rejected": -3.646402597427368,
      "logps/chosen": -2.241192102432251,
      "logps/rejected": -69.798583984375,
      "loss": 0.0918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.033740520477295,
      "rewards/margins": 3.033740520477295,
      "rewards/rejected": 0.0,
      "step": 1320
    },
    {
      "epoch": 7.374301675977653,
      "eval_logits/chosen": -3.412235736846924,
      "eval_logits/rejected": -3.5160396099090576,
      "eval_logps/chosen": -35.36465072631836,
      "eval_logps/rejected": -52.091651916503906,
      "eval_loss": 1.3003743886947632,
      "eval_rewards/accuracies": 0.32499998807907104,
      "eval_rewards/chosen": -0.5294608473777771,
      "eval_rewards/margins": -0.5294608473777771,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.4706,
      "eval_samples_per_second": 12.171,
      "eval_steps_per_second": 0.393,
      "step": 1320
    },
    {
      "epoch": 7.379888268156424,
      "grad_norm": 3.083268212362395,
      "learning_rate": 3.263261249828763e-06,
      "logits/chosen": -3.5714473724365234,
      "logits/rejected": -3.5772688388824463,
      "logps/chosen": -14.413954734802246,
      "logps/rejected": -72.85314178466797,
      "loss": 0.1036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6986260414123535,
      "rewards/margins": 3.6986260414123535,
      "rewards/rejected": 0.0,
      "step": 1321
    },
    {
      "epoch": 7.385474860335195,
      "grad_norm": 3.466443557719402,
      "learning_rate": 3.2603584747860716e-06,
      "logits/chosen": -3.317281723022461,
      "logits/rejected": -3.5108346939086914,
      "logps/chosen": -5.904242515563965,
      "logps/rejected": -54.712181091308594,
      "loss": 0.1568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.657688617706299,
      "rewards/margins": 3.657688617706299,
      "rewards/rejected": 0.0,
      "step": 1322
    },
    {
      "epoch": 7.391061452513966,
      "grad_norm": 2.421932112521302,
      "learning_rate": 3.2574545697152744e-06,
      "logits/chosen": -3.0831234455108643,
      "logits/rejected": -3.198187828063965,
      "logps/chosen": -6.109200477600098,
      "logps/rejected": -46.87971878051758,
      "loss": 0.1171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2298471927642822,
      "rewards/margins": 3.2298471927642822,
      "rewards/rejected": 0.0,
      "step": 1323
    },
    {
      "epoch": 7.396648044692737,
      "grad_norm": 3.9328431194172593,
      "learning_rate": 3.25454953893209e-06,
      "logits/chosen": -3.474221706390381,
      "logits/rejected": -3.7951292991638184,
      "logps/chosen": -8.363005638122559,
      "logps/rejected": -64.22753143310547,
      "loss": 0.1262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.8575968742370605,
      "rewards/margins": 1.8575968742370605,
      "rewards/rejected": 0.0,
      "step": 1324
    },
    {
      "epoch": 7.402234636871508,
      "grad_norm": 2.795137874400027,
      "learning_rate": 3.2516433867539143e-06,
      "logits/chosen": -3.678123712539673,
      "logits/rejected": -3.5199875831604004,
      "logps/chosen": -7.848306179046631,
      "logps/rejected": -54.861106872558594,
      "loss": 0.1328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4336822032928467,
      "rewards/margins": 2.4336822032928467,
      "rewards/rejected": 0.0,
      "step": 1325
    },
    {
      "epoch": 7.407821229050279,
      "grad_norm": 19.46659898696701,
      "learning_rate": 3.2487361174998045e-06,
      "logits/chosen": -3.5750749111175537,
      "logits/rejected": -3.440142869949341,
      "logps/chosen": -7.835380554199219,
      "logps/rejected": -33.178253173828125,
      "loss": 0.1556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.883575916290283,
      "rewards/margins": 3.883575916290283,
      "rewards/rejected": 0.0,
      "step": 1326
    },
    {
      "epoch": 7.41340782122905,
      "grad_norm": 3.7358914751264876,
      "learning_rate": 3.2458277354904823e-06,
      "logits/chosen": -3.5492796897888184,
      "logits/rejected": -3.43564772605896,
      "logps/chosen": -1.5840706825256348,
      "logps/rejected": -44.022308349609375,
      "loss": 0.1073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.253330707550049,
      "rewards/margins": 2.253330707550049,
      "rewards/rejected": 0.0,
      "step": 1327
    },
    {
      "epoch": 7.418994413407821,
      "grad_norm": 8.403646644185233,
      "learning_rate": 3.2429182450483198e-06,
      "logits/chosen": -3.6578879356384277,
      "logits/rejected": -3.659271478652954,
      "logps/chosen": -3.5212926864624023,
      "logps/rejected": -55.934326171875,
      "loss": 0.1203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4843029975891113,
      "rewards/margins": 2.4843029975891113,
      "rewards/rejected": 0.0,
      "step": 1328
    },
    {
      "epoch": 7.424581005586592,
      "grad_norm": 2.1659399692366184,
      "learning_rate": 3.2400076504973395e-06,
      "logits/chosen": -2.8627889156341553,
      "logits/rejected": -2.8859362602233887,
      "logps/chosen": -9.25882339477539,
      "logps/rejected": -56.77227783203125,
      "loss": 0.0921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4028468132019043,
      "rewards/margins": 3.4028468132019043,
      "rewards/rejected": 0.0,
      "step": 1329
    },
    {
      "epoch": 7.430167597765363,
      "grad_norm": 2.102797390979227,
      "learning_rate": 3.237095956163202e-06,
      "logits/chosen": -3.3749005794525146,
      "logits/rejected": -3.4110710620880127,
      "logps/chosen": -1.4855245351791382,
      "logps/rejected": -51.24951934814453,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.247781991958618,
      "rewards/margins": 2.247781991958618,
      "rewards/rejected": 0.0,
      "step": 1330
    },
    {
      "epoch": 7.435754189944134,
      "grad_norm": 1.9115081861132446,
      "learning_rate": 3.234183166373203e-06,
      "logits/chosen": -3.480276346206665,
      "logits/rejected": -3.4345109462738037,
      "logps/chosen": -2.935847520828247,
      "logps/rejected": -52.25277328491211,
      "loss": 0.0867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.362515449523926,
      "rewards/margins": 2.362515449523926,
      "rewards/rejected": 0.0,
      "step": 1331
    },
    {
      "epoch": 7.441340782122905,
      "grad_norm": 1.6950362172099829,
      "learning_rate": 3.2312692854562693e-06,
      "logits/chosen": -3.2999985218048096,
      "logits/rejected": -3.493135929107666,
      "logps/chosen": -2.9454398155212402,
      "logps/rejected": -63.926780700683594,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.134915828704834,
      "rewards/margins": 3.134915828704834,
      "rewards/rejected": 0.0,
      "step": 1332
    },
    {
      "epoch": 7.446927374301676,
      "grad_norm": 2.266222600367518,
      "learning_rate": 3.228354317742945e-06,
      "logits/chosen": -3.515037775039673,
      "logits/rejected": -3.4014949798583984,
      "logps/chosen": -7.704611301422119,
      "logps/rejected": -42.15219497680664,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4340057373046875,
      "rewards/margins": 2.4340057373046875,
      "rewards/rejected": 0.0,
      "step": 1333
    },
    {
      "epoch": 7.452513966480447,
      "grad_norm": 24.32467813213131,
      "learning_rate": 3.225438267565391e-06,
      "logits/chosen": -3.476290464401245,
      "logits/rejected": -3.6402134895324707,
      "logps/chosen": -0.9417152404785156,
      "logps/rejected": -66.51226806640625,
      "loss": 0.2159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.071113109588623,
      "rewards/margins": 2.071113109588623,
      "rewards/rejected": 0.0,
      "step": 1334
    },
    {
      "epoch": 7.4581005586592175,
      "grad_norm": 4.823796629763401,
      "learning_rate": 3.222521139257379e-06,
      "logits/chosen": -3.3456265926361084,
      "logits/rejected": -3.2495291233062744,
      "logps/chosen": -6.194191932678223,
      "logps/rejected": -67.00006866455078,
      "loss": 0.1096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2671353816986084,
      "rewards/margins": 2.2671353816986084,
      "rewards/rejected": 0.0,
      "step": 1335
    },
    {
      "epoch": 7.4636871508379885,
      "grad_norm": 1.7427669647435053,
      "learning_rate": 3.21960293715428e-06,
      "logits/chosen": -3.5218353271484375,
      "logits/rejected": -3.622636079788208,
      "logps/chosen": -3.350526809692383,
      "logps/rejected": -50.82618713378906,
      "loss": 0.0989,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6574349403381348,
      "rewards/margins": 2.6574349403381348,
      "rewards/rejected": 0.0,
      "step": 1336
    },
    {
      "epoch": 7.4692737430167595,
      "grad_norm": 5.0278446600895865,
      "learning_rate": 3.216683665593061e-06,
      "logits/chosen": -3.8524169921875,
      "logits/rejected": -3.8543829917907715,
      "logps/chosen": -1.4711885452270508,
      "logps/rejected": -46.61532211303711,
      "loss": 0.1394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3168816566467285,
      "rewards/margins": 2.3168816566467285,
      "rewards/rejected": 0.0,
      "step": 1337
    },
    {
      "epoch": 7.4748603351955305,
      "grad_norm": 2.9176400142913668,
      "learning_rate": 3.2137633289122816e-06,
      "logits/chosen": -3.0433549880981445,
      "logits/rejected": -3.1683080196380615,
      "logps/chosen": -7.32882022857666,
      "logps/rejected": -60.660335540771484,
      "loss": 0.1312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8179211616516113,
      "rewards/margins": 2.8179211616516113,
      "rewards/rejected": 0.0,
      "step": 1338
    },
    {
      "epoch": 7.4804469273743015,
      "grad_norm": 1.5168588099727152,
      "learning_rate": 3.210841931452082e-06,
      "logits/chosen": -3.7370526790618896,
      "logits/rejected": -3.8117258548736572,
      "logps/chosen": -7.37396240234375,
      "logps/rejected": -62.42028045654297,
      "loss": 0.1316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7181217670440674,
      "rewards/margins": 3.7181217670440674,
      "rewards/rejected": 0.0,
      "step": 1339
    },
    {
      "epoch": 7.4860335195530725,
      "grad_norm": 4.747575180710352,
      "learning_rate": 3.2079194775541764e-06,
      "logits/chosen": -3.752203941345215,
      "logits/rejected": -3.6974785327911377,
      "logps/chosen": -2.8657240867614746,
      "logps/rejected": -58.952415466308594,
      "loss": 0.1644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.605613946914673,
      "rewards/margins": 2.605613946914673,
      "rewards/rejected": 0.0,
      "step": 1340
    },
    {
      "epoch": 7.4860335195530725,
      "eval_logits/chosen": -3.4443905353546143,
      "eval_logits/rejected": -3.5561306476593018,
      "eval_logps/chosen": -35.925209045410156,
      "eval_logps/rejected": -52.63322830200195,
      "eval_loss": 1.3549634218215942,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.5855164527893066,
      "eval_rewards/margins": -0.5855164527893066,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.561,
      "eval_samples_per_second": 12.128,
      "eval_steps_per_second": 0.391,
      "step": 1340
    },
    {
      "epoch": 7.4916201117318435,
      "grad_norm": 3.215069700465506,
      "learning_rate": 3.2049959715618544e-06,
      "logits/chosen": -3.785815715789795,
      "logits/rejected": -3.8046321868896484,
      "logps/chosen": -7.688183784484863,
      "logps/rejected": -62.23958206176758,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7231624126434326,
      "rewards/margins": 3.7231624126434326,
      "rewards/rejected": 0.0,
      "step": 1341
    },
    {
      "epoch": 7.4972067039106145,
      "grad_norm": 5.395156209507993,
      "learning_rate": 3.2020714178199658e-06,
      "logits/chosen": -3.452718496322632,
      "logits/rejected": -3.4678802490234375,
      "logps/chosen": -28.704601287841797,
      "logps/rejected": -54.036781311035156,
      "loss": 0.1594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.193969488143921,
      "rewards/margins": 3.193969488143921,
      "rewards/rejected": 0.0,
      "step": 1342
    },
    {
      "epoch": 7.5027932960893855,
      "grad_norm": 2.7690991491285364,
      "learning_rate": 3.199145820674918e-06,
      "logits/chosen": -3.4777438640594482,
      "logits/rejected": -3.1783926486968994,
      "logps/chosen": -2.8784759044647217,
      "logps/rejected": -61.61484146118164,
      "loss": 0.1265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.324458122253418,
      "rewards/margins": 2.324458122253418,
      "rewards/rejected": 0.0,
      "step": 1343
    },
    {
      "epoch": 7.5083798882681565,
      "grad_norm": 4.900231401747802,
      "learning_rate": 3.1962191844746693e-06,
      "logits/chosen": -3.616570472717285,
      "logits/rejected": -3.644756555557251,
      "logps/chosen": -19.081872940063477,
      "logps/rejected": -53.490169525146484,
      "loss": 0.1673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3759145736694336,
      "rewards/margins": 2.3759145736694336,
      "rewards/rejected": 0.0,
      "step": 1344
    },
    {
      "epoch": 7.5139664804469275,
      "grad_norm": 2.44710154672917,
      "learning_rate": 3.1932915135687238e-06,
      "logits/chosen": -3.604095220565796,
      "logits/rejected": -3.3629226684570312,
      "logps/chosen": -5.479571342468262,
      "logps/rejected": -83.82327270507812,
      "loss": 0.1304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7140591144561768,
      "rewards/margins": 2.7140591144561768,
      "rewards/rejected": 0.0,
      "step": 1345
    },
    {
      "epoch": 7.5195530726256985,
      "grad_norm": 3.55320312271632,
      "learning_rate": 3.19036281230812e-06,
      "logits/chosen": -3.501920461654663,
      "logits/rejected": -3.680137872695923,
      "logps/chosen": -7.02781343460083,
      "logps/rejected": -71.39877319335938,
      "loss": 0.1667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.661180257797241,
      "rewards/margins": 2.661180257797241,
      "rewards/rejected": 0.0,
      "step": 1346
    },
    {
      "epoch": 7.5251396648044695,
      "grad_norm": 2.9725081511421445,
      "learning_rate": 3.1874330850454296e-06,
      "logits/chosen": -3.6799070835113525,
      "logits/rejected": -3.4960525035858154,
      "logps/chosen": -4.058706283569336,
      "logps/rejected": -62.46827697753906,
      "loss": 0.1507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9179224967956543,
      "rewards/margins": 2.9179224967956543,
      "rewards/rejected": 0.0,
      "step": 1347
    },
    {
      "epoch": 7.5307262569832405,
      "grad_norm": 3.0130065941336124,
      "learning_rate": 3.1845023361347504e-06,
      "logits/chosen": -3.593304395675659,
      "logits/rejected": -3.5922703742980957,
      "logps/chosen": -8.317378997802734,
      "logps/rejected": -69.3636245727539,
      "loss": 0.1392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0079398155212402,
      "rewards/margins": 3.0079398155212402,
      "rewards/rejected": 0.0,
      "step": 1348
    },
    {
      "epoch": 7.5363128491620115,
      "grad_norm": 7.452007550149491,
      "learning_rate": 3.181570569931697e-06,
      "logits/chosen": -3.5826056003570557,
      "logits/rejected": -3.8414840698242188,
      "logps/chosen": -1.6331062316894531,
      "logps/rejected": -63.48921203613281,
      "loss": 0.147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3719987869262695,
      "rewards/margins": 2.3719987869262695,
      "rewards/rejected": 0.0,
      "step": 1349
    },
    {
      "epoch": 7.5418994413407825,
      "grad_norm": 2.5569341988501852,
      "learning_rate": 3.178637790793394e-06,
      "logits/chosen": -3.6372716426849365,
      "logits/rejected": -3.68072247505188,
      "logps/chosen": -3.1798272132873535,
      "logps/rejected": -51.362545013427734,
      "loss": 0.1038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8970563411712646,
      "rewards/margins": 2.8970563411712646,
      "rewards/rejected": 0.0,
      "step": 1350
    },
    {
      "epoch": 7.547486033519553,
      "grad_norm": 4.03850359669383,
      "learning_rate": 3.1757040030784765e-06,
      "logits/chosen": -3.4484925270080566,
      "logits/rejected": -3.6352837085723877,
      "logps/chosen": -3.53483247756958,
      "logps/rejected": -78.0581283569336,
      "loss": 0.1256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.029569387435913,
      "rewards/margins": 2.029569387435913,
      "rewards/rejected": 0.0,
      "step": 1351
    },
    {
      "epoch": 7.553072625698324,
      "grad_norm": 1.8224792072169842,
      "learning_rate": 3.172769211147074e-06,
      "logits/chosen": -3.478386640548706,
      "logits/rejected": -3.2914645671844482,
      "logps/chosen": -7.935140609741211,
      "logps/rejected": -32.99390411376953,
      "loss": 0.1023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.001741886138916,
      "rewards/margins": 3.001741886138916,
      "rewards/rejected": 0.0,
      "step": 1352
    },
    {
      "epoch": 7.558659217877095,
      "grad_norm": 2.885176432329783,
      "learning_rate": 3.1698334193608093e-06,
      "logits/chosen": -3.6538565158843994,
      "logits/rejected": -3.5279300212860107,
      "logps/chosen": -4.185461521148682,
      "logps/rejected": -59.39427947998047,
      "loss": 0.1717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7091779708862305,
      "rewards/margins": 2.7091779708862305,
      "rewards/rejected": 0.0,
      "step": 1353
    },
    {
      "epoch": 7.564245810055866,
      "grad_norm": 2.0535783875907567,
      "learning_rate": 3.166896632082793e-06,
      "logits/chosen": -3.5203163623809814,
      "logits/rejected": -3.3737142086029053,
      "logps/chosen": -2.6565892696380615,
      "logps/rejected": -69.26266479492188,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2165238857269287,
      "rewards/margins": 2.2165238857269287,
      "rewards/rejected": 0.0,
      "step": 1354
    },
    {
      "epoch": 7.569832402234637,
      "grad_norm": 2.6883245959989575,
      "learning_rate": 3.163958853677614e-06,
      "logits/chosen": -3.7188827991485596,
      "logits/rejected": -3.4897189140319824,
      "logps/chosen": -2.7787325382232666,
      "logps/rejected": -80.61790466308594,
      "loss": 0.1353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.28757905960083,
      "rewards/margins": 2.28757905960083,
      "rewards/rejected": 0.0,
      "step": 1355
    },
    {
      "epoch": 7.575418994413408,
      "grad_norm": 1.502082662290015,
      "learning_rate": 3.1610200885113334e-06,
      "logits/chosen": -3.7268221378326416,
      "logits/rejected": -3.5969953536987305,
      "logps/chosen": -8.095895767211914,
      "logps/rejected": -51.88740158081055,
      "loss": 0.1142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.357868194580078,
      "rewards/margins": 3.357868194580078,
      "rewards/rejected": 0.0,
      "step": 1356
    },
    {
      "epoch": 7.581005586592179,
      "grad_norm": 2.1821026048360843,
      "learning_rate": 3.15808034095148e-06,
      "logits/chosen": -3.6329166889190674,
      "logits/rejected": -3.6366608142852783,
      "logps/chosen": -4.38239860534668,
      "logps/rejected": -65.6606674194336,
      "loss": 0.1143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2038028240203857,
      "rewards/margins": 2.2038028240203857,
      "rewards/rejected": 0.0,
      "step": 1357
    },
    {
      "epoch": 7.58659217877095,
      "grad_norm": 2.4336679842556763,
      "learning_rate": 3.155139615367042e-06,
      "logits/chosen": -3.534165382385254,
      "logits/rejected": -3.3962552547454834,
      "logps/chosen": -2.97927188873291,
      "logps/rejected": -52.38287353515625,
      "loss": 0.1167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.593069553375244,
      "rewards/margins": 2.593069553375244,
      "rewards/rejected": 0.0,
      "step": 1358
    },
    {
      "epoch": 7.592178770949721,
      "grad_norm": 1.420333908559547,
      "learning_rate": 3.152197916128463e-06,
      "logits/chosen": -3.4254343509674072,
      "logits/rejected": -3.3874571323394775,
      "logps/chosen": -2.719111442565918,
      "logps/rejected": -76.45808410644531,
      "loss": 0.1207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5983614921569824,
      "rewards/margins": 2.5983614921569824,
      "rewards/rejected": 0.0,
      "step": 1359
    },
    {
      "epoch": 7.597765363128492,
      "grad_norm": 4.756761644121932,
      "learning_rate": 3.1492552476076293e-06,
      "logits/chosen": -3.8204615116119385,
      "logits/rejected": -3.777147054672241,
      "logps/chosen": -4.0819196701049805,
      "logps/rejected": -27.354595184326172,
      "loss": 0.141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6972618103027344,
      "rewards/margins": 2.6972618103027344,
      "rewards/rejected": 0.0,
      "step": 1360
    },
    {
      "epoch": 7.597765363128492,
      "eval_logits/chosen": -3.409665584564209,
      "eval_logits/rejected": -3.519807815551758,
      "eval_logps/chosen": -34.8736457824707,
      "eval_logps/rejected": -51.911598205566406,
      "eval_loss": 1.2767908573150635,
      "eval_rewards/accuracies": 0.32499998807907104,
      "eval_rewards/chosen": -0.48036032915115356,
      "eval_rewards/margins": -0.48036032915115356,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5031,
      "eval_samples_per_second": 12.155,
      "eval_steps_per_second": 0.392,
      "step": 1360
    },
    {
      "epoch": 7.603351955307263,
      "grad_norm": 2.463615593994579,
      "learning_rate": 3.1463116141778717e-06,
      "logits/chosen": -3.9249067306518555,
      "logits/rejected": -3.905846357345581,
      "logps/chosen": -11.578957557678223,
      "logps/rejected": -38.343116760253906,
      "loss": 0.1252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.981604814529419,
      "rewards/margins": 3.981604814529419,
      "rewards/rejected": 0.0,
      "step": 1361
    },
    {
      "epoch": 7.608938547486034,
      "grad_norm": 3.0697918985831834,
      "learning_rate": 3.1433670202139542e-06,
      "logits/chosen": -3.640622854232788,
      "logits/rejected": -3.6219167709350586,
      "logps/chosen": -4.74077033996582,
      "logps/rejected": -61.617889404296875,
      "loss": 0.1144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1759181022644043,
      "rewards/margins": 3.1759181022644043,
      "rewards/rejected": 0.0,
      "step": 1362
    },
    {
      "epoch": 7.614525139664805,
      "grad_norm": 1.962954237542897,
      "learning_rate": 3.1404214700920668e-06,
      "logits/chosen": -3.3541553020477295,
      "logits/rejected": -3.4311394691467285,
      "logps/chosen": -2.2466211318969727,
      "logps/rejected": -58.777069091796875,
      "loss": 0.1026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.025327444076538,
      "rewards/margins": 3.025327444076538,
      "rewards/rejected": 0.0,
      "step": 1363
    },
    {
      "epoch": 7.620111731843576,
      "grad_norm": 5.1430318198858975,
      "learning_rate": 3.137474968189822e-06,
      "logits/chosen": -3.207082986831665,
      "logits/rejected": -3.2589504718780518,
      "logps/chosen": -15.36477279663086,
      "logps/rejected": -48.52099609375,
      "loss": 0.1847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.777343511581421,
      "rewards/margins": 2.777343511581421,
      "rewards/rejected": 0.0,
      "step": 1364
    },
    {
      "epoch": 7.625698324022347,
      "grad_norm": 4.135243744087232,
      "learning_rate": 3.1345275188862462e-06,
      "logits/chosen": -3.6232612133026123,
      "logits/rejected": -3.87027907371521,
      "logps/chosen": -0.6130459308624268,
      "logps/rejected": -62.01293182373047,
      "loss": 0.1327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0511629581451416,
      "rewards/margins": 2.0511629581451416,
      "rewards/rejected": 0.0,
      "step": 1365
    },
    {
      "epoch": 7.631284916201118,
      "grad_norm": 2.4218890380784672,
      "learning_rate": 3.131579126561774e-06,
      "logits/chosen": -3.552637815475464,
      "logits/rejected": -3.5700271129608154,
      "logps/chosen": -6.522797584533691,
      "logps/rejected": -73.9941177368164,
      "loss": 0.1243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1606764793395996,
      "rewards/margins": 3.1606764793395996,
      "rewards/rejected": 0.0,
      "step": 1366
    },
    {
      "epoch": 7.636871508379889,
      "grad_norm": 4.432651267972531,
      "learning_rate": 3.1286297955982426e-06,
      "logits/chosen": -3.9012386798858643,
      "logits/rejected": -3.902534246444702,
      "logps/chosen": -7.529688835144043,
      "logps/rejected": -49.77862548828125,
      "loss": 0.1477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.865600109100342,
      "rewards/margins": 2.865600109100342,
      "rewards/rejected": 0.0,
      "step": 1367
    },
    {
      "epoch": 7.64245810055866,
      "grad_norm": 2.7371134566199613,
      "learning_rate": 3.1256795303788813e-06,
      "logits/chosen": -3.371051073074341,
      "logits/rejected": -3.369716167449951,
      "logps/chosen": -2.244645595550537,
      "logps/rejected": -46.796531677246094,
      "loss": 0.1204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6337101459503174,
      "rewards/margins": 1.6337101459503174,
      "rewards/rejected": 0.0,
      "step": 1368
    },
    {
      "epoch": 7.648044692737431,
      "grad_norm": 2.421714615535267,
      "learning_rate": 3.1227283352883107e-06,
      "logits/chosen": -3.2259230613708496,
      "logits/rejected": -3.2365615367889404,
      "logps/chosen": -18.12116813659668,
      "logps/rejected": -59.30097198486328,
      "loss": 0.1331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0394697189331055,
      "rewards/margins": 3.0394697189331055,
      "rewards/rejected": 0.0,
      "step": 1369
    },
    {
      "epoch": 7.653631284916202,
      "grad_norm": 3.9097430681566796,
      "learning_rate": 3.1197762147125322e-06,
      "logits/chosen": -3.4634430408477783,
      "logits/rejected": -3.4725284576416016,
      "logps/chosen": -3.8994178771972656,
      "logps/rejected": -66.87500762939453,
      "loss": 0.1666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7446184158325195,
      "rewards/margins": 2.7446184158325195,
      "rewards/rejected": 0.0,
      "step": 1370
    },
    {
      "epoch": 7.659217877094972,
      "grad_norm": 6.060606224645243,
      "learning_rate": 3.1168231730389225e-06,
      "logits/chosen": -3.667689085006714,
      "logits/rejected": -3.6567580699920654,
      "logps/chosen": -3.604226589202881,
      "logps/rejected": -59.204551696777344,
      "loss": 0.1432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.295738935470581,
      "rewards/margins": 2.295738935470581,
      "rewards/rejected": 0.0,
      "step": 1371
    },
    {
      "epoch": 7.664804469273743,
      "grad_norm": 2.5709756443159475,
      "learning_rate": 3.1138692146562276e-06,
      "logits/chosen": -3.592604875564575,
      "logits/rejected": -3.414590835571289,
      "logps/chosen": -2.858914613723755,
      "logps/rejected": -52.36857986450195,
      "loss": 0.1273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.824383020401001,
      "rewards/margins": 1.824383020401001,
      "rewards/rejected": 0.0,
      "step": 1372
    },
    {
      "epoch": 7.670391061452514,
      "grad_norm": 2.5932635587137907,
      "learning_rate": 3.1109143439545547e-06,
      "logits/chosen": -3.56778621673584,
      "logits/rejected": -3.5810909271240234,
      "logps/chosen": -2.5128588676452637,
      "logps/rejected": -68.16189575195312,
      "loss": 0.1298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5411250591278076,
      "rewards/margins": 2.5411250591278076,
      "rewards/rejected": 0.0,
      "step": 1373
    },
    {
      "epoch": 7.675977653631285,
      "grad_norm": 2.5478922747203585,
      "learning_rate": 3.1079585653253696e-06,
      "logits/chosen": -3.455864429473877,
      "logits/rejected": -3.2851929664611816,
      "logps/chosen": -2.3699049949645996,
      "logps/rejected": -48.381317138671875,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4015491008758545,
      "rewards/margins": 2.4015491008758545,
      "rewards/rejected": 0.0,
      "step": 1374
    },
    {
      "epoch": 7.681564245810056,
      "grad_norm": 3.653921230890823,
      "learning_rate": 3.105001883161485e-06,
      "logits/chosen": -3.51157283782959,
      "logits/rejected": -3.388047218322754,
      "logps/chosen": -10.215753555297852,
      "logps/rejected": -41.251522064208984,
      "loss": 0.1479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.662996768951416,
      "rewards/margins": 3.662996768951416,
      "rewards/rejected": 0.0,
      "step": 1375
    },
    {
      "epoch": 7.687150837988827,
      "grad_norm": 2.56015076322343,
      "learning_rate": 3.102044301857056e-06,
      "logits/chosen": -3.539910316467285,
      "logits/rejected": -3.556859254837036,
      "logps/chosen": -3.0419719219207764,
      "logps/rejected": -58.7469482421875,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.30098557472229,
      "rewards/margins": 2.30098557472229,
      "rewards/rejected": 0.0,
      "step": 1376
    },
    {
      "epoch": 7.692737430167598,
      "grad_norm": 5.1393947869446075,
      "learning_rate": 3.099085825807577e-06,
      "logits/chosen": -3.26743483543396,
      "logits/rejected": -3.504014015197754,
      "logps/chosen": -2.5393426418304443,
      "logps/rejected": -68.99530029296875,
      "loss": 0.1334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4442813396453857,
      "rewards/margins": 1.4442813396453857,
      "rewards/rejected": 0.0,
      "step": 1377
    },
    {
      "epoch": 7.698324022346369,
      "grad_norm": 8.58281407999365,
      "learning_rate": 3.09612645940987e-06,
      "logits/chosen": -3.485020399093628,
      "logits/rejected": -3.417962074279785,
      "logps/chosen": -21.03693389892578,
      "logps/rejected": -51.497528076171875,
      "loss": 0.1348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.229602813720703,
      "rewards/margins": 4.229602813720703,
      "rewards/rejected": 0.0,
      "step": 1378
    },
    {
      "epoch": 7.70391061452514,
      "grad_norm": 3.92197636210839,
      "learning_rate": 3.09316620706208e-06,
      "logits/chosen": -2.8361122608184814,
      "logits/rejected": -2.8699021339416504,
      "logps/chosen": -5.829306602478027,
      "logps/rejected": -56.972618103027344,
      "loss": 0.114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8764588832855225,
      "rewards/margins": 2.8764588832855225,
      "rewards/rejected": 0.0,
      "step": 1379
    },
    {
      "epoch": 7.709497206703911,
      "grad_norm": 3.5119506881620572,
      "learning_rate": 3.0902050731636685e-06,
      "logits/chosen": -3.380638360977173,
      "logits/rejected": -3.081064462661743,
      "logps/chosen": -9.904090881347656,
      "logps/rejected": -58.33541488647461,
      "loss": 0.1011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.739204168319702,
      "rewards/margins": 2.739204168319702,
      "rewards/rejected": 0.0,
      "step": 1380
    },
    {
      "epoch": 7.709497206703911,
      "eval_logits/chosen": -3.3902347087860107,
      "eval_logits/rejected": -3.4941604137420654,
      "eval_logps/chosen": -34.373714447021484,
      "eval_logps/rejected": -53.45854568481445,
      "eval_loss": 1.3291808366775513,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.43036699295043945,
      "eval_rewards/margins": -0.43036699295043945,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.475,
      "eval_samples_per_second": 12.169,
      "eval_steps_per_second": 0.393,
      "step": 1380
    },
    {
      "epoch": 7.715083798882682,
      "grad_norm": 4.331985983662938,
      "learning_rate": 3.0872430621154087e-06,
      "logits/chosen": -3.6620662212371826,
      "logits/rejected": -3.6444718837738037,
      "logps/chosen": -2.4742214679718018,
      "logps/rejected": -51.723670959472656,
      "loss": 0.1583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6138008832931519,
      "rewards/margins": 1.6138008832931519,
      "rewards/rejected": 0.0,
      "step": 1381
    },
    {
      "epoch": 7.720670391061453,
      "grad_norm": 1.6349628978093282,
      "learning_rate": 3.084280178319376e-06,
      "logits/chosen": -3.6577446460723877,
      "logits/rejected": -3.894690990447998,
      "logps/chosen": -5.953662872314453,
      "logps/rejected": -56.611717224121094,
      "loss": 0.0927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2990598678588867,
      "rewards/margins": 3.2990598678588867,
      "rewards/rejected": 0.0,
      "step": 1382
    },
    {
      "epoch": 7.726256983240224,
      "grad_norm": 4.348648236423435,
      "learning_rate": 3.0813164261789443e-06,
      "logits/chosen": -3.465481758117676,
      "logits/rejected": -3.3303732872009277,
      "logps/chosen": -6.55466890335083,
      "logps/rejected": -35.517333984375,
      "loss": 0.1152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5683705806732178,
      "rewards/margins": 2.5683705806732178,
      "rewards/rejected": 0.0,
      "step": 1383
    },
    {
      "epoch": 7.731843575418995,
      "grad_norm": 1.7960571089801618,
      "learning_rate": 3.0783518100987763e-06,
      "logits/chosen": -3.529374599456787,
      "logits/rejected": -3.60518741607666,
      "logps/chosen": -2.0932393074035645,
      "logps/rejected": -47.64811706542969,
      "loss": 0.0958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7808260917663574,
      "rewards/margins": 2.7808260917663574,
      "rewards/rejected": 0.0,
      "step": 1384
    },
    {
      "epoch": 7.737430167597766,
      "grad_norm": 2.764928910064598,
      "learning_rate": 3.07538633448482e-06,
      "logits/chosen": -3.7431788444519043,
      "logits/rejected": -3.72646164894104,
      "logps/chosen": -9.642322540283203,
      "logps/rejected": -49.51025390625,
      "loss": 0.1326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7861838340759277,
      "rewards/margins": 3.7861838340759277,
      "rewards/rejected": 0.0,
      "step": 1385
    },
    {
      "epoch": 7.743016759776537,
      "grad_norm": 1.9178214191614542,
      "learning_rate": 3.072420003744299e-06,
      "logits/chosen": -4.070353984832764,
      "logits/rejected": -3.518778085708618,
      "logps/chosen": -2.2087819576263428,
      "logps/rejected": -43.407249450683594,
      "loss": 0.1188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3620052337646484,
      "rewards/margins": 2.3620052337646484,
      "rewards/rejected": 0.0,
      "step": 1386
    },
    {
      "epoch": 7.748603351955307,
      "grad_norm": 3.314724615321935,
      "learning_rate": 3.0694528222857107e-06,
      "logits/chosen": -3.546510696411133,
      "logits/rejected": -3.5180137157440186,
      "logps/chosen": -4.285896301269531,
      "logps/rejected": -86.68562316894531,
      "loss": 0.1273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9737682342529297,
      "rewards/margins": 2.9737682342529297,
      "rewards/rejected": 0.0,
      "step": 1387
    },
    {
      "epoch": 7.754189944134078,
      "grad_norm": 2.282045328795444,
      "learning_rate": 3.0664847945188148e-06,
      "logits/chosen": -3.680393934249878,
      "logits/rejected": -3.607264280319214,
      "logps/chosen": -1.7830851078033447,
      "logps/rejected": -41.605079650878906,
      "loss": 0.1106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8900634050369263,
      "rewards/margins": 1.8900634050369263,
      "rewards/rejected": 0.0,
      "step": 1388
    },
    {
      "epoch": 7.759776536312849,
      "grad_norm": 1.7697302145887677,
      "learning_rate": 3.063515924854628e-06,
      "logits/chosen": -3.6563711166381836,
      "logits/rejected": -3.5798933506011963,
      "logps/chosen": -7.2806196212768555,
      "logps/rejected": -69.15750122070312,
      "loss": 0.1175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.100311040878296,
      "rewards/margins": 3.100311040878296,
      "rewards/rejected": 0.0,
      "step": 1389
    },
    {
      "epoch": 7.76536312849162,
      "grad_norm": 2.138463127428812,
      "learning_rate": 3.060546217705421e-06,
      "logits/chosen": -3.6086127758026123,
      "logits/rejected": -3.6807634830474854,
      "logps/chosen": -1.9000324010849,
      "logps/rejected": -69.54060363769531,
      "loss": 0.13,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5105600357055664,
      "rewards/margins": 2.5105600357055664,
      "rewards/rejected": 0.0,
      "step": 1390
    },
    {
      "epoch": 7.770949720670391,
      "grad_norm": 3.528358493610092,
      "learning_rate": 3.0575756774847065e-06,
      "logits/chosen": -3.5012831687927246,
      "logits/rejected": -3.3500845432281494,
      "logps/chosen": -10.939798355102539,
      "logps/rejected": -36.076820373535156,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5201034545898438,
      "rewards/margins": 3.5201034545898438,
      "rewards/rejected": 0.0,
      "step": 1391
    },
    {
      "epoch": 7.776536312849162,
      "grad_norm": 2.218370893160874,
      "learning_rate": 3.0546043086072353e-06,
      "logits/chosen": -3.742645740509033,
      "logits/rejected": -3.6259357929229736,
      "logps/chosen": -2.6414127349853516,
      "logps/rejected": -93.8579330444336,
      "loss": 0.1332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.054394245147705,
      "rewards/margins": 2.054394245147705,
      "rewards/rejected": 0.0,
      "step": 1392
    },
    {
      "epoch": 7.782122905027933,
      "grad_norm": 1.4887905981463296,
      "learning_rate": 3.0516321154889932e-06,
      "logits/chosen": -3.496241331100464,
      "logits/rejected": -3.4997353553771973,
      "logps/chosen": -8.601003646850586,
      "logps/rejected": -54.27935028076172,
      "loss": 0.1205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0011746883392334,
      "rewards/margins": 3.0011746883392334,
      "rewards/rejected": 0.0,
      "step": 1393
    },
    {
      "epoch": 7.787709497206704,
      "grad_norm": 7.777164100758812,
      "learning_rate": 3.048659102547187e-06,
      "logits/chosen": -3.3679654598236084,
      "logits/rejected": -3.5198373794555664,
      "logps/chosen": -9.458548545837402,
      "logps/rejected": -83.47138977050781,
      "loss": 0.1229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6082558631896973,
      "rewards/margins": 2.6082558631896973,
      "rewards/rejected": 0.0,
      "step": 1394
    },
    {
      "epoch": 7.793296089385475,
      "grad_norm": 3.7901309793249567,
      "learning_rate": 3.045685274200243e-06,
      "logits/chosen": -3.4281082153320312,
      "logits/rejected": -3.102095127105713,
      "logps/chosen": -1.3251571655273438,
      "logps/rejected": -82.11453247070312,
      "loss": 0.121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0855770111083984,
      "rewards/margins": 2.0855770111083984,
      "rewards/rejected": 0.0,
      "step": 1395
    },
    {
      "epoch": 7.798882681564246,
      "grad_norm": 2.476356646181835,
      "learning_rate": 3.0427106348678004e-06,
      "logits/chosen": -3.716895818710327,
      "logits/rejected": -3.3840885162353516,
      "logps/chosen": -7.129241466522217,
      "logps/rejected": -37.8686637878418,
      "loss": 0.1262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.815098285675049,
      "rewards/margins": 2.815098285675049,
      "rewards/rejected": 0.0,
      "step": 1396
    },
    {
      "epoch": 7.804469273743017,
      "grad_norm": 3.222762556240401,
      "learning_rate": 3.039735188970703e-06,
      "logits/chosen": -3.4366586208343506,
      "logits/rejected": -3.4673283100128174,
      "logps/chosen": -2.503100633621216,
      "logps/rejected": -74.53016662597656,
      "loss": 0.1398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6988320350646973,
      "rewards/margins": 2.6988320350646973,
      "rewards/rejected": 0.0,
      "step": 1397
    },
    {
      "epoch": 7.810055865921788,
      "grad_norm": 1.1618949786792796,
      "learning_rate": 3.036758940930994e-06,
      "logits/chosen": -3.8279500007629395,
      "logits/rejected": -3.8289871215820312,
      "logps/chosen": -1.886917233467102,
      "logps/rejected": -38.7005500793457,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9489541053771973,
      "rewards/margins": 2.9489541053771973,
      "rewards/rejected": 0.0,
      "step": 1398
    },
    {
      "epoch": 7.815642458100559,
      "grad_norm": 5.267776581101705,
      "learning_rate": 3.0337818951719068e-06,
      "logits/chosen": -3.496551990509033,
      "logits/rejected": -3.5676047801971436,
      "logps/chosen": -4.322014808654785,
      "logps/rejected": -54.158390045166016,
      "loss": 0.1725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.727402687072754,
      "rewards/margins": 2.727402687072754,
      "rewards/rejected": 0.0,
      "step": 1399
    },
    {
      "epoch": 7.82122905027933,
      "grad_norm": 4.477179418627329,
      "learning_rate": 3.030804056117862e-06,
      "logits/chosen": -3.4560041427612305,
      "logits/rejected": -3.569385528564453,
      "logps/chosen": -1.1904453039169312,
      "logps/rejected": -75.06147766113281,
      "loss": 0.1391,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8219037055969238,
      "rewards/margins": 1.8219037055969238,
      "rewards/rejected": 0.0,
      "step": 1400
    },
    {
      "epoch": 7.82122905027933,
      "eval_logits/chosen": -3.3972725868225098,
      "eval_logits/rejected": -3.5094947814941406,
      "eval_logps/chosen": -35.928401947021484,
      "eval_logps/rejected": -54.28185272216797,
      "eval_loss": 1.4082993268966675,
      "eval_rewards/accuracies": 0.30000001192092896,
      "eval_rewards/chosen": -0.5858360528945923,
      "eval_rewards/margins": -0.5858360528945923,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5569,
      "eval_samples_per_second": 12.13,
      "eval_steps_per_second": 0.391,
      "step": 1400
    },
    {
      "epoch": 7.826815642458101,
      "grad_norm": 2.2911556744686052,
      "learning_rate": 3.0278254281944584e-06,
      "logits/chosen": -3.48480486869812,
      "logits/rejected": -3.4247400760650635,
      "logps/chosen": -2.6769797801971436,
      "logps/rejected": -43.408416748046875,
      "loss": 0.1068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9026929140090942,
      "rewards/margins": 1.9026929140090942,
      "rewards/rejected": 0.0,
      "step": 1401
    },
    {
      "epoch": 7.832402234636872,
      "grad_norm": 3.234901697713226,
      "learning_rate": 3.0248460158284677e-06,
      "logits/chosen": -3.5034048557281494,
      "logits/rejected": -3.4550492763519287,
      "logps/chosen": -3.36755108833313,
      "logps/rejected": -62.9892578125,
      "loss": 0.1121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8882899284362793,
      "rewards/margins": 2.8882899284362793,
      "rewards/rejected": 0.0,
      "step": 1402
    },
    {
      "epoch": 7.837988826815643,
      "grad_norm": 1.3735263279667314,
      "learning_rate": 3.021865823447828e-06,
      "logits/chosen": -3.3763656616210938,
      "logits/rejected": -3.233564853668213,
      "logps/chosen": -1.3390686511993408,
      "logps/rejected": -39.76877975463867,
      "loss": 0.1266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3653974533081055,
      "rewards/margins": 2.3653974533081055,
      "rewards/rejected": 0.0,
      "step": 1403
    },
    {
      "epoch": 7.843575418994414,
      "grad_norm": 4.628679176086199,
      "learning_rate": 3.018884855481636e-06,
      "logits/chosen": -3.4719772338867188,
      "logits/rejected": -3.5933260917663574,
      "logps/chosen": -3.76198410987854,
      "logps/rejected": -52.99223709106445,
      "loss": 0.1412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5160179138183594,
      "rewards/margins": 2.5160179138183594,
      "rewards/rejected": 0.0,
      "step": 1404
    },
    {
      "epoch": 7.849162011173185,
      "grad_norm": 3.3779483742598715,
      "learning_rate": 3.0159031163601386e-06,
      "logits/chosen": -3.1099605560302734,
      "logits/rejected": -3.063612699508667,
      "logps/chosen": -27.253095626831055,
      "logps/rejected": -36.2408561706543,
      "loss": 0.1134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.201067924499512,
      "rewards/margins": 4.201067924499512,
      "rewards/rejected": 0.0,
      "step": 1405
    },
    {
      "epoch": 7.854748603351956,
      "grad_norm": 3.4447253236569884,
      "learning_rate": 3.0129206105147347e-06,
      "logits/chosen": -3.4780075550079346,
      "logits/rejected": -3.6027719974517822,
      "logps/chosen": -6.0404510498046875,
      "logps/rejected": -69.64227294921875,
      "loss": 0.1057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9819236993789673,
      "rewards/margins": 1.9819236993789673,
      "rewards/rejected": 0.0,
      "step": 1406
    },
    {
      "epoch": 7.860335195530726,
      "grad_norm": 3.3425354752431464,
      "learning_rate": 3.009937342377957e-06,
      "logits/chosen": -3.454876184463501,
      "logits/rejected": -3.550920009613037,
      "logps/chosen": -3.3695409297943115,
      "logps/rejected": -125.97801208496094,
      "loss": 0.1551,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5013387203216553,
      "rewards/margins": 1.5013387203216553,
      "rewards/rejected": 0.0,
      "step": 1407
    },
    {
      "epoch": 7.865921787709497,
      "grad_norm": 4.177460827649314,
      "learning_rate": 3.0069533163834737e-06,
      "logits/chosen": -3.6601874828338623,
      "logits/rejected": -3.661719799041748,
      "logps/chosen": -3.2557311058044434,
      "logps/rejected": -65.666748046875,
      "loss": 0.1336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2359769344329834,
      "rewards/margins": 3.2359769344329834,
      "rewards/rejected": 0.0,
      "step": 1408
    },
    {
      "epoch": 7.871508379888268,
      "grad_norm": 1.287463732433173,
      "learning_rate": 3.0039685369660785e-06,
      "logits/chosen": -3.7420754432678223,
      "logits/rejected": -3.71686053276062,
      "logps/chosen": -1.5339550971984863,
      "logps/rejected": -39.578216552734375,
      "loss": 0.107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.629178285598755,
      "rewards/margins": 2.629178285598755,
      "rewards/rejected": 0.0,
      "step": 1409
    },
    {
      "epoch": 7.877094972067039,
      "grad_norm": 2.4214516771049905,
      "learning_rate": 3.000983008561686e-06,
      "logits/chosen": -3.5862627029418945,
      "logits/rejected": -3.5147924423217773,
      "logps/chosen": -1.6267634630203247,
      "logps/rejected": -45.14462661743164,
      "loss": 0.129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5251516103744507,
      "rewards/margins": 1.5251516103744507,
      "rewards/rejected": 0.0,
      "step": 1410
    },
    {
      "epoch": 7.88268156424581,
      "grad_norm": 2.645000860927049,
      "learning_rate": 2.9979967356073227e-06,
      "logits/chosen": -3.5036404132843018,
      "logits/rejected": -3.5593414306640625,
      "logps/chosen": -18.91761589050293,
      "logps/rejected": -56.36187744140625,
      "loss": 0.1564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.567337989807129,
      "rewards/margins": 3.567337989807129,
      "rewards/rejected": 0.0,
      "step": 1411
    },
    {
      "epoch": 7.888268156424581,
      "grad_norm": 4.7414266909138325,
      "learning_rate": 2.9950097225411207e-06,
      "logits/chosen": -3.191188097000122,
      "logits/rejected": -3.529094934463501,
      "logps/chosen": -5.066041469573975,
      "logps/rejected": -49.09911346435547,
      "loss": 0.1444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8874638080596924,
      "rewards/margins": 2.8874638080596924,
      "rewards/rejected": 0.0,
      "step": 1412
    },
    {
      "epoch": 7.893854748603352,
      "grad_norm": 4.616107951175197,
      "learning_rate": 2.9920219738023154e-06,
      "logits/chosen": -3.3550543785095215,
      "logits/rejected": -3.0897629261016846,
      "logps/chosen": -6.496947288513184,
      "logps/rejected": -44.454078674316406,
      "loss": 0.1569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.010401725769043,
      "rewards/margins": 3.010401725769043,
      "rewards/rejected": 0.0,
      "step": 1413
    },
    {
      "epoch": 7.899441340782123,
      "grad_norm": 4.645250551757876,
      "learning_rate": 2.9890334938312328e-06,
      "logits/chosen": -3.5583345890045166,
      "logits/rejected": -3.419954538345337,
      "logps/chosen": -5.553355693817139,
      "logps/rejected": -42.300167083740234,
      "loss": 0.1237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8079168796539307,
      "rewards/margins": 2.8079168796539307,
      "rewards/rejected": 0.0,
      "step": 1414
    },
    {
      "epoch": 7.905027932960894,
      "grad_norm": 4.329967906369613,
      "learning_rate": 2.9860442870692857e-06,
      "logits/chosen": -3.622997522354126,
      "logits/rejected": -3.2384307384490967,
      "logps/chosen": -4.745515823364258,
      "logps/rejected": -43.36189270019531,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2934422492980957,
      "rewards/margins": 2.2934422492980957,
      "rewards/rejected": 0.0,
      "step": 1415
    },
    {
      "epoch": 7.910614525139665,
      "grad_norm": 2.533519405998842,
      "learning_rate": 2.983054357958969e-06,
      "logits/chosen": -3.6792590618133545,
      "logits/rejected": -3.6193299293518066,
      "logps/chosen": -6.5248517990112305,
      "logps/rejected": -70.97792053222656,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.69710636138916,
      "rewards/margins": 2.69710636138916,
      "rewards/rejected": 0.0,
      "step": 1416
    },
    {
      "epoch": 7.916201117318436,
      "grad_norm": 4.052568157466324,
      "learning_rate": 2.9800637109438483e-06,
      "logits/chosen": -3.3659873008728027,
      "logits/rejected": -3.3682074546813965,
      "logps/chosen": -9.128846168518066,
      "logps/rejected": -40.86427307128906,
      "loss": 0.1438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.866758346557617,
      "rewards/margins": 2.866758346557617,
      "rewards/rejected": 0.0,
      "step": 1417
    },
    {
      "epoch": 7.921787709497207,
      "grad_norm": 2.4902061802825632,
      "learning_rate": 2.9770723504685593e-06,
      "logits/chosen": -3.449894666671753,
      "logits/rejected": -3.3100852966308594,
      "logps/chosen": -11.096256256103516,
      "logps/rejected": -32.96156692504883,
      "loss": 0.1195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.214679002761841,
      "rewards/margins": 3.214679002761841,
      "rewards/rejected": 0.0,
      "step": 1418
    },
    {
      "epoch": 7.927374301675978,
      "grad_norm": 2.704663112301389,
      "learning_rate": 2.9740802809787954e-06,
      "logits/chosen": -3.0681869983673096,
      "logits/rejected": -3.1609270572662354,
      "logps/chosen": -2.6014633178710938,
      "logps/rejected": -34.29265594482422,
      "loss": 0.1329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.443974018096924,
      "rewards/margins": 2.443974018096924,
      "rewards/rejected": 0.0,
      "step": 1419
    },
    {
      "epoch": 7.932960893854749,
      "grad_norm": 1.6728628679956523,
      "learning_rate": 2.9710875069213046e-06,
      "logits/chosen": -3.151428461074829,
      "logits/rejected": -3.384648323059082,
      "logps/chosen": -13.95676326751709,
      "logps/rejected": -68.54710388183594,
      "loss": 0.1183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0886096954345703,
      "rewards/margins": 3.0886096954345703,
      "rewards/rejected": 0.0,
      "step": 1420
    },
    {
      "epoch": 7.932960893854749,
      "eval_logits/chosen": -3.3513782024383545,
      "eval_logits/rejected": -3.4555461406707764,
      "eval_logps/chosen": -34.935523986816406,
      "eval_logps/rejected": -52.64850997924805,
      "eval_loss": 1.3438135385513306,
      "eval_rewards/accuracies": 0.25,
      "eval_rewards/chosen": -0.48654842376708984,
      "eval_rewards/margins": -0.48654842376708984,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6387,
      "eval_samples_per_second": 12.091,
      "eval_steps_per_second": 0.39,
      "step": 1420
    },
    {
      "epoch": 7.93854748603352,
      "grad_norm": 1.7125449486474407,
      "learning_rate": 2.9680940327438825e-06,
      "logits/chosen": -3.6445083618164062,
      "logits/rejected": -3.7479381561279297,
      "logps/chosen": -0.40857958793640137,
      "logps/rejected": -36.38641357421875,
      "loss": 0.1279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8336868286132812,
      "rewards/margins": 1.8336868286132812,
      "rewards/rejected": 0.0,
      "step": 1421
    },
    {
      "epoch": 7.94413407821229,
      "grad_norm": 1.6963828817777127,
      "learning_rate": 2.965099862895365e-06,
      "logits/chosen": -3.2377912998199463,
      "logits/rejected": -3.1857833862304688,
      "logps/chosen": -17.120317459106445,
      "logps/rejected": -55.302330017089844,
      "loss": 0.0845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9127249717712402,
      "rewards/margins": 3.9127249717712402,
      "rewards/rejected": 0.0,
      "step": 1422
    },
    {
      "epoch": 7.949720670391061,
      "grad_norm": 1.3204670516264987,
      "learning_rate": 2.962105001825621e-06,
      "logits/chosen": -3.6733810901641846,
      "logits/rejected": -3.713507890701294,
      "logps/chosen": -1.8921575546264648,
      "logps/rejected": -51.94319152832031,
      "loss": 0.0936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.590512275695801,
      "rewards/margins": 2.590512275695801,
      "rewards/rejected": 0.0,
      "step": 1423
    },
    {
      "epoch": 7.955307262569832,
      "grad_norm": 7.83436576419265,
      "learning_rate": 2.9591094539855475e-06,
      "logits/chosen": -3.7395339012145996,
      "logits/rejected": -3.6417617797851562,
      "logps/chosen": -6.966512680053711,
      "logps/rejected": -35.771297454833984,
      "loss": 0.1612,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4603602886199951,
      "rewards/margins": 1.4603602886199951,
      "rewards/rejected": 0.0,
      "step": 1424
    },
    {
      "epoch": 7.960893854748603,
      "grad_norm": 1.600240172263794,
      "learning_rate": 2.9561132238270635e-06,
      "logits/chosen": -3.5078811645507812,
      "logits/rejected": -3.5220775604248047,
      "logps/chosen": -5.553656578063965,
      "logps/rejected": -43.1175537109375,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.547321081161499,
      "rewards/margins": 3.547321081161499,
      "rewards/rejected": 0.0,
      "step": 1425
    },
    {
      "epoch": 7.966480446927374,
      "grad_norm": 2.8058111656744233,
      "learning_rate": 2.953116315803099e-06,
      "logits/chosen": -3.501645565032959,
      "logits/rejected": -3.5383598804473877,
      "logps/chosen": -13.801538467407227,
      "logps/rejected": -47.86945724487305,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7008280754089355,
      "rewards/margins": 2.7008280754089355,
      "rewards/rejected": 0.0,
      "step": 1426
    },
    {
      "epoch": 7.972067039106145,
      "grad_norm": 2.422939572368586,
      "learning_rate": 2.950118734367594e-06,
      "logits/chosen": -3.6706302165985107,
      "logits/rejected": -3.7204620838165283,
      "logps/chosen": -21.466999053955078,
      "logps/rejected": -58.365150451660156,
      "loss": 0.1072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.183065891265869,
      "rewards/margins": 3.183065891265869,
      "rewards/rejected": 0.0,
      "step": 1427
    },
    {
      "epoch": 7.977653631284916,
      "grad_norm": 3.602657174336879,
      "learning_rate": 2.947120483975488e-06,
      "logits/chosen": -3.760525941848755,
      "logits/rejected": -3.499629497528076,
      "logps/chosen": -7.373886585235596,
      "logps/rejected": -83.7888412475586,
      "loss": 0.1403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.235901355743408,
      "rewards/margins": 2.235901355743408,
      "rewards/rejected": 0.0,
      "step": 1428
    },
    {
      "epoch": 7.983240223463687,
      "grad_norm": 2.578025719873282,
      "learning_rate": 2.9441215690827146e-06,
      "logits/chosen": -3.6086368560791016,
      "logits/rejected": -3.526695728302002,
      "logps/chosen": -9.002800941467285,
      "logps/rejected": -43.326133728027344,
      "loss": 0.1239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4554648399353027,
      "rewards/margins": 2.4554648399353027,
      "rewards/rejected": 0.0,
      "step": 1429
    },
    {
      "epoch": 7.988826815642458,
      "grad_norm": 1.2591458393623527,
      "learning_rate": 2.9411219941461967e-06,
      "logits/chosen": -3.517293691635132,
      "logits/rejected": -3.5608794689178467,
      "logps/chosen": -0.774393618106842,
      "logps/rejected": -42.839168548583984,
      "loss": 0.0867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.288191795349121,
      "rewards/margins": 2.288191795349121,
      "rewards/rejected": 0.0,
      "step": 1430
    },
    {
      "epoch": 7.994413407821229,
      "grad_norm": 1.3368323089265581,
      "learning_rate": 2.9381217636238344e-06,
      "logits/chosen": -3.6473817825317383,
      "logits/rejected": -3.5799190998077393,
      "logps/chosen": -8.469541549682617,
      "logps/rejected": -45.30213928222656,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.86891508102417,
      "rewards/margins": 2.86891508102417,
      "rewards/rejected": 0.0,
      "step": 1431
    },
    {
      "epoch": 8.0,
      "grad_norm": 1.4253851944129272,
      "learning_rate": 2.935120881974507e-06,
      "logits/chosen": -3.5863380432128906,
      "logits/rejected": -3.6218883991241455,
      "logps/chosen": -1.095492959022522,
      "logps/rejected": -46.607078552246094,
      "loss": 0.1063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3520774841308594,
      "rewards/margins": 2.3520774841308594,
      "rewards/rejected": 0.0,
      "step": 1432
    },
    {
      "epoch": 8.005586592178771,
      "grad_norm": 1.6828489484865374,
      "learning_rate": 2.9321193536580583e-06,
      "logits/chosen": -3.262920618057251,
      "logits/rejected": -3.3500819206237793,
      "logps/chosen": -19.797988891601562,
      "logps/rejected": -58.5503044128418,
      "loss": 0.0923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8223438262939453,
      "rewards/margins": 3.8223438262939453,
      "rewards/rejected": 0.0,
      "step": 1433
    },
    {
      "epoch": 8.011173184357542,
      "grad_norm": 0.7161242414324326,
      "learning_rate": 2.9291171831352934e-06,
      "logits/chosen": -3.501568078994751,
      "logits/rejected": -3.345961332321167,
      "logps/chosen": -0.33421677350997925,
      "logps/rejected": -58.570579528808594,
      "loss": 0.0842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.872872233390808,
      "rewards/margins": 1.872872233390808,
      "rewards/rejected": 0.0,
      "step": 1434
    },
    {
      "epoch": 8.016759776536313,
      "grad_norm": 1.3316744454989298,
      "learning_rate": 2.9261143748679734e-06,
      "logits/chosen": -3.5479249954223633,
      "logits/rejected": -3.6649396419525146,
      "logps/chosen": -0.6760504245758057,
      "logps/rejected": -51.16801452636719,
      "loss": 0.126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9639060497283936,
      "rewards/margins": 1.9639060497283936,
      "rewards/rejected": 0.0,
      "step": 1435
    },
    {
      "epoch": 8.022346368715084,
      "grad_norm": 1.9277087304775484,
      "learning_rate": 2.923110933318805e-06,
      "logits/chosen": -3.442756414413452,
      "logits/rejected": -3.3442494869232178,
      "logps/chosen": -1.3118164539337158,
      "logps/rejected": -41.47234344482422,
      "loss": 0.095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7216362953186035,
      "rewards/margins": 2.7216362953186035,
      "rewards/rejected": 0.0,
      "step": 1436
    },
    {
      "epoch": 8.027932960893855,
      "grad_norm": 3.0251349418342977,
      "learning_rate": 2.9201068629514386e-06,
      "logits/chosen": -3.5080149173736572,
      "logits/rejected": -3.5067083835601807,
      "logps/chosen": -3.3619930744171143,
      "logps/rejected": -49.55982208251953,
      "loss": 0.1479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.901059627532959,
      "rewards/margins": 2.901059627532959,
      "rewards/rejected": 0.0,
      "step": 1437
    },
    {
      "epoch": 8.033519553072626,
      "grad_norm": 1.1884928316230357,
      "learning_rate": 2.917102168230456e-06,
      "logits/chosen": -3.6663241386413574,
      "logits/rejected": -3.849565029144287,
      "logps/chosen": -3.8331799507141113,
      "logps/rejected": -69.32089233398438,
      "loss": 0.1109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3444271087646484,
      "rewards/margins": 3.3444271087646484,
      "rewards/rejected": 0.0,
      "step": 1438
    },
    {
      "epoch": 8.039106145251397,
      "grad_norm": 4.322319228371034,
      "learning_rate": 2.91409685362137e-06,
      "logits/chosen": -3.701253652572632,
      "logits/rejected": -3.7062911987304688,
      "logps/chosen": -16.568843841552734,
      "logps/rejected": -71.52427673339844,
      "loss": 0.1057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.871524810791016,
      "rewards/margins": 4.871524810791016,
      "rewards/rejected": 0.0,
      "step": 1439
    },
    {
      "epoch": 8.044692737430168,
      "grad_norm": 1.7070741285345203,
      "learning_rate": 2.9110909235906125e-06,
      "logits/chosen": -3.5033366680145264,
      "logits/rejected": -3.4958784580230713,
      "logps/chosen": -0.8068684339523315,
      "logps/rejected": -66.22521209716797,
      "loss": 0.1508,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3587948083877563,
      "rewards/margins": 1.3587948083877563,
      "rewards/rejected": 0.0,
      "step": 1440
    },
    {
      "epoch": 8.044692737430168,
      "eval_logits/chosen": -3.3931045532226562,
      "eval_logits/rejected": -3.507702589035034,
      "eval_logps/chosen": -36.95717239379883,
      "eval_logps/rejected": -57.39923858642578,
      "eval_loss": 1.5143834352493286,
      "eval_rewards/accuracies": 0.25,
      "eval_rewards/chosen": -0.6887128949165344,
      "eval_rewards/margins": -0.6887128949165344,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5142,
      "eval_samples_per_second": 12.15,
      "eval_steps_per_second": 0.392,
      "step": 1440
    },
    {
      "epoch": 8.050279329608939,
      "grad_norm": 2.0251747561020554,
      "learning_rate": 2.9080843826055316e-06,
      "logits/chosen": -3.5404040813446045,
      "logits/rejected": -3.464491128921509,
      "logps/chosen": -5.118672847747803,
      "logps/rejected": -59.09503173828125,
      "loss": 0.1161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5263547897338867,
      "rewards/margins": 3.5263547897338867,
      "rewards/rejected": 0.0,
      "step": 1441
    },
    {
      "epoch": 8.05586592178771,
      "grad_norm": 3.142543656341117,
      "learning_rate": 2.905077235134382e-06,
      "logits/chosen": -3.5473763942718506,
      "logits/rejected": -3.4302377700805664,
      "logps/chosen": -3.108088731765747,
      "logps/rejected": -69.26390838623047,
      "loss": 0.143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8629984855651855,
      "rewards/margins": 3.8629984855651855,
      "rewards/rejected": 0.0,
      "step": 1442
    },
    {
      "epoch": 8.061452513966481,
      "grad_norm": 1.7325888030840484,
      "learning_rate": 2.902069485646321e-06,
      "logits/chosen": -3.6541664600372314,
      "logits/rejected": -3.6830408573150635,
      "logps/chosen": -7.037563323974609,
      "logps/rejected": -84.62738037109375,
      "loss": 0.1556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3406262397766113,
      "rewards/margins": 3.3406262397766113,
      "rewards/rejected": 0.0,
      "step": 1443
    },
    {
      "epoch": 8.067039106145252,
      "grad_norm": 1.2829673035939555,
      "learning_rate": 2.899061138611399e-06,
      "logits/chosen": -3.67911434173584,
      "logits/rejected": -3.6591012477874756,
      "logps/chosen": -3.561357259750366,
      "logps/rejected": -72.29397583007812,
      "loss": 0.0766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4915695190429688,
      "rewards/margins": 2.4915695190429688,
      "rewards/rejected": 0.0,
      "step": 1444
    },
    {
      "epoch": 8.072625698324023,
      "grad_norm": 4.000903129302735,
      "learning_rate": 2.896052198500557e-06,
      "logits/chosen": -3.142383337020874,
      "logits/rejected": -3.309530258178711,
      "logps/chosen": -0.7317455410957336,
      "logps/rejected": -81.34931945800781,
      "loss": 0.1379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.039340019226074,
      "rewards/margins": 2.039340019226074,
      "rewards/rejected": 0.0,
      "step": 1445
    },
    {
      "epoch": 8.078212290502794,
      "grad_norm": 1.9905737935581174,
      "learning_rate": 2.8930426697856143e-06,
      "logits/chosen": -3.545384168624878,
      "logits/rejected": -3.5094659328460693,
      "logps/chosen": -0.6739726066589355,
      "logps/rejected": -62.30235290527344,
      "loss": 0.109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.271129846572876,
      "rewards/margins": 2.271129846572876,
      "rewards/rejected": 0.0,
      "step": 1446
    },
    {
      "epoch": 8.083798882681565,
      "grad_norm": 1.5326270796579526,
      "learning_rate": 2.8900325569392683e-06,
      "logits/chosen": -3.553866386413574,
      "logits/rejected": -3.6070351600646973,
      "logps/chosen": -1.7856090068817139,
      "logps/rejected": -69.31573486328125,
      "loss": 0.1019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.788356304168701,
      "rewards/margins": 2.788356304168701,
      "rewards/rejected": 0.0,
      "step": 1447
    },
    {
      "epoch": 8.089385474860336,
      "grad_norm": 1.5053813162256457,
      "learning_rate": 2.887021864435081e-06,
      "logits/chosen": -3.615330696105957,
      "logits/rejected": -3.4762494564056396,
      "logps/chosen": -4.209636688232422,
      "logps/rejected": -32.17325210571289,
      "loss": 0.0959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3859171867370605,
      "rewards/margins": 3.3859171867370605,
      "rewards/rejected": 0.0,
      "step": 1448
    },
    {
      "epoch": 8.094972067039107,
      "grad_norm": 2.725269307843547,
      "learning_rate": 2.8840105967474784e-06,
      "logits/chosen": -3.6424052715301514,
      "logits/rejected": -3.5844156742095947,
      "logps/chosen": -0.8862940073013306,
      "logps/rejected": -66.27734375,
      "loss": 0.1134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.095827102661133,
      "rewards/margins": 2.095827102661133,
      "rewards/rejected": 0.0,
      "step": 1449
    },
    {
      "epoch": 8.100558659217878,
      "grad_norm": 1.6178617890595979,
      "learning_rate": 2.8809987583517397e-06,
      "logits/chosen": -3.324767827987671,
      "logits/rejected": -3.2823827266693115,
      "logps/chosen": -2.44427752494812,
      "logps/rejected": -76.26038360595703,
      "loss": 0.1373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.085387945175171,
      "rewards/margins": 2.085387945175171,
      "rewards/rejected": 0.0,
      "step": 1450
    },
    {
      "epoch": 8.106145251396647,
      "grad_norm": 1.60172981436379,
      "learning_rate": 2.877986353723995e-06,
      "logits/chosen": -3.8072872161865234,
      "logits/rejected": -3.9065279960632324,
      "logps/chosen": -1.4042706489562988,
      "logps/rejected": -73.13301849365234,
      "loss": 0.1146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5309760570526123,
      "rewards/margins": 2.5309760570526123,
      "rewards/rejected": 0.0,
      "step": 1451
    },
    {
      "epoch": 8.111731843575418,
      "grad_norm": 3.4452319455345783,
      "learning_rate": 2.874973387341213e-06,
      "logits/chosen": -3.5722577571868896,
      "logits/rejected": -3.4148242473602295,
      "logps/chosen": -13.362710952758789,
      "logps/rejected": -70.48816680908203,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.452395439147949,
      "rewards/margins": 4.452395439147949,
      "rewards/rejected": 0.0,
      "step": 1452
    },
    {
      "epoch": 8.11731843575419,
      "grad_norm": 1.9447919297302694,
      "learning_rate": 2.8719598636811974e-06,
      "logits/chosen": -3.287785053253174,
      "logits/rejected": -3.587829828262329,
      "logps/chosen": -0.8271405100822449,
      "logps/rejected": -84.34754943847656,
      "loss": 0.1193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.189757823944092,
      "rewards/margins": 2.189757823944092,
      "rewards/rejected": 0.0,
      "step": 1453
    },
    {
      "epoch": 8.12290502793296,
      "grad_norm": 2.1992394047877433,
      "learning_rate": 2.8689457872225824e-06,
      "logits/chosen": -3.35762619972229,
      "logits/rejected": -3.5390701293945312,
      "logps/chosen": -7.305087089538574,
      "logps/rejected": -51.17259216308594,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1283674240112305,
      "rewards/margins": 3.1283674240112305,
      "rewards/rejected": 0.0,
      "step": 1454
    },
    {
      "epoch": 8.128491620111731,
      "grad_norm": 6.725015079840259,
      "learning_rate": 2.865931162444822e-06,
      "logits/chosen": -3.35797381401062,
      "logits/rejected": -3.46256160736084,
      "logps/chosen": -1.6660610437393188,
      "logps/rejected": -47.790504455566406,
      "loss": 0.1813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8502397537231445,
      "rewards/margins": 2.8502397537231445,
      "rewards/rejected": 0.0,
      "step": 1455
    },
    {
      "epoch": 8.134078212290502,
      "grad_norm": 2.270134533422108,
      "learning_rate": 2.8629159938281852e-06,
      "logits/chosen": -3.4082865715026855,
      "logits/rejected": -3.359618663787842,
      "logps/chosen": -2.4077844619750977,
      "logps/rejected": -48.261478424072266,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6673564910888672,
      "rewards/margins": 1.6673564910888672,
      "rewards/rejected": 0.0,
      "step": 1456
    },
    {
      "epoch": 8.139664804469273,
      "grad_norm": 1.674461286098193,
      "learning_rate": 2.8599002858537508e-06,
      "logits/chosen": -3.569650888442993,
      "logits/rejected": -3.6624841690063477,
      "logps/chosen": -1.263794183731079,
      "logps/rejected": -81.54490661621094,
      "loss": 0.1218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2649950981140137,
      "rewards/margins": 2.2649950981140137,
      "rewards/rejected": 0.0,
      "step": 1457
    },
    {
      "epoch": 8.145251396648044,
      "grad_norm": 1.38809095775955,
      "learning_rate": 2.8568840430033957e-06,
      "logits/chosen": -3.178231954574585,
      "logits/rejected": -3.5865042209625244,
      "logps/chosen": -4.243362903594971,
      "logps/rejected": -86.17424011230469,
      "loss": 0.1028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6749072074890137,
      "rewards/margins": 2.6749072074890137,
      "rewards/rejected": 0.0,
      "step": 1458
    },
    {
      "epoch": 8.150837988826815,
      "grad_norm": 6.865399133923297,
      "learning_rate": 2.853867269759796e-06,
      "logits/chosen": -3.4443235397338867,
      "logits/rejected": -3.1622979640960693,
      "logps/chosen": -11.291679382324219,
      "logps/rejected": -39.42967224121094,
      "loss": 0.1243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.426790237426758,
      "rewards/margins": 4.426790237426758,
      "rewards/rejected": 0.0,
      "step": 1459
    },
    {
      "epoch": 8.156424581005586,
      "grad_norm": 2.734484331607521,
      "learning_rate": 2.8508499706064117e-06,
      "logits/chosen": -3.53515625,
      "logits/rejected": -3.65216326713562,
      "logps/chosen": -0.9559468030929565,
      "logps/rejected": -41.94486999511719,
      "loss": 0.1378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2366690635681152,
      "rewards/margins": 2.2366690635681152,
      "rewards/rejected": 0.0,
      "step": 1460
    },
    {
      "epoch": 8.156424581005586,
      "eval_logits/chosen": -3.362084150314331,
      "eval_logits/rejected": -3.4654693603515625,
      "eval_logps/chosen": -35.35859680175781,
      "eval_logps/rejected": -54.34197235107422,
      "eval_loss": 1.3800759315490723,
      "eval_rewards/accuracies": 0.3499999940395355,
      "eval_rewards/chosen": -0.5288556218147278,
      "eval_rewards/margins": -0.5288556218147278,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5309,
      "eval_samples_per_second": 12.142,
      "eval_steps_per_second": 0.392,
      "step": 1460
    },
    {
      "epoch": 8.162011173184357,
      "grad_norm": 0.9438532588839996,
      "learning_rate": 2.847832150027489e-06,
      "logits/chosen": -3.4414689540863037,
      "logits/rejected": -3.589860677719116,
      "logps/chosen": -7.320827484130859,
      "logps/rejected": -77.96403503417969,
      "loss": 0.0971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7381949424743652,
      "rewards/margins": 3.7381949424743652,
      "rewards/rejected": 0.0,
      "step": 1461
    },
    {
      "epoch": 8.167597765363128,
      "grad_norm": 1.7271065208641423,
      "learning_rate": 2.844813812508046e-06,
      "logits/chosen": -3.579096794128418,
      "logits/rejected": -3.589195489883423,
      "logps/chosen": -1.5120635032653809,
      "logps/rejected": -78.04534149169922,
      "loss": 0.1193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8025869131088257,
      "rewards/margins": 1.8025869131088257,
      "rewards/rejected": 0.0,
      "step": 1462
    },
    {
      "epoch": 8.1731843575419,
      "grad_norm": 2.4441980161202204,
      "learning_rate": 2.8417949625338694e-06,
      "logits/chosen": -3.643726348876953,
      "logits/rejected": -3.5921921730041504,
      "logps/chosen": -2.9244742393493652,
      "logps/rejected": -59.972206115722656,
      "loss": 0.1179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2572765350341797,
      "rewards/margins": 3.2572765350341797,
      "rewards/rejected": 0.0,
      "step": 1463
    },
    {
      "epoch": 8.17877094972067,
      "grad_norm": 1.0313664217588967,
      "learning_rate": 2.8387756045915084e-06,
      "logits/chosen": -3.7048521041870117,
      "logits/rejected": -3.5639841556549072,
      "logps/chosen": -2.1664843559265137,
      "logps/rejected": -57.07040786743164,
      "loss": 0.0974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9559290409088135,
      "rewards/margins": 2.9559290409088135,
      "rewards/rejected": 0.0,
      "step": 1464
    },
    {
      "epoch": 8.184357541899441,
      "grad_norm": 1.472420775265949,
      "learning_rate": 2.8357557431682657e-06,
      "logits/chosen": -3.668116569519043,
      "logits/rejected": -3.68455171585083,
      "logps/chosen": -0.9398554563522339,
      "logps/rejected": -36.25319290161133,
      "loss": 0.1031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.32269287109375,
      "rewards/margins": 2.32269287109375,
      "rewards/rejected": 0.0,
      "step": 1465
    },
    {
      "epoch": 8.189944134078212,
      "grad_norm": 4.591916131215083,
      "learning_rate": 2.8327353827521947e-06,
      "logits/chosen": -3.3412139415740967,
      "logits/rejected": -3.1578209400177,
      "logps/chosen": -8.640987396240234,
      "logps/rejected": -39.16083908081055,
      "loss": 0.1627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.969564437866211,
      "rewards/margins": 2.969564437866211,
      "rewards/rejected": 0.0,
      "step": 1466
    },
    {
      "epoch": 8.195530726256983,
      "grad_norm": 1.0501446459242327,
      "learning_rate": 2.829714527832088e-06,
      "logits/chosen": -3.4891436100006104,
      "logits/rejected": -3.502720832824707,
      "logps/chosen": -5.599181652069092,
      "logps/rejected": -64.33663940429688,
      "loss": 0.1147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9423909187316895,
      "rewards/margins": 3.9423909187316895,
      "rewards/rejected": 0.0,
      "step": 1467
    },
    {
      "epoch": 8.201117318435754,
      "grad_norm": 0.6329073289165194,
      "learning_rate": 2.8266931828974738e-06,
      "logits/chosen": -3.5291056632995605,
      "logits/rejected": -3.5647377967834473,
      "logps/chosen": -3.160949230194092,
      "logps/rejected": -49.30059814453125,
      "loss": 0.0821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.184675455093384,
      "rewards/margins": 3.184675455093384,
      "rewards/rejected": 0.0,
      "step": 1468
    },
    {
      "epoch": 8.206703910614525,
      "grad_norm": 1.7259432104981933,
      "learning_rate": 2.8236713524386085e-06,
      "logits/chosen": -3.7496891021728516,
      "logits/rejected": -3.6690938472747803,
      "logps/chosen": -1.5001325607299805,
      "logps/rejected": -43.63496398925781,
      "loss": 0.1149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4227023124694824,
      "rewards/margins": 2.4227023124694824,
      "rewards/rejected": 0.0,
      "step": 1469
    },
    {
      "epoch": 8.212290502793296,
      "grad_norm": 2.6165309190389947,
      "learning_rate": 2.8206490409464705e-06,
      "logits/chosen": -3.679755687713623,
      "logits/rejected": -3.5419762134552,
      "logps/chosen": -2.9272310733795166,
      "logps/rejected": -37.93617248535156,
      "loss": 0.0916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.240577459335327,
      "rewards/margins": 3.240577459335327,
      "rewards/rejected": 0.0,
      "step": 1470
    },
    {
      "epoch": 8.217877094972067,
      "grad_norm": 2.84900237674439,
      "learning_rate": 2.817626252912754e-06,
      "logits/chosen": -3.296971559524536,
      "logits/rejected": -3.3733139038085938,
      "logps/chosen": -1.359489917755127,
      "logps/rejected": -129.94342041015625,
      "loss": 0.132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3791165351867676,
      "rewards/margins": 2.3791165351867676,
      "rewards/rejected": 0.0,
      "step": 1471
    },
    {
      "epoch": 8.223463687150838,
      "grad_norm": 1.196262730995356,
      "learning_rate": 2.814602992829859e-06,
      "logits/chosen": -3.6383249759674072,
      "logits/rejected": -3.4820988178253174,
      "logps/chosen": -8.850275039672852,
      "logps/rejected": -65.16409301757812,
      "loss": 0.0921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.083110809326172,
      "rewards/margins": 4.083110809326172,
      "rewards/rejected": 0.0,
      "step": 1472
    },
    {
      "epoch": 8.22905027932961,
      "grad_norm": 5.618215917289481,
      "learning_rate": 2.811579265190889e-06,
      "logits/chosen": -3.6187779903411865,
      "logits/rejected": -3.4401001930236816,
      "logps/chosen": -2.0156450271606445,
      "logps/rejected": -36.312217712402344,
      "loss": 0.166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.657099485397339,
      "rewards/margins": 2.657099485397339,
      "rewards/rejected": 0.0,
      "step": 1473
    },
    {
      "epoch": 8.23463687150838,
      "grad_norm": 5.145878166579033,
      "learning_rate": 2.8085550744896418e-06,
      "logits/chosen": -3.2527849674224854,
      "logits/rejected": -3.2175331115722656,
      "logps/chosen": -0.7259204387664795,
      "logps/rejected": -47.123817443847656,
      "loss": 0.1371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0112404823303223,
      "rewards/margins": 2.0112404823303223,
      "rewards/rejected": 0.0,
      "step": 1474
    },
    {
      "epoch": 8.240223463687151,
      "grad_norm": 2.6731558139640654,
      "learning_rate": 2.805530425220603e-06,
      "logits/chosen": -3.491462230682373,
      "logits/rejected": -3.4772276878356934,
      "logps/chosen": -3.6653261184692383,
      "logps/rejected": -49.90317916870117,
      "loss": 0.136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5852317810058594,
      "rewards/margins": 3.5852317810058594,
      "rewards/rejected": 0.0,
      "step": 1475
    },
    {
      "epoch": 8.245810055865922,
      "grad_norm": 2.7544074433326875,
      "learning_rate": 2.8025053218789407e-06,
      "logits/chosen": -3.5588221549987793,
      "logits/rejected": -3.5973000526428223,
      "logps/chosen": -5.310393810272217,
      "logps/rejected": -76.7413558959961,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7508702278137207,
      "rewards/margins": 2.7508702278137207,
      "rewards/rejected": 0.0,
      "step": 1476
    },
    {
      "epoch": 8.251396648044693,
      "grad_norm": 2.458671593536282,
      "learning_rate": 2.7994797689604976e-06,
      "logits/chosen": -3.854295253753662,
      "logits/rejected": -3.69282603263855,
      "logps/chosen": -8.344049453735352,
      "logps/rejected": -60.37919998168945,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7179341316223145,
      "rewards/margins": 2.7179341316223145,
      "rewards/rejected": 0.0,
      "step": 1477
    },
    {
      "epoch": 8.256983240223464,
      "grad_norm": 1.6719786728526207,
      "learning_rate": 2.7964537709617822e-06,
      "logits/chosen": -3.7080092430114746,
      "logits/rejected": -3.505143642425537,
      "logps/chosen": -0.8561469912528992,
      "logps/rejected": -75.92698669433594,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.021745204925537,
      "rewards/margins": 2.021745204925537,
      "rewards/rejected": 0.0,
      "step": 1478
    },
    {
      "epoch": 8.262569832402235,
      "grad_norm": 1.6356134921719185,
      "learning_rate": 2.793427332379968e-06,
      "logits/chosen": -3.625220775604248,
      "logits/rejected": -3.7256906032562256,
      "logps/chosen": -6.212230682373047,
      "logps/rejected": -60.492916107177734,
      "loss": 0.1001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.303191184997559,
      "rewards/margins": 4.303191184997559,
      "rewards/rejected": 0.0,
      "step": 1479
    },
    {
      "epoch": 8.268156424581006,
      "grad_norm": 1.43175832705412,
      "learning_rate": 2.7904004577128814e-06,
      "logits/chosen": -3.906503915786743,
      "logits/rejected": -3.6731841564178467,
      "logps/chosen": -10.193510055541992,
      "logps/rejected": -47.555809020996094,
      "loss": 0.0849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.674679756164551,
      "rewards/margins": 3.674679756164551,
      "rewards/rejected": 0.0,
      "step": 1480
    },
    {
      "epoch": 8.268156424581006,
      "eval_logits/chosen": -3.433626174926758,
      "eval_logits/rejected": -3.54734468460083,
      "eval_logps/chosen": -37.004066467285156,
      "eval_logps/rejected": -55.990638732910156,
      "eval_loss": 1.4347591400146484,
      "eval_rewards/accuracies": 0.25,
      "eval_rewards/chosen": -0.6934025883674622,
      "eval_rewards/margins": -0.6934025883674622,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5513,
      "eval_samples_per_second": 12.132,
      "eval_steps_per_second": 0.391,
      "step": 1480
    },
    {
      "epoch": 8.273743016759777,
      "grad_norm": 2.654401101366398,
      "learning_rate": 2.787373151458997e-06,
      "logits/chosen": -3.760444402694702,
      "logits/rejected": -3.2412967681884766,
      "logps/chosen": -0.7370131015777588,
      "logps/rejected": -45.599178314208984,
      "loss": 0.124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.337430953979492,
      "rewards/margins": 2.337430953979492,
      "rewards/rejected": 0.0,
      "step": 1481
    },
    {
      "epoch": 8.279329608938548,
      "grad_norm": 3.9296248965965335,
      "learning_rate": 2.7843454181174305e-06,
      "logits/chosen": -3.419602632522583,
      "logits/rejected": -3.604227066040039,
      "logps/chosen": -0.7293708324432373,
      "logps/rejected": -80.20232391357422,
      "loss": 0.1035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.151524782180786,
      "rewards/margins": 2.151524782180786,
      "rewards/rejected": 0.0,
      "step": 1482
    },
    {
      "epoch": 8.28491620111732,
      "grad_norm": 3.2572945910195026,
      "learning_rate": 2.7813172621879347e-06,
      "logits/chosen": -3.2488813400268555,
      "logits/rejected": -3.3378024101257324,
      "logps/chosen": -5.086645126342773,
      "logps/rejected": -34.395652770996094,
      "loss": 0.0899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.855431079864502,
      "rewards/margins": 2.855431079864502,
      "rewards/rejected": 0.0,
      "step": 1483
    },
    {
      "epoch": 8.29050279329609,
      "grad_norm": 1.142949993563128,
      "learning_rate": 2.778288688170887e-06,
      "logits/chosen": -3.9185476303100586,
      "logits/rejected": -4.0069379806518555,
      "logps/chosen": -5.808602333068848,
      "logps/rejected": -54.71819305419922,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7306644916534424,
      "rewards/margins": 2.7306644916534424,
      "rewards/rejected": 0.0,
      "step": 1484
    },
    {
      "epoch": 8.296089385474861,
      "grad_norm": 2.7067398007375045,
      "learning_rate": 2.7752597005672883e-06,
      "logits/chosen": -3.132387399673462,
      "logits/rejected": -3.1080572605133057,
      "logps/chosen": -16.409099578857422,
      "logps/rejected": -31.638917922973633,
      "loss": 0.1246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.68231201171875,
      "rewards/margins": 2.68231201171875,
      "rewards/rejected": 0.0,
      "step": 1485
    },
    {
      "epoch": 8.30167597765363,
      "grad_norm": 1.6630551891788876,
      "learning_rate": 2.772230303878753e-06,
      "logits/chosen": -3.7062883377075195,
      "logits/rejected": -3.7956950664520264,
      "logps/chosen": -17.015830993652344,
      "logps/rejected": -57.28956604003906,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.053874969482422,
      "rewards/margins": 4.053874969482422,
      "rewards/rejected": 0.0,
      "step": 1486
    },
    {
      "epoch": 8.307262569832401,
      "grad_norm": 0.802984493678618,
      "learning_rate": 2.769200502607505e-06,
      "logits/chosen": -3.7574946880340576,
      "logits/rejected": -3.7019076347351074,
      "logps/chosen": -0.9702209234237671,
      "logps/rejected": -82.30574035644531,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3135929107666016,
      "rewards/margins": 2.3135929107666016,
      "rewards/rejected": 0.0,
      "step": 1487
    },
    {
      "epoch": 8.312849162011172,
      "grad_norm": 0.9731373220551094,
      "learning_rate": 2.7661703012563685e-06,
      "logits/chosen": -3.719076633453369,
      "logits/rejected": -3.7044520378112793,
      "logps/chosen": -2.3006656169891357,
      "logps/rejected": -89.47259521484375,
      "loss": 0.1225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7315735816955566,
      "rewards/margins": 2.7315735816955566,
      "rewards/rejected": 0.0,
      "step": 1488
    },
    {
      "epoch": 8.318435754189943,
      "grad_norm": 2.6788399334399884,
      "learning_rate": 2.763139704328761e-06,
      "logits/chosen": -3.565258741378784,
      "logits/rejected": -3.6647109985351562,
      "logps/chosen": -11.796125411987305,
      "logps/rejected": -73.03248596191406,
      "loss": 0.1024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9920763969421387,
      "rewards/margins": 2.9920763969421387,
      "rewards/rejected": 0.0,
      "step": 1489
    },
    {
      "epoch": 8.324022346368714,
      "grad_norm": 1.4266968242994968,
      "learning_rate": 2.760108716328691e-06,
      "logits/chosen": -3.927682638168335,
      "logits/rejected": -3.551032066345215,
      "logps/chosen": -4.772064685821533,
      "logps/rejected": -83.26787567138672,
      "loss": 0.1189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.997985363006592,
      "rewards/margins": 2.997985363006592,
      "rewards/rejected": 0.0,
      "step": 1490
    },
    {
      "epoch": 8.329608938547485,
      "grad_norm": 2.3996720197787473,
      "learning_rate": 2.7570773417607454e-06,
      "logits/chosen": -3.6805436611175537,
      "logits/rejected": -3.5667126178741455,
      "logps/chosen": -3.081847906112671,
      "logps/rejected": -51.98122024536133,
      "loss": 0.1171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.731851100921631,
      "rewards/margins": 2.731851100921631,
      "rewards/rejected": 0.0,
      "step": 1491
    },
    {
      "epoch": 8.335195530726256,
      "grad_norm": 8.464690883026867,
      "learning_rate": 2.7540455851300868e-06,
      "logits/chosen": -3.7190332412719727,
      "logits/rejected": -3.6966638565063477,
      "logps/chosen": -3.096757650375366,
      "logps/rejected": -67.1214599609375,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8623766899108887,
      "rewards/margins": 1.8623766899108887,
      "rewards/rejected": 0.0,
      "step": 1492
    },
    {
      "epoch": 8.340782122905027,
      "grad_norm": 2.722070109774152,
      "learning_rate": 2.7510134509424464e-06,
      "logits/chosen": -3.46327543258667,
      "logits/rejected": -3.443427085876465,
      "logps/chosen": -4.101189613342285,
      "logps/rejected": -64.37994384765625,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5772738456726074,
      "rewards/margins": 2.5772738456726074,
      "rewards/rejected": 0.0,
      "step": 1493
    },
    {
      "epoch": 8.346368715083798,
      "grad_norm": 1.5411273326054822,
      "learning_rate": 2.7479809437041146e-06,
      "logits/chosen": -3.3528661727905273,
      "logits/rejected": -3.192028522491455,
      "logps/chosen": -6.054741859436035,
      "logps/rejected": -43.41411590576172,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.86449933052063,
      "rewards/margins": 2.86449933052063,
      "rewards/rejected": 0.0,
      "step": 1494
    },
    {
      "epoch": 8.35195530726257,
      "grad_norm": 1.7859907032781313,
      "learning_rate": 2.7449480679219367e-06,
      "logits/chosen": -3.7347464561462402,
      "logits/rejected": -3.6083147525787354,
      "logps/chosen": -0.9454731941223145,
      "logps/rejected": -66.28323364257812,
      "loss": 0.1363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.10971736907959,
      "rewards/margins": 2.10971736907959,
      "rewards/rejected": 0.0,
      "step": 1495
    },
    {
      "epoch": 8.35754189944134,
      "grad_norm": 3.0438958524574535,
      "learning_rate": 2.7419148281033075e-06,
      "logits/chosen": -3.5652506351470947,
      "logits/rejected": -3.636260509490967,
      "logps/chosen": -18.428049087524414,
      "logps/rejected": -84.24748229980469,
      "loss": 0.1387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.480862140655518,
      "rewards/margins": 4.480862140655518,
      "rewards/rejected": 0.0,
      "step": 1496
    },
    {
      "epoch": 8.363128491620111,
      "grad_norm": 1.0744122558250029,
      "learning_rate": 2.738881228756162e-06,
      "logits/chosen": -3.3941361904144287,
      "logits/rejected": -3.433868646621704,
      "logps/chosen": -9.500045776367188,
      "logps/rejected": -70.28333282470703,
      "loss": 0.07,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7741289138793945,
      "rewards/margins": 3.7741289138793945,
      "rewards/rejected": 0.0,
      "step": 1497
    },
    {
      "epoch": 8.368715083798882,
      "grad_norm": 1.769871138660432,
      "learning_rate": 2.7358472743889667e-06,
      "logits/chosen": -3.528223752975464,
      "logits/rejected": -3.505769968032837,
      "logps/chosen": -8.766440391540527,
      "logps/rejected": -43.309532165527344,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.635216236114502,
      "rewards/margins": 3.635216236114502,
      "rewards/rejected": 0.0,
      "step": 1498
    },
    {
      "epoch": 8.374301675977653,
      "grad_norm": 1.1502579860531783,
      "learning_rate": 2.7328129695107205e-06,
      "logits/chosen": -3.7007319927215576,
      "logits/rejected": -3.8269479274749756,
      "logps/chosen": -3.395720958709717,
      "logps/rejected": -68.22859954833984,
      "loss": 0.0769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5477240085601807,
      "rewards/margins": 3.5477240085601807,
      "rewards/rejected": 0.0,
      "step": 1499
    },
    {
      "epoch": 8.379888268156424,
      "grad_norm": 4.88654428069592,
      "learning_rate": 2.729778318630939e-06,
      "logits/chosen": -3.542070150375366,
      "logits/rejected": -3.625944137573242,
      "logps/chosen": -1.0368086099624634,
      "logps/rejected": -116.1341781616211,
      "loss": 0.1042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4311299324035645,
      "rewards/margins": 1.4311299324035645,
      "rewards/rejected": 0.0,
      "step": 1500
    },
    {
      "epoch": 8.379888268156424,
      "eval_logits/chosen": -3.419374465942383,
      "eval_logits/rejected": -3.542577028274536,
      "eval_logps/chosen": -38.859580993652344,
      "eval_logps/rejected": -60.3948860168457,
      "eval_loss": 1.6198784112930298,
      "eval_rewards/accuracies": 0.25,
      "eval_rewards/chosen": -0.8789544105529785,
      "eval_rewards/margins": -0.8789544105529785,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5564,
      "eval_samples_per_second": 12.13,
      "eval_steps_per_second": 0.391,
      "step": 1500
    },
    {
      "epoch": 8.385474860335195,
      "grad_norm": 6.321368261835719,
      "learning_rate": 2.7267433262596553e-06,
      "logits/chosen": -3.6432912349700928,
      "logits/rejected": -3.63980770111084,
      "logps/chosen": -23.16826057434082,
      "logps/rejected": -84.58120727539062,
      "loss": 0.1107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.0225510597229004,
      "rewards/margins": 3.0225510597229004,
      "rewards/rejected": 0.0,
      "step": 1501
    },
    {
      "epoch": 8.391061452513966,
      "grad_norm": 2.7104333020630005,
      "learning_rate": 2.7237079969074074e-06,
      "logits/chosen": -3.5188257694244385,
      "logits/rejected": -3.637770175933838,
      "logps/chosen": -7.654501914978027,
      "logps/rejected": -58.46940612792969,
      "loss": 0.1326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4619736671447754,
      "rewards/margins": 3.4619736671447754,
      "rewards/rejected": 0.0,
      "step": 1502
    },
    {
      "epoch": 8.396648044692737,
      "grad_norm": 5.915818620248281,
      "learning_rate": 2.720672335085237e-06,
      "logits/chosen": -3.58001708984375,
      "logits/rejected": -3.5838003158569336,
      "logps/chosen": -14.693309783935547,
      "logps/rejected": -90.53781127929688,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2527523040771484,
      "rewards/margins": 3.2527523040771484,
      "rewards/rejected": 0.0,
      "step": 1503
    },
    {
      "epoch": 8.402234636871508,
      "grad_norm": 1.7076257868328333,
      "learning_rate": 2.7176363453046765e-06,
      "logits/chosen": -2.925823450088501,
      "logits/rejected": -2.77778959274292,
      "logps/chosen": -13.092662811279297,
      "logps/rejected": -35.567726135253906,
      "loss": 0.0954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3782002925872803,
      "rewards/margins": 3.3782002925872803,
      "rewards/rejected": 0.0,
      "step": 1504
    },
    {
      "epoch": 8.40782122905028,
      "grad_norm": 3.4179789301372487,
      "learning_rate": 2.714600032077748e-06,
      "logits/chosen": -3.038050413131714,
      "logits/rejected": -3.0425872802734375,
      "logps/chosen": -3.425931453704834,
      "logps/rejected": -47.249183654785156,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0700454711914062,
      "rewards/margins": 3.0700454711914062,
      "rewards/rejected": 0.0,
      "step": 1505
    },
    {
      "epoch": 8.41340782122905,
      "grad_norm": 1.5794771508792036,
      "learning_rate": 2.7115633999169533e-06,
      "logits/chosen": -3.488541841506958,
      "logits/rejected": -3.7082598209381104,
      "logps/chosen": -1.8185997009277344,
      "logps/rejected": -89.05207061767578,
      "loss": 0.0954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6630451679229736,
      "rewards/margins": 2.6630451679229736,
      "rewards/rejected": 0.0,
      "step": 1506
    },
    {
      "epoch": 8.418994413407821,
      "grad_norm": 4.412978807580679,
      "learning_rate": 2.70852645333527e-06,
      "logits/chosen": -3.645735263824463,
      "logits/rejected": -3.6127378940582275,
      "logps/chosen": -16.888261795043945,
      "logps/rejected": -61.02373504638672,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.220813751220703,
      "rewards/margins": 3.220813751220703,
      "rewards/rejected": 0.0,
      "step": 1507
    },
    {
      "epoch": 8.424581005586592,
      "grad_norm": 3.3898110859779527,
      "learning_rate": 2.7054891968461393e-06,
      "logits/chosen": -3.5111894607543945,
      "logits/rejected": -3.537436008453369,
      "logps/chosen": -4.2579545974731445,
      "logps/rejected": -84.65794372558594,
      "loss": 0.1113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7180850505828857,
      "rewards/margins": 2.7180850505828857,
      "rewards/rejected": 0.0,
      "step": 1508
    },
    {
      "epoch": 8.430167597765363,
      "grad_norm": 2.461461438860545,
      "learning_rate": 2.7024516349634666e-06,
      "logits/chosen": -3.6532905101776123,
      "logits/rejected": -3.854016065597534,
      "logps/chosen": -5.62567138671875,
      "logps/rejected": -55.10467529296875,
      "loss": 0.1162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5602970123291016,
      "rewards/margins": 3.5602970123291016,
      "rewards/rejected": 0.0,
      "step": 1509
    },
    {
      "epoch": 8.435754189944134,
      "grad_norm": 1.0132055257635295,
      "learning_rate": 2.6994137722016107e-06,
      "logits/chosen": -3.4633679389953613,
      "logits/rejected": -3.4900691509246826,
      "logps/chosen": -0.9874974489212036,
      "logps/rejected": -78.52311706542969,
      "loss": 0.0985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.464461326599121,
      "rewards/margins": 2.464461326599121,
      "rewards/rejected": 0.0,
      "step": 1510
    },
    {
      "epoch": 8.441340782122905,
      "grad_norm": 1.9847026613434222,
      "learning_rate": 2.696375613075374e-06,
      "logits/chosen": -3.4868884086608887,
      "logits/rejected": -3.4106228351593018,
      "logps/chosen": -0.7014120817184448,
      "logps/rejected": -30.1532039642334,
      "loss": 0.1081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4834740161895752,
      "rewards/margins": 1.4834740161895752,
      "rewards/rejected": 0.0,
      "step": 1511
    },
    {
      "epoch": 8.446927374301676,
      "grad_norm": 0.7785059732144779,
      "learning_rate": 2.693337162100005e-06,
      "logits/chosen": -3.6966309547424316,
      "logits/rejected": -3.749687910079956,
      "logps/chosen": -2.207249164581299,
      "logps/rejected": -68.25053405761719,
      "loss": 0.0879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8150486946105957,
      "rewards/margins": 2.8150486946105957,
      "rewards/rejected": 0.0,
      "step": 1512
    },
    {
      "epoch": 8.452513966480447,
      "grad_norm": 1.027687509306549,
      "learning_rate": 2.6902984237911815e-06,
      "logits/chosen": -3.608856439590454,
      "logits/rejected": -3.697610378265381,
      "logps/chosen": -1.3855103254318237,
      "logps/rejected": -54.51496887207031,
      "loss": 0.0818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7458338737487793,
      "rewards/margins": 2.7458338737487793,
      "rewards/rejected": 0.0,
      "step": 1513
    },
    {
      "epoch": 8.458100558659218,
      "grad_norm": 1.6948246007496741,
      "learning_rate": 2.6872594026650102e-06,
      "logits/chosen": -3.786410093307495,
      "logits/rejected": -3.646768808364868,
      "logps/chosen": -0.5488488674163818,
      "logps/rejected": -46.17889404296875,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.264526844024658,
      "rewards/margins": 2.264526844024658,
      "rewards/rejected": 0.0,
      "step": 1514
    },
    {
      "epoch": 8.46368715083799,
      "grad_norm": 2.7715145486326844,
      "learning_rate": 2.684220103238016e-06,
      "logits/chosen": -3.540884256362915,
      "logits/rejected": -3.3021605014801025,
      "logps/chosen": -0.8746012449264526,
      "logps/rejected": -51.94891357421875,
      "loss": 0.1288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.51383638381958,
      "rewards/margins": 2.51383638381958,
      "rewards/rejected": 0.0,
      "step": 1515
    },
    {
      "epoch": 8.46927374301676,
      "grad_norm": 0.846967124861549,
      "learning_rate": 2.681180530027142e-06,
      "logits/chosen": -3.611408233642578,
      "logits/rejected": -3.763216972351074,
      "logps/chosen": -0.8894081115722656,
      "logps/rejected": -101.64353942871094,
      "loss": 0.1088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3988070487976074,
      "rewards/margins": 2.3988070487976074,
      "rewards/rejected": 0.0,
      "step": 1516
    },
    {
      "epoch": 8.474860335195531,
      "grad_norm": 1.7462214738717303,
      "learning_rate": 2.6781406875497334e-06,
      "logits/chosen": -3.3275065422058105,
      "logits/rejected": -3.4706504344940186,
      "logps/chosen": -3.054823637008667,
      "logps/rejected": -35.881439208984375,
      "loss": 0.1146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.677504062652588,
      "rewards/margins": 2.677504062652588,
      "rewards/rejected": 0.0,
      "step": 1517
    },
    {
      "epoch": 8.480446927374302,
      "grad_norm": 1.3274868574378063,
      "learning_rate": 2.6751005803235386e-06,
      "logits/chosen": -3.7082977294921875,
      "logits/rejected": -3.585683584213257,
      "logps/chosen": -3.116821765899658,
      "logps/rejected": -46.379024505615234,
      "loss": 0.1014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.01646089553833,
      "rewards/margins": 3.01646089553833,
      "rewards/rejected": 0.0,
      "step": 1518
    },
    {
      "epoch": 8.486033519553073,
      "grad_norm": 2.39595207824617,
      "learning_rate": 2.672060212866697e-06,
      "logits/chosen": -3.3693084716796875,
      "logits/rejected": -3.299992322921753,
      "logps/chosen": -1.1652400493621826,
      "logps/rejected": -106.39352416992188,
      "loss": 0.1131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4652018547058105,
      "rewards/margins": 2.4652018547058105,
      "rewards/rejected": 0.0,
      "step": 1519
    },
    {
      "epoch": 8.491620111731844,
      "grad_norm": 3.0805481576384284,
      "learning_rate": 2.669019589697737e-06,
      "logits/chosen": -3.5822062492370605,
      "logits/rejected": -3.725630760192871,
      "logps/chosen": -0.33949747681617737,
      "logps/rejected": -65.30386352539062,
      "loss": 0.1332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6086657047271729,
      "rewards/margins": 1.6086657047271729,
      "rewards/rejected": 0.0,
      "step": 1520
    },
    {
      "epoch": 8.491620111731844,
      "eval_logits/chosen": -3.41969633102417,
      "eval_logits/rejected": -3.5442123413085938,
      "eval_logps/chosen": -39.151519775390625,
      "eval_logps/rejected": -60.09822463989258,
      "eval_loss": 1.674224615097046,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -0.9081476330757141,
      "eval_rewards/margins": -0.9081476330757141,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5747,
      "eval_samples_per_second": 12.121,
      "eval_steps_per_second": 0.391,
      "step": 1520
    },
    {
      "epoch": 8.497206703910614,
      "grad_norm": 4.2054606587954995,
      "learning_rate": 2.665978715335566e-06,
      "logits/chosen": -3.6756653785705566,
      "logits/rejected": -3.783625841140747,
      "logps/chosen": -3.6759989261627197,
      "logps/rejected": -69.34093475341797,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1941280364990234,
      "rewards/margins": 3.1941280364990234,
      "rewards/rejected": 0.0,
      "step": 1521
    },
    {
      "epoch": 8.502793296089386,
      "grad_norm": 1.2597667793914389,
      "learning_rate": 2.6629375942994644e-06,
      "logits/chosen": -3.7643916606903076,
      "logits/rejected": -3.7357630729675293,
      "logps/chosen": -2.838388681411743,
      "logps/rejected": -69.65699005126953,
      "loss": 0.1125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.679844856262207,
      "rewards/margins": 2.679844856262207,
      "rewards/rejected": 0.0,
      "step": 1522
    },
    {
      "epoch": 8.508379888268156,
      "grad_norm": 6.986772303182529,
      "learning_rate": 2.659896231109081e-06,
      "logits/chosen": -3.493443012237549,
      "logits/rejected": -3.5422134399414062,
      "logps/chosen": -5.59070348739624,
      "logps/rejected": -42.92654037475586,
      "loss": 0.1174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.797053098678589,
      "rewards/margins": 2.797053098678589,
      "rewards/rejected": 0.0,
      "step": 1523
    },
    {
      "epoch": 8.513966480446927,
      "grad_norm": 3.567571524696719,
      "learning_rate": 2.6568546302844213e-06,
      "logits/chosen": -3.6094727516174316,
      "logits/rejected": -3.5562949180603027,
      "logps/chosen": -8.26915454864502,
      "logps/rejected": -45.09614181518555,
      "loss": 0.0951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4999563694000244,
      "rewards/margins": 3.4999563694000244,
      "rewards/rejected": 0.0,
      "step": 1524
    },
    {
      "epoch": 8.519553072625698,
      "grad_norm": 0.9748072901806756,
      "learning_rate": 2.653812796345847e-06,
      "logits/chosen": -3.5133981704711914,
      "logits/rejected": -3.533158779144287,
      "logps/chosen": -1.904573678970337,
      "logps/rejected": -41.02873992919922,
      "loss": 0.088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.041907787322998,
      "rewards/margins": 3.041907787322998,
      "rewards/rejected": 0.0,
      "step": 1525
    },
    {
      "epoch": 8.525139664804469,
      "grad_norm": 4.038181860512951,
      "learning_rate": 2.6507707338140653e-06,
      "logits/chosen": -3.518911838531494,
      "logits/rejected": -3.1509087085723877,
      "logps/chosen": -3.7440357208251953,
      "logps/rejected": -45.32233428955078,
      "loss": 0.0933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3859703540802,
      "rewards/margins": 3.3859703540802,
      "rewards/rejected": 0.0,
      "step": 1526
    },
    {
      "epoch": 8.53072625698324,
      "grad_norm": 1.0560094015440002,
      "learning_rate": 2.647728447210122e-06,
      "logits/chosen": -3.6893086433410645,
      "logits/rejected": -3.6511759757995605,
      "logps/chosen": -3.730302333831787,
      "logps/rejected": -39.756996154785156,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.877318859100342,
      "rewards/margins": 2.877318859100342,
      "rewards/rejected": 0.0,
      "step": 1527
    },
    {
      "epoch": 8.53631284916201,
      "grad_norm": 2.613012175013889,
      "learning_rate": 2.6446859410553972e-06,
      "logits/chosen": -3.499605417251587,
      "logits/rejected": -3.443293333053589,
      "logps/chosen": -3.2901456356048584,
      "logps/rejected": -60.15270233154297,
      "loss": 0.138,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.5509884357452393,
      "rewards/margins": 1.5509884357452393,
      "rewards/rejected": 0.0,
      "step": 1528
    },
    {
      "epoch": 8.541899441340782,
      "grad_norm": 3.6139671739459827,
      "learning_rate": 2.641643219871597e-06,
      "logits/chosen": -3.654892683029175,
      "logits/rejected": -3.8836495876312256,
      "logps/chosen": -1.575989842414856,
      "logps/rejected": -79.9444580078125,
      "loss": 0.1175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6612486839294434,
      "rewards/margins": 2.6612486839294434,
      "rewards/rejected": 0.0,
      "step": 1529
    },
    {
      "epoch": 8.547486033519553,
      "grad_norm": 1.7352751981174663,
      "learning_rate": 2.6386002881807465e-06,
      "logits/chosen": -3.512723684310913,
      "logits/rejected": -3.49920654296875,
      "logps/chosen": -0.8457837104797363,
      "logps/rejected": -111.80772399902344,
      "loss": 0.1069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.454284191131592,
      "rewards/margins": 2.454284191131592,
      "rewards/rejected": 0.0,
      "step": 1530
    },
    {
      "epoch": 8.553072625698324,
      "grad_norm": 1.4147377100430438,
      "learning_rate": 2.635557150505184e-06,
      "logits/chosen": -3.811117649078369,
      "logits/rejected": -3.523709535598755,
      "logps/chosen": -2.9019510746002197,
      "logps/rejected": -67.46919250488281,
      "loss": 0.0929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7863125801086426,
      "rewards/margins": 2.7863125801086426,
      "rewards/rejected": 0.0,
      "step": 1531
    },
    {
      "epoch": 8.558659217877095,
      "grad_norm": 2.0204059065471975,
      "learning_rate": 2.6325138113675547e-06,
      "logits/chosen": -3.370208740234375,
      "logits/rejected": -3.5299763679504395,
      "logps/chosen": -0.9528137445449829,
      "logps/rejected": -132.08938598632812,
      "loss": 0.116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6224901676177979,
      "rewards/margins": 1.6224901676177979,
      "rewards/rejected": 0.0,
      "step": 1532
    },
    {
      "epoch": 8.564245810055866,
      "grad_norm": 1.6800467893092341,
      "learning_rate": 2.629470275290803e-06,
      "logits/chosen": -3.7035539150238037,
      "logits/rejected": -3.7274081707000732,
      "logps/chosen": -3.2978785037994385,
      "logps/rejected": -61.30768966674805,
      "loss": 0.1088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.521111249923706,
      "rewards/margins": 2.521111249923706,
      "rewards/rejected": 0.0,
      "step": 1533
    },
    {
      "epoch": 8.569832402234637,
      "grad_norm": 0.9863320454888819,
      "learning_rate": 2.6264265467981637e-06,
      "logits/chosen": -3.630131483078003,
      "logits/rejected": -3.641563653945923,
      "logps/chosen": -23.508220672607422,
      "logps/rejected": -48.64502716064453,
      "loss": 0.0736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.480875492095947,
      "rewards/margins": 4.480875492095947,
      "rewards/rejected": 0.0,
      "step": 1534
    },
    {
      "epoch": 8.575418994413408,
      "grad_norm": 1.2360053339872976,
      "learning_rate": 2.623382630413161e-06,
      "logits/chosen": -3.730125665664673,
      "logits/rejected": -3.527583599090576,
      "logps/chosen": -10.285735130310059,
      "logps/rejected": -47.715301513671875,
      "loss": 0.101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.992774486541748,
      "rewards/margins": 3.992774486541748,
      "rewards/rejected": 0.0,
      "step": 1535
    },
    {
      "epoch": 8.581005586592179,
      "grad_norm": 3.632440496370386,
      "learning_rate": 2.6203385306595968e-06,
      "logits/chosen": -3.4103240966796875,
      "logits/rejected": -3.243973970413208,
      "logps/chosen": -11.181845664978027,
      "logps/rejected": -61.362796783447266,
      "loss": 0.1148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9717040061950684,
      "rewards/margins": 2.9717040061950684,
      "rewards/rejected": 0.0,
      "step": 1536
    },
    {
      "epoch": 8.58659217877095,
      "grad_norm": 3.027006391183029,
      "learning_rate": 2.617294252061545e-06,
      "logits/chosen": -3.6196792125701904,
      "logits/rejected": -3.5216732025146484,
      "logps/chosen": -1.745917797088623,
      "logps/rejected": -64.54409790039062,
      "loss": 0.1336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2990479469299316,
      "rewards/margins": 2.2990479469299316,
      "rewards/rejected": 0.0,
      "step": 1537
    },
    {
      "epoch": 8.59217877094972,
      "grad_norm": 2.4125114026008907,
      "learning_rate": 2.6142497991433445e-06,
      "logits/chosen": -3.634298086166382,
      "logits/rejected": -3.571178674697876,
      "logps/chosen": -1.789893627166748,
      "logps/rejected": -63.82410430908203,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.451350688934326,
      "rewards/margins": 3.451350688934326,
      "rewards/rejected": 0.0,
      "step": 1538
    },
    {
      "epoch": 8.597765363128492,
      "grad_norm": 2.6671662061670642,
      "learning_rate": 2.6112051764295963e-06,
      "logits/chosen": -3.7983036041259766,
      "logits/rejected": -3.872203826904297,
      "logps/chosen": -1.7464205026626587,
      "logps/rejected": -54.16558074951172,
      "loss": 0.1079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5436596870422363,
      "rewards/margins": 3.5436596870422363,
      "rewards/rejected": 0.0,
      "step": 1539
    },
    {
      "epoch": 8.603351955307263,
      "grad_norm": 0.9568048866141464,
      "learning_rate": 2.6081603884451516e-06,
      "logits/chosen": -3.7311439514160156,
      "logits/rejected": -3.5566489696502686,
      "logps/chosen": -10.50998592376709,
      "logps/rejected": -59.53440856933594,
      "loss": 0.0576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9379031658172607,
      "rewards/margins": 3.9379031658172607,
      "rewards/rejected": 0.0,
      "step": 1540
    },
    {
      "epoch": 8.603351955307263,
      "eval_logits/chosen": -3.4055514335632324,
      "eval_logits/rejected": -3.523360013961792,
      "eval_logps/chosen": -37.543785095214844,
      "eval_logps/rejected": -57.6811408996582,
      "eval_loss": 1.507727026939392,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.7473741769790649,
      "eval_rewards/margins": -0.7473741769790649,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5594,
      "eval_samples_per_second": 12.129,
      "eval_steps_per_second": 0.391,
      "step": 1540
    },
    {
      "epoch": 8.608938547486034,
      "grad_norm": 1.1739105671759542,
      "learning_rate": 2.6051154397151068e-06,
      "logits/chosen": -3.7827975749969482,
      "logits/rejected": -3.676950693130493,
      "logps/chosen": -4.35445499420166,
      "logps/rejected": -52.82600402832031,
      "loss": 0.0861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6433558464050293,
      "rewards/margins": 3.6433558464050293,
      "rewards/rejected": 0.0,
      "step": 1541
    },
    {
      "epoch": 8.614525139664805,
      "grad_norm": 2.287708218841047,
      "learning_rate": 2.602070334764799e-06,
      "logits/chosen": -3.557152032852173,
      "logits/rejected": -3.597674608230591,
      "logps/chosen": -1.5062284469604492,
      "logps/rejected": -49.37000274658203,
      "loss": 0.1249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3009233474731445,
      "rewards/margins": 2.3009233474731445,
      "rewards/rejected": 0.0,
      "step": 1542
    },
    {
      "epoch": 8.620111731843576,
      "grad_norm": 2.361535776955546,
      "learning_rate": 2.599025078119796e-06,
      "logits/chosen": -3.688591480255127,
      "logits/rejected": -3.8464322090148926,
      "logps/chosen": -6.165342330932617,
      "logps/rejected": -62.73087692260742,
      "loss": 0.1212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3152492046356201,
      "rewards/margins": 1.3152492046356201,
      "rewards/rejected": 0.0,
      "step": 1543
    },
    {
      "epoch": 8.625698324022347,
      "grad_norm": 1.8217242222270686,
      "learning_rate": 2.595979674305891e-06,
      "logits/chosen": -3.5372087955474854,
      "logits/rejected": -3.4019129276275635,
      "logps/chosen": -4.7520976066589355,
      "logps/rejected": -57.49534225463867,
      "loss": 0.1091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7331607341766357,
      "rewards/margins": 3.7331607341766357,
      "rewards/rejected": 0.0,
      "step": 1544
    },
    {
      "epoch": 8.631284916201118,
      "grad_norm": 1.4772002939085853,
      "learning_rate": 2.5929341278490976e-06,
      "logits/chosen": -3.478128433227539,
      "logits/rejected": -3.074795722961426,
      "logps/chosen": -5.920810699462891,
      "logps/rejected": -74.96614074707031,
      "loss": 0.0789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.901327133178711,
      "rewards/margins": 2.901327133178711,
      "rewards/rejected": 0.0,
      "step": 1545
    },
    {
      "epoch": 8.636871508379889,
      "grad_norm": 1.5661122633163551,
      "learning_rate": 2.58988844327564e-06,
      "logits/chosen": -3.166557550430298,
      "logits/rejected": -3.108229875564575,
      "logps/chosen": -5.654612064361572,
      "logps/rejected": -66.01828002929688,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.263838768005371,
      "rewards/margins": 3.263838768005371,
      "rewards/rejected": 0.0,
      "step": 1546
    },
    {
      "epoch": 8.64245810055866,
      "grad_norm": 2.5411453329751947,
      "learning_rate": 2.5868426251119467e-06,
      "logits/chosen": -3.5209572315216064,
      "logits/rejected": -3.5456316471099854,
      "logps/chosen": -4.039822101593018,
      "logps/rejected": -57.46992111206055,
      "loss": 0.0985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.854198455810547,
      "rewards/margins": 3.854198455810547,
      "rewards/rejected": 0.0,
      "step": 1547
    },
    {
      "epoch": 8.64804469273743,
      "grad_norm": 2.754020596083281,
      "learning_rate": 2.583796677884647e-06,
      "logits/chosen": -3.1704142093658447,
      "logits/rejected": -3.054870128631592,
      "logps/chosen": -17.565780639648438,
      "logps/rejected": -46.973304748535156,
      "loss": 0.1166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.557734966278076,
      "rewards/margins": 3.557734966278076,
      "rewards/rejected": 0.0,
      "step": 1548
    },
    {
      "epoch": 8.653631284916202,
      "grad_norm": 2.19802435477049,
      "learning_rate": 2.58075060612056e-06,
      "logits/chosen": -3.7241220474243164,
      "logits/rejected": -3.6091885566711426,
      "logps/chosen": -0.385955274105072,
      "logps/rejected": -65.23695373535156,
      "loss": 0.1285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7608189582824707,
      "rewards/margins": 1.7608189582824707,
      "rewards/rejected": 0.0,
      "step": 1549
    },
    {
      "epoch": 8.659217877094973,
      "grad_norm": 3.1177150142959174,
      "learning_rate": 2.577704414346692e-06,
      "logits/chosen": -3.60054612159729,
      "logits/rejected": -3.637138843536377,
      "logps/chosen": -6.935941696166992,
      "logps/rejected": -43.17105484008789,
      "loss": 0.1266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.73331880569458,
      "rewards/margins": 2.73331880569458,
      "rewards/rejected": 0.0,
      "step": 1550
    },
    {
      "epoch": 8.664804469273744,
      "grad_norm": 4.216635381139763,
      "learning_rate": 2.5746581070902254e-06,
      "logits/chosen": -3.449320077896118,
      "logits/rejected": -3.2388927936553955,
      "logps/chosen": -4.776787281036377,
      "logps/rejected": -57.326377868652344,
      "loss": 0.1345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.620677947998047,
      "rewards/margins": 2.620677947998047,
      "rewards/rejected": 0.0,
      "step": 1551
    },
    {
      "epoch": 8.670391061452515,
      "grad_norm": 3.606959565537814,
      "learning_rate": 2.571611688878516e-06,
      "logits/chosen": -3.6217195987701416,
      "logits/rejected": -3.545945644378662,
      "logps/chosen": -2.935847043991089,
      "logps/rejected": -69.07091522216797,
      "loss": 0.1058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.912437915802002,
      "rewards/margins": 2.912437915802002,
      "rewards/rejected": 0.0,
      "step": 1552
    },
    {
      "epoch": 8.675977653631286,
      "grad_norm": 3.4258560940113814,
      "learning_rate": 2.568565164239083e-06,
      "logits/chosen": -3.4941234588623047,
      "logits/rejected": -3.439089059829712,
      "logps/chosen": -10.518077850341797,
      "logps/rejected": -68.21786499023438,
      "loss": 0.1433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.664921522140503,
      "rewards/margins": 3.664921522140503,
      "rewards/rejected": 0.0,
      "step": 1553
    },
    {
      "epoch": 8.681564245810057,
      "grad_norm": 2.1825877815230643,
      "learning_rate": 2.565518537699606e-06,
      "logits/chosen": -3.214479923248291,
      "logits/rejected": -3.161247491836548,
      "logps/chosen": -21.04151725769043,
      "logps/rejected": -83.88583374023438,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.155160427093506,
      "rewards/margins": 3.155160427093506,
      "rewards/rejected": 0.0,
      "step": 1554
    },
    {
      "epoch": 8.687150837988828,
      "grad_norm": 2.9600038867767275,
      "learning_rate": 2.5624718137879136e-06,
      "logits/chosen": -3.4247941970825195,
      "logits/rejected": -3.6640141010284424,
      "logps/chosen": -12.062772750854492,
      "logps/rejected": -69.23075866699219,
      "loss": 0.1358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.852980613708496,
      "rewards/margins": 3.852980613708496,
      "rewards/rejected": 0.0,
      "step": 1555
    },
    {
      "epoch": 8.692737430167599,
      "grad_norm": 2.5087744126980858,
      "learning_rate": 2.55942499703198e-06,
      "logits/chosen": -3.2515170574188232,
      "logits/rejected": -3.2639100551605225,
      "logps/chosen": -18.739761352539062,
      "logps/rejected": -67.07742309570312,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.372884750366211,
      "rewards/margins": 4.372884750366211,
      "rewards/rejected": 0.0,
      "step": 1556
    },
    {
      "epoch": 8.69832402234637,
      "grad_norm": 3.4341940412006937,
      "learning_rate": 2.5563780919599183e-06,
      "logits/chosen": -3.2914013862609863,
      "logits/rejected": -3.3461127281188965,
      "logps/chosen": -11.530704498291016,
      "logps/rejected": -69.9637451171875,
      "loss": 0.1173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.642434597015381,
      "rewards/margins": 2.642434597015381,
      "rewards/rejected": 0.0,
      "step": 1557
    },
    {
      "epoch": 8.703910614525139,
      "grad_norm": 12.32768769457262,
      "learning_rate": 2.5533311030999715e-06,
      "logits/chosen": -3.4198105335235596,
      "logits/rejected": -3.6282694339752197,
      "logps/chosen": -2.1106514930725098,
      "logps/rejected": -37.22623062133789,
      "loss": 0.1236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.070591688156128,
      "rewards/margins": 3.070591688156128,
      "rewards/rejected": 0.0,
      "step": 1558
    },
    {
      "epoch": 8.70949720670391,
      "grad_norm": 2.2422688524752306,
      "learning_rate": 2.5502840349805074e-06,
      "logits/chosen": -3.59513783454895,
      "logits/rejected": -3.5073933601379395,
      "logps/chosen": -2.1260929107666016,
      "logps/rejected": -76.25456237792969,
      "loss": 0.1079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6555097103118896,
      "rewards/margins": 2.6555097103118896,
      "rewards/rejected": 0.0,
      "step": 1559
    },
    {
      "epoch": 8.71508379888268,
      "grad_norm": 3.3345677769464728,
      "learning_rate": 2.547236892130012e-06,
      "logits/chosen": -3.3801772594451904,
      "logits/rejected": -3.534531354904175,
      "logps/chosen": -8.211731910705566,
      "logps/rejected": -63.61349105834961,
      "loss": 0.1233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.1710314750671387,
      "rewards/margins": 2.1710314750671387,
      "rewards/rejected": 0.0,
      "step": 1560
    },
    {
      "epoch": 8.71508379888268,
      "eval_logits/chosen": -3.3712356090545654,
      "eval_logits/rejected": -3.4837329387664795,
      "eval_logps/chosen": -36.37420654296875,
      "eval_logps/rejected": -56.35734939575195,
      "eval_loss": 1.396371603012085,
      "eval_rewards/accuracies": 0.30000001192092896,
      "eval_rewards/chosen": -0.6304165720939636,
      "eval_rewards/margins": -0.6304165720939636,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5766,
      "eval_samples_per_second": 12.12,
      "eval_steps_per_second": 0.391,
      "step": 1560
    },
    {
      "epoch": 8.720670391061452,
      "grad_norm": 1.5905347468718198,
      "learning_rate": 2.5441896790770826e-06,
      "logits/chosen": -3.822633743286133,
      "logits/rejected": -3.5464420318603516,
      "logps/chosen": -1.4218837022781372,
      "logps/rejected": -47.05280303955078,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.822071075439453,
      "rewards/margins": 2.822071075439453,
      "rewards/rejected": 0.0,
      "step": 1561
    },
    {
      "epoch": 8.726256983240223,
      "grad_norm": 1.404723317431034,
      "learning_rate": 2.541142400350421e-06,
      "logits/chosen": -3.4902172088623047,
      "logits/rejected": -3.410508871078491,
      "logps/chosen": -2.073394775390625,
      "logps/rejected": -47.63264465332031,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.712949514389038,
      "rewards/margins": 2.712949514389038,
      "rewards/rejected": 0.0,
      "step": 1562
    },
    {
      "epoch": 8.731843575418994,
      "grad_norm": 2.664623007600075,
      "learning_rate": 2.5380950604788245e-06,
      "logits/chosen": -3.4265425205230713,
      "logits/rejected": -3.486457109451294,
      "logps/chosen": -1.3141865730285645,
      "logps/rejected": -49.7767333984375,
      "loss": 0.1064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5481839179992676,
      "rewards/margins": 2.5481839179992676,
      "rewards/rejected": 0.0,
      "step": 1563
    },
    {
      "epoch": 8.737430167597765,
      "grad_norm": 2.4722779552483223,
      "learning_rate": 2.535047663991184e-06,
      "logits/chosen": -3.555300712585449,
      "logits/rejected": -3.5615170001983643,
      "logps/chosen": -12.433175086975098,
      "logps/rejected": -59.24013137817383,
      "loss": 0.1151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.81034517288208,
      "rewards/margins": 3.81034517288208,
      "rewards/rejected": 0.0,
      "step": 1564
    },
    {
      "epoch": 8.743016759776536,
      "grad_norm": 2.037035541374221,
      "learning_rate": 2.5320002154164735e-06,
      "logits/chosen": -3.717317819595337,
      "logits/rejected": -3.6899237632751465,
      "logps/chosen": -4.153696060180664,
      "logps/rejected": -46.19728088378906,
      "loss": 0.1227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.756897449493408,
      "rewards/margins": 3.756897449493408,
      "rewards/rejected": 0.0,
      "step": 1565
    },
    {
      "epoch": 8.748603351955307,
      "grad_norm": 4.205642601548032,
      "learning_rate": 2.528952719283744e-06,
      "logits/chosen": -3.613790512084961,
      "logits/rejected": -3.4666500091552734,
      "logps/chosen": -4.4737091064453125,
      "logps/rejected": -52.80308532714844,
      "loss": 0.1502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0527639389038086,
      "rewards/margins": 2.0527639389038086,
      "rewards/rejected": 0.0,
      "step": 1566
    },
    {
      "epoch": 8.754189944134078,
      "grad_norm": 1.4262190226518148,
      "learning_rate": 2.5259051801221176e-06,
      "logits/chosen": -3.48189377784729,
      "logits/rejected": -3.67895245552063,
      "logps/chosen": -14.629449844360352,
      "logps/rejected": -77.30708312988281,
      "loss": 0.1221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8406264781951904,
      "rewards/margins": 3.8406264781951904,
      "rewards/rejected": 0.0,
      "step": 1567
    },
    {
      "epoch": 8.759776536312849,
      "grad_norm": 1.2566643959701946,
      "learning_rate": 2.5228576024607797e-06,
      "logits/chosen": -3.616258144378662,
      "logits/rejected": -3.4685988426208496,
      "logps/chosen": -14.299919128417969,
      "logps/rejected": -50.056331634521484,
      "loss": 0.1016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2395143508911133,
      "rewards/margins": 3.2395143508911133,
      "rewards/rejected": 0.0,
      "step": 1568
    },
    {
      "epoch": 8.76536312849162,
      "grad_norm": 0.8734926784445393,
      "learning_rate": 2.5198099908289742e-06,
      "logits/chosen": -3.339543581008911,
      "logits/rejected": -3.224435329437256,
      "logps/chosen": -15.698992729187012,
      "logps/rejected": -78.56088256835938,
      "loss": 0.0786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.213607311248779,
      "rewards/margins": 4.213607311248779,
      "rewards/rejected": 0.0,
      "step": 1569
    },
    {
      "epoch": 8.77094972067039,
      "grad_norm": 4.45750202591776,
      "learning_rate": 2.5167623497559933e-06,
      "logits/chosen": -3.8722143173217773,
      "logits/rejected": -3.816788911819458,
      "logps/chosen": -8.380523681640625,
      "logps/rejected": -36.62765884399414,
      "loss": 0.1006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3050522804260254,
      "rewards/margins": 2.3050522804260254,
      "rewards/rejected": 0.0,
      "step": 1570
    },
    {
      "epoch": 8.776536312849162,
      "grad_norm": 1.562915784069867,
      "learning_rate": 2.5137146837711762e-06,
      "logits/chosen": -3.2481935024261475,
      "logits/rejected": -3.3793370723724365,
      "logps/chosen": -3.106114387512207,
      "logps/rejected": -55.32252883911133,
      "loss": 0.1292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.928974151611328,
      "rewards/margins": 2.928974151611328,
      "rewards/rejected": 0.0,
      "step": 1571
    },
    {
      "epoch": 8.782122905027933,
      "grad_norm": 0.9144558286545849,
      "learning_rate": 2.510666997403897e-06,
      "logits/chosen": -3.6538805961608887,
      "logits/rejected": -3.4918813705444336,
      "logps/chosen": -1.4865634441375732,
      "logps/rejected": -53.217689514160156,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6693599224090576,
      "rewards/margins": 2.6693599224090576,
      "rewards/rejected": 0.0,
      "step": 1572
    },
    {
      "epoch": 8.787709497206704,
      "grad_norm": 1.423005512639322,
      "learning_rate": 2.5076192951835585e-06,
      "logits/chosen": -3.446031093597412,
      "logits/rejected": -3.71781063079834,
      "logps/chosen": -1.6264525651931763,
      "logps/rejected": -49.07367706298828,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3281326293945312,
      "rewards/margins": 3.3281326293945312,
      "rewards/rejected": 0.0,
      "step": 1573
    },
    {
      "epoch": 8.793296089385475,
      "grad_norm": 2.510025428893756,
      "learning_rate": 2.5045715816395923e-06,
      "logits/chosen": -3.540785074234009,
      "logits/rejected": -3.6976959705352783,
      "logps/chosen": -0.5399941802024841,
      "logps/rejected": -72.07991027832031,
      "loss": 0.1337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.815924048423767,
      "rewards/margins": 1.815924048423767,
      "rewards/rejected": 0.0,
      "step": 1574
    },
    {
      "epoch": 8.798882681564246,
      "grad_norm": 1.8350631926967051,
      "learning_rate": 2.501523861301441e-06,
      "logits/chosen": -3.6166505813598633,
      "logits/rejected": -3.556051015853882,
      "logps/chosen": -6.444705486297607,
      "logps/rejected": -52.813880920410156,
      "loss": 0.104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1321091651916504,
      "rewards/margins": 3.1321091651916504,
      "rewards/rejected": 0.0,
      "step": 1575
    },
    {
      "epoch": 8.804469273743017,
      "grad_norm": 1.6567652531533414,
      "learning_rate": 2.4984761386985603e-06,
      "logits/chosen": -3.73422908782959,
      "logits/rejected": -3.524181365966797,
      "logps/chosen": -6.961304664611816,
      "logps/rejected": -79.25092315673828,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4176836013793945,
      "rewards/margins": 2.4176836013793945,
      "rewards/rejected": 0.0,
      "step": 1576
    },
    {
      "epoch": 8.810055865921788,
      "grad_norm": 1.2982115689135205,
      "learning_rate": 2.4954284183604085e-06,
      "logits/chosen": -3.6170034408569336,
      "logits/rejected": -3.555020570755005,
      "logps/chosen": -2.1881585121154785,
      "logps/rejected": -41.33513641357422,
      "loss": 0.1052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6514949798583984,
      "rewards/margins": 2.6514949798583984,
      "rewards/rejected": 0.0,
      "step": 1577
    },
    {
      "epoch": 8.815642458100559,
      "grad_norm": 3.5882110161265177,
      "learning_rate": 2.492380704816441e-06,
      "logits/chosen": -3.639709949493408,
      "logits/rejected": -3.582338333129883,
      "logps/chosen": -2.60422682762146,
      "logps/rejected": -54.09840393066406,
      "loss": 0.1299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2991325855255127,
      "rewards/margins": 3.2991325855255127,
      "rewards/rejected": 0.0,
      "step": 1578
    },
    {
      "epoch": 8.82122905027933,
      "grad_norm": 1.1827136283831017,
      "learning_rate": 2.489333002596104e-06,
      "logits/chosen": -3.6028923988342285,
      "logits/rejected": -3.5657222270965576,
      "logps/chosen": -25.17790985107422,
      "logps/rejected": -84.19961547851562,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.75345778465271,
      "rewards/margins": 3.75345778465271,
      "rewards/rejected": 0.0,
      "step": 1579
    },
    {
      "epoch": 8.8268156424581,
      "grad_norm": 3.596910978714065,
      "learning_rate": 2.486285316228824e-06,
      "logits/chosen": -3.5380654335021973,
      "logits/rejected": -3.4095938205718994,
      "logps/chosen": -9.625916481018066,
      "logps/rejected": -125.92970275878906,
      "loss": 0.1698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.199869155883789,
      "rewards/margins": 3.199869155883789,
      "rewards/rejected": 0.0,
      "step": 1580
    },
    {
      "epoch": 8.8268156424581,
      "eval_logits/chosen": -3.4126994609832764,
      "eval_logits/rejected": -3.5353426933288574,
      "eval_logps/chosen": -39.0997428894043,
      "eval_logps/rejected": -60.84949493408203,
      "eval_loss": 1.6609798669815063,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -0.9029701352119446,
      "eval_rewards/margins": -0.9029701352119446,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5365,
      "eval_samples_per_second": 12.139,
      "eval_steps_per_second": 0.392,
      "step": 1580
    },
    {
      "epoch": 8.832402234636872,
      "grad_norm": 1.8802611492426964,
      "learning_rate": 2.4832376502440067e-06,
      "logits/chosen": -3.482680082321167,
      "logits/rejected": -3.159538984298706,
      "logps/chosen": -4.927022457122803,
      "logps/rejected": -97.85589599609375,
      "loss": 0.1537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5578765869140625,
      "rewards/margins": 2.5578765869140625,
      "rewards/rejected": 0.0,
      "step": 1581
    },
    {
      "epoch": 8.837988826815643,
      "grad_norm": 1.5230454199066226,
      "learning_rate": 2.4801900091710274e-06,
      "logits/chosen": -3.308466672897339,
      "logits/rejected": -3.5091381072998047,
      "logps/chosen": -3.5210628509521484,
      "logps/rejected": -40.93510055541992,
      "loss": 0.1185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.491297960281372,
      "rewards/margins": 2.491297960281372,
      "rewards/rejected": 0.0,
      "step": 1582
    },
    {
      "epoch": 8.843575418994414,
      "grad_norm": 1.590809276460304,
      "learning_rate": 2.4771423975392207e-06,
      "logits/chosen": -3.3487393856048584,
      "logits/rejected": -3.3147225379943848,
      "logps/chosen": -12.321377754211426,
      "logps/rejected": -55.06529235839844,
      "loss": 0.1207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9932808876037598,
      "rewards/margins": 3.9932808876037598,
      "rewards/rejected": 0.0,
      "step": 1583
    },
    {
      "epoch": 8.849162011173185,
      "grad_norm": 4.062247017655583,
      "learning_rate": 2.4740948198778833e-06,
      "logits/chosen": -3.384859323501587,
      "logits/rejected": -3.3682804107666016,
      "logps/chosen": -37.9945182800293,
      "logps/rejected": -43.57602310180664,
      "loss": 0.1302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.56231689453125,
      "rewards/margins": 3.56231689453125,
      "rewards/rejected": 0.0,
      "step": 1584
    },
    {
      "epoch": 8.854748603351956,
      "grad_norm": 2.0151125179819553,
      "learning_rate": 2.4710472807162567e-06,
      "logits/chosen": -3.43684458732605,
      "logits/rejected": -3.372046709060669,
      "logps/chosen": -10.452337265014648,
      "logps/rejected": -50.176300048828125,
      "loss": 0.0929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8272757530212402,
      "rewards/margins": 3.8272757530212402,
      "rewards/rejected": 0.0,
      "step": 1585
    },
    {
      "epoch": 8.860335195530727,
      "grad_norm": 4.370459386468105,
      "learning_rate": 2.4679997845835273e-06,
      "logits/chosen": -3.831144332885742,
      "logits/rejected": -3.5808212757110596,
      "logps/chosen": -1.2491925954818726,
      "logps/rejected": -63.930381774902344,
      "loss": 0.136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.244367837905884,
      "rewards/margins": 2.244367837905884,
      "rewards/rejected": 0.0,
      "step": 1586
    },
    {
      "epoch": 8.865921787709498,
      "grad_norm": 2.4304205233124736,
      "learning_rate": 2.464952336008816e-06,
      "logits/chosen": -3.325908899307251,
      "logits/rejected": -3.2119204998016357,
      "logps/chosen": -4.026869773864746,
      "logps/rejected": -70.08206176757812,
      "loss": 0.1307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.167901992797852,
      "rewards/margins": 4.167901992797852,
      "rewards/rejected": 0.0,
      "step": 1587
    },
    {
      "epoch": 8.871508379888269,
      "grad_norm": 2.3182261896269747,
      "learning_rate": 2.4619049395211763e-06,
      "logits/chosen": -3.5078353881835938,
      "logits/rejected": -3.2411890029907227,
      "logps/chosen": -2.4770419597625732,
      "logps/rejected": -55.134178161621094,
      "loss": 0.0903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.450540542602539,
      "rewards/margins": 3.450540542602539,
      "rewards/rejected": 0.0,
      "step": 1588
    },
    {
      "epoch": 8.87709497206704,
      "grad_norm": 2.9219534349020013,
      "learning_rate": 2.4588575996495797e-06,
      "logits/chosen": -3.614839792251587,
      "logits/rejected": -3.513150930404663,
      "logps/chosen": -0.927389919757843,
      "logps/rejected": -40.221031188964844,
      "loss": 0.1322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0376744270324707,
      "rewards/margins": 2.0376744270324707,
      "rewards/rejected": 0.0,
      "step": 1589
    },
    {
      "epoch": 8.88268156424581,
      "grad_norm": 1.9029304500926165,
      "learning_rate": 2.4558103209229182e-06,
      "logits/chosen": -3.8720664978027344,
      "logits/rejected": -3.8261325359344482,
      "logps/chosen": -0.6672800779342651,
      "logps/rejected": -80.14985656738281,
      "loss": 0.1533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2662591934204102,
      "rewards/margins": 1.2662591934204102,
      "rewards/rejected": 0.0,
      "step": 1590
    },
    {
      "epoch": 8.888268156424582,
      "grad_norm": 1.7634061653199016,
      "learning_rate": 2.4527631078699885e-06,
      "logits/chosen": -3.5931620597839355,
      "logits/rejected": -3.477020502090454,
      "logps/chosen": -16.21788215637207,
      "logps/rejected": -60.155696868896484,
      "loss": 0.1251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.631615161895752,
      "rewards/margins": 3.631615161895752,
      "rewards/rejected": 0.0,
      "step": 1591
    },
    {
      "epoch": 8.893854748603353,
      "grad_norm": 3.490362098413876,
      "learning_rate": 2.449715965019493e-06,
      "logits/chosen": -3.373277187347412,
      "logits/rejected": -3.383537769317627,
      "logps/chosen": -14.246670722961426,
      "logps/rejected": -68.29085540771484,
      "loss": 0.1197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8464980125427246,
      "rewards/margins": 2.8464980125427246,
      "rewards/rejected": 0.0,
      "step": 1592
    },
    {
      "epoch": 8.899441340782122,
      "grad_norm": 4.695423862129022,
      "learning_rate": 2.4466688969000297e-06,
      "logits/chosen": -3.7495615482330322,
      "logits/rejected": -3.902355909347534,
      "logps/chosen": -4.328911781311035,
      "logps/rejected": -73.06156921386719,
      "loss": 0.1183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4384520053863525,
      "rewards/margins": 3.4384520053863525,
      "rewards/rejected": 0.0,
      "step": 1593
    },
    {
      "epoch": 8.905027932960895,
      "grad_norm": 1.4134770368336163,
      "learning_rate": 2.4436219080400825e-06,
      "logits/chosen": -3.357111930847168,
      "logits/rejected": -3.3764805793762207,
      "logps/chosen": -3.5745842456817627,
      "logps/rejected": -76.4804458618164,
      "loss": 0.1268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.867988109588623,
      "rewards/margins": 2.867988109588623,
      "rewards/rejected": 0.0,
      "step": 1594
    },
    {
      "epoch": 8.910614525139664,
      "grad_norm": 2.176119488878262,
      "learning_rate": 2.4405750029680207e-06,
      "logits/chosen": -3.0706160068511963,
      "logits/rejected": -3.034123420715332,
      "logps/chosen": -5.00052547454834,
      "logps/rejected": -51.0892333984375,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6546578407287598,
      "rewards/margins": 2.6546578407287598,
      "rewards/rejected": 0.0,
      "step": 1595
    },
    {
      "epoch": 8.916201117318435,
      "grad_norm": 1.8532556191027207,
      "learning_rate": 2.4375281862120872e-06,
      "logits/chosen": -3.453268527984619,
      "logits/rejected": -3.5261502265930176,
      "logps/chosen": -0.9184298515319824,
      "logps/rejected": -165.75672912597656,
      "loss": 0.0877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.368056297302246,
      "rewards/margins": 2.368056297302246,
      "rewards/rejected": 0.0,
      "step": 1596
    },
    {
      "epoch": 8.921787709497206,
      "grad_norm": 2.113705246699753,
      "learning_rate": 2.4344814623003948e-06,
      "logits/chosen": -3.635657548904419,
      "logits/rejected": -3.5226595401763916,
      "logps/chosen": -7.1199049949646,
      "logps/rejected": -48.57292938232422,
      "loss": 0.092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.439810276031494,
      "rewards/margins": 3.439810276031494,
      "rewards/rejected": 0.0,
      "step": 1597
    },
    {
      "epoch": 8.927374301675977,
      "grad_norm": 2.0719068560898313,
      "learning_rate": 2.431434835760917e-06,
      "logits/chosen": -3.3773372173309326,
      "logits/rejected": -3.439774990081787,
      "logps/chosen": -2.305224895477295,
      "logps/rejected": -58.95974349975586,
      "loss": 0.1135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5911073684692383,
      "rewards/margins": 2.5911073684692383,
      "rewards/rejected": 0.0,
      "step": 1598
    },
    {
      "epoch": 8.932960893854748,
      "grad_norm": 1.4870601244527688,
      "learning_rate": 2.4283883111214852e-06,
      "logits/chosen": -3.70503568649292,
      "logits/rejected": -3.888012409210205,
      "logps/chosen": -3.7565321922302246,
      "logps/rejected": -86.91462707519531,
      "loss": 0.1225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0723509788513184,
      "rewards/margins": 3.0723509788513184,
      "rewards/rejected": 0.0,
      "step": 1599
    },
    {
      "epoch": 8.938547486033519,
      "grad_norm": 1.6341399170090618,
      "learning_rate": 2.4253418929097754e-06,
      "logits/chosen": -3.5469861030578613,
      "logits/rejected": -3.6204020977020264,
      "logps/chosen": -9.50607967376709,
      "logps/rejected": -40.22233581542969,
      "loss": 0.1047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.065687656402588,
      "rewards/margins": 4.065687656402588,
      "rewards/rejected": 0.0,
      "step": 1600
    },
    {
      "epoch": 8.938547486033519,
      "eval_logits/chosen": -3.406601667404175,
      "eval_logits/rejected": -3.5261478424072266,
      "eval_logps/chosen": -37.751686096191406,
      "eval_logps/rejected": -59.648719787597656,
      "eval_loss": 1.5966306924819946,
      "eval_rewards/accuracies": 0.22499999403953552,
      "eval_rewards/chosen": -0.768164336681366,
      "eval_rewards/margins": -0.768164336681366,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5467,
      "eval_samples_per_second": 12.135,
      "eval_steps_per_second": 0.391,
      "step": 1600
    },
    {
      "epoch": 8.94413407821229,
      "grad_norm": 1.4426923069875368,
      "learning_rate": 2.4222955856533085e-06,
      "logits/chosen": -3.324084758758545,
      "logits/rejected": -3.346860885620117,
      "logps/chosen": -5.540281295776367,
      "logps/rejected": -58.69535446166992,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.783517360687256,
      "rewards/margins": 3.783517360687256,
      "rewards/rejected": 0.0,
      "step": 1601
    },
    {
      "epoch": 8.949720670391061,
      "grad_norm": 1.7755280617076714,
      "learning_rate": 2.4192493938794413e-06,
      "logits/chosen": -3.4509174823760986,
      "logits/rejected": -3.5872111320495605,
      "logps/chosen": -1.5066447257995605,
      "logps/rejected": -52.799583435058594,
      "loss": 0.0965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0031745433807373,
      "rewards/margins": 2.0031745433807373,
      "rewards/rejected": 0.0,
      "step": 1602
    },
    {
      "epoch": 8.955307262569832,
      "grad_norm": 1.787433821792082,
      "learning_rate": 2.4162033221153543e-06,
      "logits/chosen": -3.458743095397949,
      "logits/rejected": -3.6874516010284424,
      "logps/chosen": -2.8907113075256348,
      "logps/rejected": -72.11075592041016,
      "loss": 0.1124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3514716625213623,
      "rewards/margins": 2.3514716625213623,
      "rewards/rejected": 0.0,
      "step": 1603
    },
    {
      "epoch": 8.960893854748603,
      "grad_norm": 1.3825657436626122,
      "learning_rate": 2.413157374888054e-06,
      "logits/chosen": -3.4359607696533203,
      "logits/rejected": -3.329340696334839,
      "logps/chosen": -11.735916137695312,
      "logps/rejected": -55.90647506713867,
      "loss": 0.0785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.469033718109131,
      "rewards/margins": 3.469033718109131,
      "rewards/rejected": 0.0,
      "step": 1604
    },
    {
      "epoch": 8.966480446927374,
      "grad_norm": 1.6950838593948356,
      "learning_rate": 2.410111556724361e-06,
      "logits/chosen": -3.568549156188965,
      "logits/rejected": -3.533691167831421,
      "logps/chosen": -0.4438386857509613,
      "logps/rejected": -62.711639404296875,
      "loss": 0.158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4886550903320312,
      "rewards/margins": 1.4886550903320312,
      "rewards/rejected": 0.0,
      "step": 1605
    },
    {
      "epoch": 8.972067039106145,
      "grad_norm": 2.734610559901552,
      "learning_rate": 2.407065872150903e-06,
      "logits/chosen": -3.5166938304901123,
      "logits/rejected": -3.435047149658203,
      "logps/chosen": -2.71932315826416,
      "logps/rejected": -76.244873046875,
      "loss": 0.1248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.922492504119873,
      "rewards/margins": 2.922492504119873,
      "rewards/rejected": 0.0,
      "step": 1606
    },
    {
      "epoch": 8.977653631284916,
      "grad_norm": 7.702635573280518,
      "learning_rate": 2.404020325694109e-06,
      "logits/chosen": -3.4688665866851807,
      "logits/rejected": -3.512995481491089,
      "logps/chosen": -1.0453927516937256,
      "logps/rejected": -57.26140594482422,
      "loss": 0.1569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.484281539916992,
      "rewards/margins": 2.484281539916992,
      "rewards/rejected": 0.0,
      "step": 1607
    },
    {
      "epoch": 8.983240223463687,
      "grad_norm": 1.559057950526759,
      "learning_rate": 2.4009749218802052e-06,
      "logits/chosen": -3.6115097999572754,
      "logits/rejected": -3.5285885334014893,
      "logps/chosen": -4.126657009124756,
      "logps/rejected": -47.563316345214844,
      "loss": 0.1154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0110716819763184,
      "rewards/margins": 3.0110716819763184,
      "rewards/rejected": 0.0,
      "step": 1608
    },
    {
      "epoch": 8.988826815642458,
      "grad_norm": 1.97979461762736,
      "learning_rate": 2.3979296652352014e-06,
      "logits/chosen": -3.5183780193328857,
      "logits/rejected": -3.6166579723358154,
      "logps/chosen": -23.723848342895508,
      "logps/rejected": -56.634952545166016,
      "loss": 0.0969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.817300319671631,
      "rewards/margins": 3.817300319671631,
      "rewards/rejected": 0.0,
      "step": 1609
    },
    {
      "epoch": 8.994413407821229,
      "grad_norm": 3.7898743724826764,
      "learning_rate": 2.3948845602848932e-06,
      "logits/chosen": -3.2656071186065674,
      "logits/rejected": -3.0541229248046875,
      "logps/chosen": -3.3059325218200684,
      "logps/rejected": -74.19775390625,
      "loss": 0.1378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5556764602661133,
      "rewards/margins": 2.5556764602661133,
      "rewards/rejected": 0.0,
      "step": 1610
    },
    {
      "epoch": 9.0,
      "grad_norm": 1.1689640971471116,
      "learning_rate": 2.3918396115548497e-06,
      "logits/chosen": -3.431910514831543,
      "logits/rejected": -3.5563037395477295,
      "logps/chosen": -1.0543690919876099,
      "logps/rejected": -44.89818572998047,
      "loss": 0.1053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.735748529434204,
      "rewards/margins": 1.735748529434204,
      "rewards/rejected": 0.0,
      "step": 1611
    },
    {
      "epoch": 9.005586592178771,
      "grad_norm": 3.052464115005597,
      "learning_rate": 2.388794823570404e-06,
      "logits/chosen": -3.4936916828155518,
      "logits/rejected": -3.454435348510742,
      "logps/chosen": -3.249143123626709,
      "logps/rejected": -67.17393493652344,
      "loss": 0.1143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4307048320770264,
      "rewards/margins": 3.4307048320770264,
      "rewards/rejected": 0.0,
      "step": 1612
    },
    {
      "epoch": 9.011173184357542,
      "grad_norm": 1.6516806511026028,
      "learning_rate": 2.3857502008566555e-06,
      "logits/chosen": -3.797671318054199,
      "logits/rejected": -3.7592289447784424,
      "logps/chosen": -1.4175983667373657,
      "logps/rejected": -89.63457489013672,
      "loss": 0.0996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.384161949157715,
      "rewards/margins": 2.384161949157715,
      "rewards/rejected": 0.0,
      "step": 1613
    },
    {
      "epoch": 9.016759776536313,
      "grad_norm": 2.203331928910367,
      "learning_rate": 2.3827057479384565e-06,
      "logits/chosen": -3.3603482246398926,
      "logits/rejected": -3.5082414150238037,
      "logps/chosen": -0.5732930898666382,
      "logps/rejected": -55.587913513183594,
      "loss": 0.1216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1096343994140625,
      "rewards/margins": 2.1096343994140625,
      "rewards/rejected": 0.0,
      "step": 1614
    },
    {
      "epoch": 9.022346368715084,
      "grad_norm": 2.882170759541383,
      "learning_rate": 2.379661469340404e-06,
      "logits/chosen": -3.627676010131836,
      "logits/rejected": -3.5997185707092285,
      "logps/chosen": -2.505263328552246,
      "logps/rejected": -40.43620300292969,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.310203790664673,
      "rewards/margins": 2.310203790664673,
      "rewards/rejected": 0.0,
      "step": 1615
    },
    {
      "epoch": 9.027932960893855,
      "grad_norm": 1.1515162873187759,
      "learning_rate": 2.3766173695868393e-06,
      "logits/chosen": -3.309307098388672,
      "logits/rejected": -3.341676712036133,
      "logps/chosen": -1.6542015075683594,
      "logps/rejected": -69.48821258544922,
      "loss": 0.1085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5210418701171875,
      "rewards/margins": 2.5210418701171875,
      "rewards/rejected": 0.0,
      "step": 1616
    },
    {
      "epoch": 9.033519553072626,
      "grad_norm": 0.7789206014739528,
      "learning_rate": 2.373573453201837e-06,
      "logits/chosen": -3.2025108337402344,
      "logits/rejected": -3.246955394744873,
      "logps/chosen": -1.8630099296569824,
      "logps/rejected": -48.03994369506836,
      "loss": 0.1333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0722174644470215,
      "rewards/margins": 2.0722174644470215,
      "rewards/rejected": 0.0,
      "step": 1617
    },
    {
      "epoch": 9.039106145251397,
      "grad_norm": 3.1024781691084664,
      "learning_rate": 2.370529724709198e-06,
      "logits/chosen": -3.798518657684326,
      "logits/rejected": -3.787907361984253,
      "logps/chosen": -3.284306526184082,
      "logps/rejected": -48.05266571044922,
      "loss": 0.0964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6928749084472656,
      "rewards/margins": 2.6928749084472656,
      "rewards/rejected": 0.0,
      "step": 1618
    },
    {
      "epoch": 9.044692737430168,
      "grad_norm": 1.184714136303844,
      "learning_rate": 2.367486188632446e-06,
      "logits/chosen": -3.6040220260620117,
      "logits/rejected": -3.3848507404327393,
      "logps/chosen": -1.9549877643585205,
      "logps/rejected": -63.428958892822266,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.721909999847412,
      "rewards/margins": 2.721909999847412,
      "rewards/rejected": 0.0,
      "step": 1619
    },
    {
      "epoch": 9.050279329608939,
      "grad_norm": 1.8674328113671244,
      "learning_rate": 2.364442849494817e-06,
      "logits/chosen": -3.518699884414673,
      "logits/rejected": -3.628020763397217,
      "logps/chosen": -0.7302020788192749,
      "logps/rejected": -55.28338623046875,
      "loss": 0.0885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.284733772277832,
      "rewards/margins": 2.284733772277832,
      "rewards/rejected": 0.0,
      "step": 1620
    },
    {
      "epoch": 9.050279329608939,
      "eval_logits/chosen": -3.40385365486145,
      "eval_logits/rejected": -3.5198073387145996,
      "eval_logps/chosen": -39.21417236328125,
      "eval_logps/rejected": -59.341651916503906,
      "eval_loss": 1.594645619392395,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -0.9144126772880554,
      "eval_rewards/margins": -0.9144126772880554,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5827,
      "eval_samples_per_second": 12.118,
      "eval_steps_per_second": 0.391,
      "step": 1620
    },
    {
      "epoch": 9.05586592178771,
      "grad_norm": 2.160888485491497,
      "learning_rate": 2.3613997118192544e-06,
      "logits/chosen": -3.7464439868927,
      "logits/rejected": -3.746434450149536,
      "logps/chosen": -1.8106029033660889,
      "logps/rejected": -64.06732940673828,
      "loss": 0.1077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.842400312423706,
      "rewards/margins": 2.842400312423706,
      "rewards/rejected": 0.0,
      "step": 1621
    },
    {
      "epoch": 9.061452513966481,
      "grad_norm": 0.7936503390298725,
      "learning_rate": 2.3583567801284048e-06,
      "logits/chosen": -3.7379863262176514,
      "logits/rejected": -3.558830738067627,
      "logps/chosen": -0.6565865278244019,
      "logps/rejected": -49.42070007324219,
      "loss": 0.0774,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.234187126159668,
      "rewards/margins": 2.234187126159668,
      "rewards/rejected": 0.0,
      "step": 1622
    },
    {
      "epoch": 9.067039106145252,
      "grad_norm": 1.055026940532643,
      "learning_rate": 2.3553140589446036e-06,
      "logits/chosen": -3.4628238677978516,
      "logits/rejected": -3.563810348510742,
      "logps/chosen": -3.68648099899292,
      "logps/rejected": -85.22700500488281,
      "loss": 0.1088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.970479726791382,
      "rewards/margins": 3.970479726791382,
      "rewards/rejected": 0.0,
      "step": 1623
    },
    {
      "epoch": 9.072625698324023,
      "grad_norm": 0.6080991727759464,
      "learning_rate": 2.3522715527898786e-06,
      "logits/chosen": -3.6163055896759033,
      "logits/rejected": -3.7806084156036377,
      "logps/chosen": -2.9546899795532227,
      "logps/rejected": -66.51451110839844,
      "loss": 0.0886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.942030429840088,
      "rewards/margins": 3.942030429840088,
      "rewards/rejected": 0.0,
      "step": 1624
    },
    {
      "epoch": 9.078212290502794,
      "grad_norm": 0.8199099246295025,
      "learning_rate": 2.3492292661859355e-06,
      "logits/chosen": -3.276588201522827,
      "logits/rejected": -3.3292946815490723,
      "logps/chosen": -0.32965853810310364,
      "logps/rejected": -65.94359588623047,
      "loss": 0.1147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1924054622650146,
      "rewards/margins": 2.1924054622650146,
      "rewards/rejected": 0.0,
      "step": 1625
    },
    {
      "epoch": 9.083798882681565,
      "grad_norm": 0.9194262601970525,
      "learning_rate": 2.3461872036541537e-06,
      "logits/chosen": -3.5228805541992188,
      "logits/rejected": -3.520357370376587,
      "logps/chosen": -3.205693244934082,
      "logps/rejected": -62.89661407470703,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.131617546081543,
      "rewards/margins": 3.131617546081543,
      "rewards/rejected": 0.0,
      "step": 1626
    },
    {
      "epoch": 9.089385474860336,
      "grad_norm": 3.7624597621365194,
      "learning_rate": 2.343145369715579e-06,
      "logits/chosen": -3.514500379562378,
      "logits/rejected": -3.4188716411590576,
      "logps/chosen": -8.6160306930542,
      "logps/rejected": -67.78939819335938,
      "loss": 0.1237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.274564027786255,
      "rewards/margins": 3.274564027786255,
      "rewards/rejected": 0.0,
      "step": 1627
    },
    {
      "epoch": 9.094972067039107,
      "grad_norm": 0.6038307670307891,
      "learning_rate": 2.3401037688909202e-06,
      "logits/chosen": -3.913691997528076,
      "logits/rejected": -3.886922836303711,
      "logps/chosen": -0.7538437843322754,
      "logps/rejected": -62.81434631347656,
      "loss": 0.0901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.881965160369873,
      "rewards/margins": 2.881965160369873,
      "rewards/rejected": 0.0,
      "step": 1628
    },
    {
      "epoch": 9.100558659217878,
      "grad_norm": 0.4337304883130042,
      "learning_rate": 2.337062405700536e-06,
      "logits/chosen": -3.4580068588256836,
      "logits/rejected": -3.5764288902282715,
      "logps/chosen": -1.3856580257415771,
      "logps/rejected": -110.1051025390625,
      "loss": 0.0984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2571706771850586,
      "rewards/margins": 3.2571706771850586,
      "rewards/rejected": 0.0,
      "step": 1629
    },
    {
      "epoch": 9.106145251396647,
      "grad_norm": 2.940859432194797,
      "learning_rate": 2.3340212846644344e-06,
      "logits/chosen": -3.6664021015167236,
      "logits/rejected": -3.477585554122925,
      "logps/chosen": -3.8446922302246094,
      "logps/rejected": -72.87457275390625,
      "loss": 0.105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.446817398071289,
      "rewards/margins": 2.446817398071289,
      "rewards/rejected": 0.0,
      "step": 1630
    },
    {
      "epoch": 9.111731843575418,
      "grad_norm": 2.519356996419478,
      "learning_rate": 2.3309804103022642e-06,
      "logits/chosen": -3.806100845336914,
      "logits/rejected": -3.7497193813323975,
      "logps/chosen": -8.70201587677002,
      "logps/rejected": -75.24946594238281,
      "loss": 0.0918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6910476684570312,
      "rewards/margins": 3.6910476684570312,
      "rewards/rejected": 0.0,
      "step": 1631
    },
    {
      "epoch": 9.11731843575419,
      "grad_norm": 2.3510186148346732,
      "learning_rate": 2.3279397871333037e-06,
      "logits/chosen": -3.728534698486328,
      "logits/rejected": -3.5846972465515137,
      "logps/chosen": -15.302197456359863,
      "logps/rejected": -60.07221221923828,
      "loss": 0.1203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.491238594055176,
      "rewards/margins": 3.491238594055176,
      "rewards/rejected": 0.0,
      "step": 1632
    },
    {
      "epoch": 9.12290502793296,
      "grad_norm": 0.9605223609236567,
      "learning_rate": 2.3248994196764618e-06,
      "logits/chosen": -3.7480220794677734,
      "logits/rejected": -3.7848281860351562,
      "logps/chosen": -1.881086826324463,
      "logps/rejected": -30.790706634521484,
      "loss": 0.0915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9885525703430176,
      "rewards/margins": 2.9885525703430176,
      "rewards/rejected": 0.0,
      "step": 1633
    },
    {
      "epoch": 9.128491620111731,
      "grad_norm": 5.833593244620963,
      "learning_rate": 2.3218593124502674e-06,
      "logits/chosen": -3.4199411869049072,
      "logits/rejected": -3.513794183731079,
      "logps/chosen": -3.001040458679199,
      "logps/rejected": -65.4173355102539,
      "loss": 0.1241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3733811378479004,
      "rewards/margins": 3.3733811378479004,
      "rewards/rejected": 0.0,
      "step": 1634
    },
    {
      "epoch": 9.134078212290502,
      "grad_norm": 0.8563440769841334,
      "learning_rate": 2.3188194699728588e-06,
      "logits/chosen": -3.424293279647827,
      "logits/rejected": -3.4968008995056152,
      "logps/chosen": -24.99411392211914,
      "logps/rejected": -72.66301727294922,
      "loss": 0.0955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.703383445739746,
      "rewards/margins": 2.703383445739746,
      "rewards/rejected": 0.0,
      "step": 1635
    },
    {
      "epoch": 9.139664804469273,
      "grad_norm": 1.9297665055692155,
      "learning_rate": 2.315779896761984e-06,
      "logits/chosen": -3.690831184387207,
      "logits/rejected": -3.459622621536255,
      "logps/chosen": -13.55887508392334,
      "logps/rejected": -66.84770965576172,
      "loss": 0.1445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.274907112121582,
      "rewards/margins": 5.274907112121582,
      "rewards/rejected": 0.0,
      "step": 1636
    },
    {
      "epoch": 9.145251396648044,
      "grad_norm": 1.080052503152181,
      "learning_rate": 2.3127405973349915e-06,
      "logits/chosen": -3.6499524116516113,
      "logits/rejected": -3.57570743560791,
      "logps/chosen": -1.9004933834075928,
      "logps/rejected": -76.92044067382812,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3760273456573486,
      "rewards/margins": 3.3760273456573486,
      "rewards/rejected": 0.0,
      "step": 1637
    },
    {
      "epoch": 9.150837988826815,
      "grad_norm": 8.531693664819006,
      "learning_rate": 2.3097015762088193e-06,
      "logits/chosen": -3.782493829727173,
      "logits/rejected": -3.673072576522827,
      "logps/chosen": -10.448180198669434,
      "logps/rejected": -41.403751373291016,
      "loss": 0.1665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.3281080722808838,
      "rewards/margins": 1.3281080722808838,
      "rewards/rejected": 0.0,
      "step": 1638
    },
    {
      "epoch": 9.156424581005586,
      "grad_norm": 1.361149032725837,
      "learning_rate": 2.306662837899995e-06,
      "logits/chosen": -3.5015451908111572,
      "logits/rejected": -3.594649076461792,
      "logps/chosen": -2.9258387088775635,
      "logps/rejected": -60.191619873046875,
      "loss": 0.0767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7045557498931885,
      "rewards/margins": 2.7045557498931885,
      "rewards/rejected": 0.0,
      "step": 1639
    },
    {
      "epoch": 9.162011173184357,
      "grad_norm": 1.7287834801454813,
      "learning_rate": 2.3036243869246263e-06,
      "logits/chosen": -3.6976265907287598,
      "logits/rejected": -3.595087766647339,
      "logps/chosen": -0.9024338722229004,
      "logps/rejected": -64.70563507080078,
      "loss": 0.1218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.405177354812622,
      "rewards/margins": 2.405177354812622,
      "rewards/rejected": 0.0,
      "step": 1640
    },
    {
      "epoch": 9.162011173184357,
      "eval_logits/chosen": -3.414255142211914,
      "eval_logits/rejected": -3.5341694355010986,
      "eval_logps/chosen": -40.5899543762207,
      "eval_logps/rejected": -61.31243896484375,
      "eval_loss": 1.7280960083007812,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.0519908666610718,
      "eval_rewards/margins": -1.0519908666610718,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6643,
      "eval_samples_per_second": 12.079,
      "eval_steps_per_second": 0.39,
      "step": 1640
    },
    {
      "epoch": 9.167597765363128,
      "grad_norm": 2.432859669133643,
      "learning_rate": 2.3005862277983906e-06,
      "logits/chosen": -3.5159599781036377,
      "logits/rejected": -3.5702500343322754,
      "logps/chosen": -2.9325690269470215,
      "logps/rejected": -77.516845703125,
      "loss": 0.0933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4420039653778076,
      "rewards/margins": 2.4420039653778076,
      "rewards/rejected": 0.0,
      "step": 1641
    },
    {
      "epoch": 9.1731843575419,
      "grad_norm": 2.197832413058362,
      "learning_rate": 2.2975483650365334e-06,
      "logits/chosen": -3.2131004333496094,
      "logits/rejected": -3.2327654361724854,
      "logps/chosen": -18.257770538330078,
      "logps/rejected": -53.571311950683594,
      "loss": 0.1294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.203636407852173,
      "rewards/margins": 3.203636407852173,
      "rewards/rejected": 0.0,
      "step": 1642
    },
    {
      "epoch": 9.17877094972067,
      "grad_norm": 2.2880929565891606,
      "learning_rate": 2.2945108031538615e-06,
      "logits/chosen": -3.151548385620117,
      "logits/rejected": -3.026824474334717,
      "logps/chosen": -1.2461849451065063,
      "logps/rejected": -78.31380462646484,
      "loss": 0.1135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.166977882385254,
      "rewards/margins": 2.166977882385254,
      "rewards/rejected": 0.0,
      "step": 1643
    },
    {
      "epoch": 9.184357541899441,
      "grad_norm": 1.342278643959503,
      "learning_rate": 2.2914735466647307e-06,
      "logits/chosen": -3.6616218090057373,
      "logits/rejected": -3.4936952590942383,
      "logps/chosen": -4.598599910736084,
      "logps/rejected": -42.816654205322266,
      "loss": 0.0943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.718451976776123,
      "rewards/margins": 3.718451976776123,
      "rewards/rejected": 0.0,
      "step": 1644
    },
    {
      "epoch": 9.189944134078212,
      "grad_norm": 2.1333782490844437,
      "learning_rate": 2.2884366000830467e-06,
      "logits/chosen": -3.611548662185669,
      "logits/rejected": -3.5795958042144775,
      "logps/chosen": -10.667173385620117,
      "logps/rejected": -60.856101989746094,
      "loss": 0.1041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.165379047393799,
      "rewards/margins": 4.165379047393799,
      "rewards/rejected": 0.0,
      "step": 1645
    },
    {
      "epoch": 9.195530726256983,
      "grad_norm": 0.7868430441098946,
      "learning_rate": 2.2853999679222534e-06,
      "logits/chosen": -3.552905321121216,
      "logits/rejected": -3.388395309448242,
      "logps/chosen": -12.222267150878906,
      "logps/rejected": -57.48219680786133,
      "loss": 0.1133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.396385669708252,
      "rewards/margins": 4.396385669708252,
      "rewards/rejected": 0.0,
      "step": 1646
    },
    {
      "epoch": 9.201117318435754,
      "grad_norm": 2.634418211824757,
      "learning_rate": 2.2823636546953243e-06,
      "logits/chosen": -3.5979654788970947,
      "logits/rejected": -3.458116054534912,
      "logps/chosen": -18.77823829650879,
      "logps/rejected": -98.61074829101562,
      "loss": 0.0727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.546763896942139,
      "rewards/margins": 4.546763896942139,
      "rewards/rejected": 0.0,
      "step": 1647
    },
    {
      "epoch": 9.206703910614525,
      "grad_norm": 1.231187024767857,
      "learning_rate": 2.2793276649147646e-06,
      "logits/chosen": -3.607403516769409,
      "logits/rejected": -3.68861985206604,
      "logps/chosen": -0.3631514012813568,
      "logps/rejected": -57.121002197265625,
      "loss": 0.1019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9118586778640747,
      "rewards/margins": 1.9118586778640747,
      "rewards/rejected": 0.0,
      "step": 1648
    },
    {
      "epoch": 9.212290502793296,
      "grad_norm": 3.007093595558698,
      "learning_rate": 2.276292003092593e-06,
      "logits/chosen": -3.7768948078155518,
      "logits/rejected": -3.886840581893921,
      "logps/chosen": -1.0003300905227661,
      "logps/rejected": -99.0381088256836,
      "loss": 0.0953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1396689414978027,
      "rewards/margins": 2.1396689414978027,
      "rewards/rejected": 0.0,
      "step": 1649
    },
    {
      "epoch": 9.217877094972067,
      "grad_norm": 2.6566008836182426,
      "learning_rate": 2.273256673740345e-06,
      "logits/chosen": -3.529860734939575,
      "logits/rejected": -3.548379421234131,
      "logps/chosen": -1.8689497709274292,
      "logps/rejected": -54.01527404785156,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.506218910217285,
      "rewards/margins": 2.506218910217285,
      "rewards/rejected": 0.0,
      "step": 1650
    },
    {
      "epoch": 9.223463687150838,
      "grad_norm": 2.914382153815242,
      "learning_rate": 2.270221681369062e-06,
      "logits/chosen": -3.6693472862243652,
      "logits/rejected": -3.641299247741699,
      "logps/chosen": -4.547330856323242,
      "logps/rejected": -57.28626251220703,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4626312255859375,
      "rewards/margins": 3.4626312255859375,
      "rewards/rejected": 0.0,
      "step": 1651
    },
    {
      "epoch": 9.22905027932961,
      "grad_norm": 2.227403996589314,
      "learning_rate": 2.2671870304892808e-06,
      "logits/chosen": -3.7049577236175537,
      "logits/rejected": -3.6530585289001465,
      "logps/chosen": -6.48146915435791,
      "logps/rejected": -84.55223083496094,
      "loss": 0.0928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.643707275390625,
      "rewards/margins": 2.643707275390625,
      "rewards/rejected": 0.0,
      "step": 1652
    },
    {
      "epoch": 9.23463687150838,
      "grad_norm": 5.509670466700544,
      "learning_rate": 2.2641527256110337e-06,
      "logits/chosen": -3.7345755100250244,
      "logits/rejected": -3.6697065830230713,
      "logps/chosen": -1.459064245223999,
      "logps/rejected": -101.71652221679688,
      "loss": 0.1471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.662590742111206,
      "rewards/margins": 1.662590742111206,
      "rewards/rejected": 0.0,
      "step": 1653
    },
    {
      "epoch": 9.240223463687151,
      "grad_norm": 2.350779090751588,
      "learning_rate": 2.2611187712438395e-06,
      "logits/chosen": -3.718533515930176,
      "logits/rejected": -3.7968356609344482,
      "logps/chosen": -8.674163818359375,
      "logps/rejected": -59.179466247558594,
      "loss": 0.0966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8430094718933105,
      "rewards/margins": 3.8430094718933105,
      "rewards/rejected": 0.0,
      "step": 1654
    },
    {
      "epoch": 9.245810055865922,
      "grad_norm": 1.0842065808348065,
      "learning_rate": 2.258085171896693e-06,
      "logits/chosen": -3.5704548358917236,
      "logits/rejected": -3.8369176387786865,
      "logps/chosen": -1.4180206060409546,
      "logps/rejected": -71.85863494873047,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6825294494628906,
      "rewards/margins": 3.6825294494628906,
      "rewards/rejected": 0.0,
      "step": 1655
    },
    {
      "epoch": 9.251396648044693,
      "grad_norm": 12.41553701831872,
      "learning_rate": 2.2550519320780638e-06,
      "logits/chosen": -3.8117623329162598,
      "logits/rejected": -3.7665083408355713,
      "logps/chosen": -2.448038339614868,
      "logps/rejected": -65.35214233398438,
      "loss": 0.139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.174436569213867,
      "rewards/margins": 3.174436569213867,
      "rewards/rejected": 0.0,
      "step": 1656
    },
    {
      "epoch": 9.256983240223464,
      "grad_norm": 1.249966508469423,
      "learning_rate": 2.2520190562958867e-06,
      "logits/chosen": -3.724407434463501,
      "logits/rejected": -3.5968785285949707,
      "logps/chosen": -2.9925925731658936,
      "logps/rejected": -38.75773620605469,
      "loss": 0.074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.690821647644043,
      "rewards/margins": 3.690821647644043,
      "rewards/rejected": 0.0,
      "step": 1657
    },
    {
      "epoch": 9.262569832402235,
      "grad_norm": 1.7578568392789182,
      "learning_rate": 2.2489865490575544e-06,
      "logits/chosen": -3.488719940185547,
      "logits/rejected": -3.4561522006988525,
      "logps/chosen": -2.3542962074279785,
      "logps/rejected": -53.026676177978516,
      "loss": 0.1114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.979828119277954,
      "rewards/margins": 1.979828119277954,
      "rewards/rejected": 0.0,
      "step": 1658
    },
    {
      "epoch": 9.268156424581006,
      "grad_norm": 1.015007866680141,
      "learning_rate": 2.245954414869913e-06,
      "logits/chosen": -3.721876621246338,
      "logits/rejected": -3.8726439476013184,
      "logps/chosen": -0.522975504398346,
      "logps/rejected": -120.0081787109375,
      "loss": 0.1251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3654541969299316,
      "rewards/margins": 2.3654541969299316,
      "rewards/rejected": 0.0,
      "step": 1659
    },
    {
      "epoch": 9.273743016759777,
      "grad_norm": 0.7276181973128844,
      "learning_rate": 2.2429226582392554e-06,
      "logits/chosen": -3.580920457839966,
      "logits/rejected": -3.6850361824035645,
      "logps/chosen": -1.103572130203247,
      "logps/rejected": -62.473628997802734,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0006725788116455,
      "rewards/margins": 3.0006725788116455,
      "rewards/rejected": 0.0,
      "step": 1660
    },
    {
      "epoch": 9.273743016759777,
      "eval_logits/chosen": -3.4381210803985596,
      "eval_logits/rejected": -3.5584728717803955,
      "eval_logps/chosen": -38.69251251220703,
      "eval_logps/rejected": -59.62604904174805,
      "eval_loss": 1.6338093280792236,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.8622466921806335,
      "eval_rewards/margins": -0.8622466921806335,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.4571,
      "eval_samples_per_second": 12.177,
      "eval_steps_per_second": 0.393,
      "step": 1660
    },
    {
      "epoch": 9.279329608938548,
      "grad_norm": 0.5253679511971978,
      "learning_rate": 2.23989128367131e-06,
      "logits/chosen": -3.583632707595825,
      "logits/rejected": -3.5278801918029785,
      "logps/chosen": -2.3266384601593018,
      "logps/rejected": -73.14852142333984,
      "loss": 0.0768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4136533737182617,
      "rewards/margins": 2.4136533737182617,
      "rewards/rejected": 0.0,
      "step": 1661
    },
    {
      "epoch": 9.28491620111732,
      "grad_norm": 0.8669222935107026,
      "learning_rate": 2.236860295671239e-06,
      "logits/chosen": -3.6186611652374268,
      "logits/rejected": -3.6361734867095947,
      "logps/chosen": -1.2373055219650269,
      "logps/rejected": -43.12721252441406,
      "loss": 0.1198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.620147705078125,
      "rewards/margins": 2.620147705078125,
      "rewards/rejected": 0.0,
      "step": 1662
    },
    {
      "epoch": 9.29050279329609,
      "grad_norm": 2.328728242018294,
      "learning_rate": 2.2338296987436327e-06,
      "logits/chosen": -3.481731414794922,
      "logits/rejected": -3.7780508995056152,
      "logps/chosen": -2.01535701751709,
      "logps/rejected": -50.25048065185547,
      "loss": 0.1231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2910890579223633,
      "rewards/margins": 2.2910890579223633,
      "rewards/rejected": 0.0,
      "step": 1663
    },
    {
      "epoch": 9.296089385474861,
      "grad_norm": 1.9826773869054934,
      "learning_rate": 2.2307994973924952e-06,
      "logits/chosen": -3.5662732124328613,
      "logits/rejected": -3.501845359802246,
      "logps/chosen": -0.8699676394462585,
      "logps/rejected": -75.76591491699219,
      "loss": 0.1222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.359041690826416,
      "rewards/margins": 2.359041690826416,
      "rewards/rejected": 0.0,
      "step": 1664
    },
    {
      "epoch": 9.30167597765363,
      "grad_norm": 2.6535068371826567,
      "learning_rate": 2.227769696121247e-06,
      "logits/chosen": -3.432183027267456,
      "logits/rejected": -3.4468228816986084,
      "logps/chosen": -3.9254648685455322,
      "logps/rejected": -45.52484130859375,
      "loss": 0.0856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.154658555984497,
      "rewards/margins": 3.154658555984497,
      "rewards/rejected": 0.0,
      "step": 1665
    },
    {
      "epoch": 9.307262569832401,
      "grad_norm": 3.9953968210723896,
      "learning_rate": 2.224740299432713e-06,
      "logits/chosen": -3.820068836212158,
      "logits/rejected": -3.5701370239257812,
      "logps/chosen": -0.5355900526046753,
      "logps/rejected": -52.439048767089844,
      "loss": 0.1651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9499369859695435,
      "rewards/margins": 1.9499369859695435,
      "rewards/rejected": 0.0,
      "step": 1666
    },
    {
      "epoch": 9.312849162011172,
      "grad_norm": 1.9419172906509212,
      "learning_rate": 2.221711311829114e-06,
      "logits/chosen": -3.800647258758545,
      "logits/rejected": -3.7483155727386475,
      "logps/chosen": -4.452126502990723,
      "logps/rejected": -42.062171936035156,
      "loss": 0.1313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.004647731781006,
      "rewards/margins": 3.004647731781006,
      "rewards/rejected": 0.0,
      "step": 1667
    },
    {
      "epoch": 9.318435754189943,
      "grad_norm": 2.35882978609257,
      "learning_rate": 2.2186827378120657e-06,
      "logits/chosen": -3.555814027786255,
      "logits/rejected": -3.48886775970459,
      "logps/chosen": -1.320338249206543,
      "logps/rejected": -61.497474670410156,
      "loss": 0.1269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.518723487854004,
      "rewards/margins": 3.518723487854004,
      "rewards/rejected": 0.0,
      "step": 1668
    },
    {
      "epoch": 9.324022346368714,
      "grad_norm": 2.078662978014028,
      "learning_rate": 2.21565458188257e-06,
      "logits/chosen": -3.6244492530822754,
      "logits/rejected": -3.5953261852264404,
      "logps/chosen": -2.3325719833374023,
      "logps/rejected": -57.63090133666992,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1896090507507324,
      "rewards/margins": 3.1896090507507324,
      "rewards/rejected": 0.0,
      "step": 1669
    },
    {
      "epoch": 9.329608938547485,
      "grad_norm": 1.3084292214996431,
      "learning_rate": 2.212626848541004e-06,
      "logits/chosen": -3.3515517711639404,
      "logits/rejected": -3.3154609203338623,
      "logps/chosen": -9.489517211914062,
      "logps/rejected": -47.17504119873047,
      "loss": 0.1032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.561506986618042,
      "rewards/margins": 3.561506986618042,
      "rewards/rejected": 0.0,
      "step": 1670
    },
    {
      "epoch": 9.335195530726256,
      "grad_norm": 2.014057014424956,
      "learning_rate": 2.209599542287119e-06,
      "logits/chosen": -3.836463212966919,
      "logits/rejected": -3.709714651107788,
      "logps/chosen": -2.5833492279052734,
      "logps/rejected": -93.7832260131836,
      "loss": 0.1202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8621363639831543,
      "rewards/margins": 2.8621363639831543,
      "rewards/rejected": 0.0,
      "step": 1671
    },
    {
      "epoch": 9.340782122905027,
      "grad_norm": 0.7378734505970255,
      "learning_rate": 2.206572667620033e-06,
      "logits/chosen": -3.2207324504852295,
      "logits/rejected": -3.554321527481079,
      "logps/chosen": -0.6592481136322021,
      "logps/rejected": -56.47861099243164,
      "loss": 0.0956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.510725975036621,
      "rewards/margins": 2.510725975036621,
      "rewards/rejected": 0.0,
      "step": 1672
    },
    {
      "epoch": 9.346368715083798,
      "grad_norm": 6.001149973127181,
      "learning_rate": 2.203546229038218e-06,
      "logits/chosen": -3.5250027179718018,
      "logits/rejected": -3.684813976287842,
      "logps/chosen": -4.779454708099365,
      "logps/rejected": -88.21293640136719,
      "loss": 0.1549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7424182891845703,
      "rewards/margins": 3.7424182891845703,
      "rewards/rejected": 0.0,
      "step": 1673
    },
    {
      "epoch": 9.35195530726257,
      "grad_norm": 4.699467499840913,
      "learning_rate": 2.2005202310395032e-06,
      "logits/chosen": -3.770249843597412,
      "logits/rejected": -3.4020276069641113,
      "logps/chosen": -0.3118252754211426,
      "logps/rejected": -52.806640625,
      "loss": 0.1111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.487989902496338,
      "rewards/margins": 2.487989902496338,
      "rewards/rejected": 0.0,
      "step": 1674
    },
    {
      "epoch": 9.35754189944134,
      "grad_norm": 2.9912875745923033,
      "learning_rate": 2.1974946781210597e-06,
      "logits/chosen": -3.7366278171539307,
      "logits/rejected": -3.8052477836608887,
      "logps/chosen": -1.284746527671814,
      "logps/rejected": -68.070556640625,
      "loss": 0.1265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.531336784362793,
      "rewards/margins": 2.531336784362793,
      "rewards/rejected": 0.0,
      "step": 1675
    },
    {
      "epoch": 9.363128491620111,
      "grad_norm": 2.1690129861774334,
      "learning_rate": 2.1944695747793972e-06,
      "logits/chosen": -3.5979037284851074,
      "logits/rejected": -3.4925341606140137,
      "logps/chosen": -0.12356159090995789,
      "logps/rejected": -76.82997131347656,
      "loss": 0.1387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7018721103668213,
      "rewards/margins": 1.7018721103668213,
      "rewards/rejected": 0.0,
      "step": 1676
    },
    {
      "epoch": 9.368715083798882,
      "grad_norm": 0.6236998478551795,
      "learning_rate": 2.191444925510359e-06,
      "logits/chosen": -3.865004539489746,
      "logits/rejected": -3.8961331844329834,
      "logps/chosen": -5.030838489532471,
      "logps/rejected": -60.273590087890625,
      "loss": 0.0939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1904730796813965,
      "rewards/margins": 3.1904730796813965,
      "rewards/rejected": 0.0,
      "step": 1677
    },
    {
      "epoch": 9.374301675977653,
      "grad_norm": 1.6341564384177885,
      "learning_rate": 2.188420734809112e-06,
      "logits/chosen": -3.695143938064575,
      "logits/rejected": -3.6784613132476807,
      "logps/chosen": -1.078840970993042,
      "logps/rejected": -66.71836853027344,
      "loss": 0.0963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.004619598388672,
      "rewards/margins": 3.004619598388672,
      "rewards/rejected": 0.0,
      "step": 1678
    },
    {
      "epoch": 9.379888268156424,
      "grad_norm": 2.2059412459841337,
      "learning_rate": 2.1853970071701415e-06,
      "logits/chosen": -3.459991693496704,
      "logits/rejected": -3.4940624237060547,
      "logps/chosen": -19.790143966674805,
      "logps/rejected": -57.421607971191406,
      "loss": 0.1518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9322509765625,
      "rewards/margins": 2.9322509765625,
      "rewards/rejected": 0.0,
      "step": 1679
    },
    {
      "epoch": 9.385474860335195,
      "grad_norm": 2.7467727386198026,
      "learning_rate": 2.1823737470872476e-06,
      "logits/chosen": -3.4871766567230225,
      "logits/rejected": -3.48354434967041,
      "logps/chosen": -0.5936512351036072,
      "logps/rejected": -62.8100700378418,
      "loss": 0.091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9888761043548584,
      "rewards/margins": 1.9888761043548584,
      "rewards/rejected": 0.0,
      "step": 1680
    },
    {
      "epoch": 9.385474860335195,
      "eval_logits/chosen": -3.441105365753174,
      "eval_logits/rejected": -3.565173625946045,
      "eval_logps/chosen": -39.373985290527344,
      "eval_logps/rejected": -60.01714324951172,
      "eval_loss": 1.6668157577514648,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.9303946495056152,
      "eval_rewards/margins": -0.9303946495056152,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5271,
      "eval_samples_per_second": 12.144,
      "eval_steps_per_second": 0.392,
      "step": 1680
    },
    {
      "epoch": 9.391061452513966,
      "grad_norm": 5.155034607605266,
      "learning_rate": 2.17935095905353e-06,
      "logits/chosen": -3.567507028579712,
      "logits/rejected": -3.618683099746704,
      "logps/chosen": -1.2278059720993042,
      "logps/rejected": -61.211753845214844,
      "loss": 0.1966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8072693347930908,
      "rewards/margins": 1.8072693347930908,
      "rewards/rejected": 0.0,
      "step": 1681
    },
    {
      "epoch": 9.396648044692737,
      "grad_norm": 0.8952907152572053,
      "learning_rate": 2.176328647561392e-06,
      "logits/chosen": -3.3998773097991943,
      "logits/rejected": -3.708226442337036,
      "logps/chosen": -1.632708191871643,
      "logps/rejected": -75.86611938476562,
      "loss": 0.0741,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1136205196380615,
      "rewards/margins": 3.1136205196380615,
      "rewards/rejected": 0.0,
      "step": 1682
    },
    {
      "epoch": 9.402234636871508,
      "grad_norm": 1.4397407623567748,
      "learning_rate": 2.1733068171025275e-06,
      "logits/chosen": -3.841458320617676,
      "logits/rejected": -3.8338463306427,
      "logps/chosen": -1.5665560960769653,
      "logps/rejected": -49.896114349365234,
      "loss": 0.1193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.077615261077881,
      "rewards/margins": 2.077615261077881,
      "rewards/rejected": 0.0,
      "step": 1683
    },
    {
      "epoch": 9.40782122905028,
      "grad_norm": 1.7239651870043156,
      "learning_rate": 2.170285472167913e-06,
      "logits/chosen": -3.5050041675567627,
      "logits/rejected": -3.6603105068206787,
      "logps/chosen": -0.9095799326896667,
      "logps/rejected": -58.076629638671875,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5005321502685547,
      "rewards/margins": 2.5005321502685547,
      "rewards/rejected": 0.0,
      "step": 1684
    },
    {
      "epoch": 9.41340782122905,
      "grad_norm": 0.7414761234409822,
      "learning_rate": 2.167264617247806e-06,
      "logits/chosen": -3.3546273708343506,
      "logits/rejected": -3.6719040870666504,
      "logps/chosen": -1.3030612468719482,
      "logps/rejected": -60.824092864990234,
      "loss": 0.0978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0881171226501465,
      "rewards/margins": 3.0881171226501465,
      "rewards/rejected": 0.0,
      "step": 1685
    },
    {
      "epoch": 9.418994413407821,
      "grad_norm": 2.450332091475145,
      "learning_rate": 2.164244256831735e-06,
      "logits/chosen": -3.5278868675231934,
      "logits/rejected": -3.559109926223755,
      "logps/chosen": -8.819181442260742,
      "logps/rejected": -50.67787170410156,
      "loss": 0.1625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.564448118209839,
      "rewards/margins": 2.564448118209839,
      "rewards/rejected": 0.0,
      "step": 1686
    },
    {
      "epoch": 9.424581005586592,
      "grad_norm": 2.048626292656879,
      "learning_rate": 2.1612243954084925e-06,
      "logits/chosen": -3.2902464866638184,
      "logits/rejected": -3.313823938369751,
      "logps/chosen": -19.599340438842773,
      "logps/rejected": -57.432220458984375,
      "loss": 0.1207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5325872898101807,
      "rewards/margins": 3.5325872898101807,
      "rewards/rejected": 0.0,
      "step": 1687
    },
    {
      "epoch": 9.430167597765363,
      "grad_norm": 0.6449076800501864,
      "learning_rate": 2.158205037466131e-06,
      "logits/chosen": -3.9791853427886963,
      "logits/rejected": -3.926405906677246,
      "logps/chosen": -3.6468160152435303,
      "logps/rejected": -43.13877868652344,
      "loss": 0.0664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.220911026000977,
      "rewards/margins": 4.220911026000977,
      "rewards/rejected": 0.0,
      "step": 1688
    },
    {
      "epoch": 9.435754189944134,
      "grad_norm": 1.2306821850977487,
      "learning_rate": 2.155186187491955e-06,
      "logits/chosen": -3.6030805110931396,
      "logits/rejected": -3.6477839946746826,
      "logps/chosen": -0.4762917459011078,
      "logps/rejected": -59.58253479003906,
      "loss": 0.123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9087982177734375,
      "rewards/margins": 1.9087982177734375,
      "rewards/rejected": 0.0,
      "step": 1689
    },
    {
      "epoch": 9.441340782122905,
      "grad_norm": 3.928204863384405,
      "learning_rate": 2.1521678499725116e-06,
      "logits/chosen": -3.634969711303711,
      "logits/rejected": -3.7370805740356445,
      "logps/chosen": -0.9355968832969666,
      "logps/rejected": -120.82345581054688,
      "loss": 0.0957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5232579708099365,
      "rewards/margins": 2.5232579708099365,
      "rewards/rejected": 0.0,
      "step": 1690
    },
    {
      "epoch": 9.446927374301676,
      "grad_norm": 2.9781109434091944,
      "learning_rate": 2.1491500293935883e-06,
      "logits/chosen": -3.5852699279785156,
      "logits/rejected": -3.739112138748169,
      "logps/chosen": -1.293238639831543,
      "logps/rejected": -106.46728515625,
      "loss": 0.1144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5322585105895996,
      "rewards/margins": 2.5322585105895996,
      "rewards/rejected": 0.0,
      "step": 1691
    },
    {
      "epoch": 9.452513966480447,
      "grad_norm": 0.7171873895661284,
      "learning_rate": 2.146132730240206e-06,
      "logits/chosen": -3.7122445106506348,
      "logits/rejected": -3.7622690200805664,
      "logps/chosen": -0.3273223638534546,
      "logps/rejected": -160.61907958984375,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7849228382110596,
      "rewards/margins": 1.7849228382110596,
      "rewards/rejected": 0.0,
      "step": 1692
    },
    {
      "epoch": 9.458100558659218,
      "grad_norm": 1.934126541328504,
      "learning_rate": 2.143115956996605e-06,
      "logits/chosen": -3.6998300552368164,
      "logits/rejected": -3.642587184906006,
      "logps/chosen": -1.8528621196746826,
      "logps/rejected": -66.40440368652344,
      "loss": 0.0753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7541375160217285,
      "rewards/margins": 3.7541375160217285,
      "rewards/rejected": 0.0,
      "step": 1693
    },
    {
      "epoch": 9.46368715083799,
      "grad_norm": 1.400211582475424,
      "learning_rate": 2.1400997141462505e-06,
      "logits/chosen": -3.57472825050354,
      "logits/rejected": -3.464585542678833,
      "logps/chosen": -4.856514930725098,
      "logps/rejected": -49.72746276855469,
      "loss": 0.129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0816569328308105,
      "rewards/margins": 3.0816569328308105,
      "rewards/rejected": 0.0,
      "step": 1694
    },
    {
      "epoch": 9.46927374301676,
      "grad_norm": 1.3858023736116918,
      "learning_rate": 2.137084006171815e-06,
      "logits/chosen": -3.429379940032959,
      "logits/rejected": -3.5766611099243164,
      "logps/chosen": -1.8888620138168335,
      "logps/rejected": -63.977806091308594,
      "loss": 0.1296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8171515464782715,
      "rewards/margins": 2.8171515464782715,
      "rewards/rejected": 0.0,
      "step": 1695
    },
    {
      "epoch": 9.474860335195531,
      "grad_norm": 0.7334651063864781,
      "learning_rate": 2.134068837555179e-06,
      "logits/chosen": -3.4126620292663574,
      "logits/rejected": -3.5817348957061768,
      "logps/chosen": -0.5675816535949707,
      "logps/rejected": -83.26220703125,
      "loss": 0.101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.599518060684204,
      "rewards/margins": 2.599518060684204,
      "rewards/rejected": 0.0,
      "step": 1696
    },
    {
      "epoch": 9.480446927374302,
      "grad_norm": 1.6228142848804827,
      "learning_rate": 2.131054212777418e-06,
      "logits/chosen": -3.3936867713928223,
      "logits/rejected": -3.5038180351257324,
      "logps/chosen": -2.649832248687744,
      "logps/rejected": -92.04671478271484,
      "loss": 0.0738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7255754470825195,
      "rewards/margins": 2.7255754470825195,
      "rewards/rejected": 0.0,
      "step": 1697
    },
    {
      "epoch": 9.486033519553073,
      "grad_norm": 0.8581790424483894,
      "learning_rate": 2.1280401363188034e-06,
      "logits/chosen": -3.7150018215179443,
      "logits/rejected": -3.6074836254119873,
      "logps/chosen": -5.8795318603515625,
      "logps/rejected": -67.60761260986328,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.842501401901245,
      "rewards/margins": 3.842501401901245,
      "rewards/rejected": 0.0,
      "step": 1698
    },
    {
      "epoch": 9.491620111731844,
      "grad_norm": 1.9319303476524727,
      "learning_rate": 2.1250266126587883e-06,
      "logits/chosen": -3.4628286361694336,
      "logits/rejected": -3.6700825691223145,
      "logps/chosen": -2.412719964981079,
      "logps/rejected": -43.326072692871094,
      "loss": 0.0877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7522010803222656,
      "rewards/margins": 2.7522010803222656,
      "rewards/rejected": 0.0,
      "step": 1699
    },
    {
      "epoch": 9.497206703910614,
      "grad_norm": 2.4443764148836853,
      "learning_rate": 2.122013646276005e-06,
      "logits/chosen": -3.5553832054138184,
      "logits/rejected": -3.544016122817993,
      "logps/chosen": -8.863181114196777,
      "logps/rejected": -73.97631072998047,
      "loss": 0.1289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.772766590118408,
      "rewards/margins": 2.772766590118408,
      "rewards/rejected": 0.0,
      "step": 1700
    },
    {
      "epoch": 9.497206703910614,
      "eval_logits/chosen": -3.4243533611297607,
      "eval_logits/rejected": -3.546977996826172,
      "eval_logps/chosen": -39.81416320800781,
      "eval_logps/rejected": -60.73054122924805,
      "eval_loss": 1.6957306861877441,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.9744120836257935,
      "eval_rewards/margins": -0.9744120836257935,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5484,
      "eval_samples_per_second": 12.134,
      "eval_steps_per_second": 0.391,
      "step": 1700
    },
    {
      "epoch": 9.502793296089386,
      "grad_norm": 2.6957120751164707,
      "learning_rate": 2.1190012416482607e-06,
      "logits/chosen": -3.56795334815979,
      "logits/rejected": -3.432469129562378,
      "logps/chosen": -3.2582926750183105,
      "logps/rejected": -42.11534881591797,
      "loss": 0.1399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4778330326080322,
      "rewards/margins": 2.4778330326080322,
      "rewards/rejected": 0.0,
      "step": 1701
    },
    {
      "epoch": 9.508379888268156,
      "grad_norm": 0.8341412949698791,
      "learning_rate": 2.1159894032525225e-06,
      "logits/chosen": -3.5943472385406494,
      "logits/rejected": -3.763629674911499,
      "logps/chosen": -10.602652549743652,
      "logps/rejected": -74.59021759033203,
      "loss": 0.0994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3031835556030273,
      "rewards/margins": 3.3031835556030273,
      "rewards/rejected": 0.0,
      "step": 1702
    },
    {
      "epoch": 9.513966480446927,
      "grad_norm": 1.095264927813998,
      "learning_rate": 2.1129781355649196e-06,
      "logits/chosen": -3.5978357791900635,
      "logits/rejected": -3.7278804779052734,
      "logps/chosen": -0.7776672840118408,
      "logps/rejected": -57.25032043457031,
      "loss": 0.1022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5711605548858643,
      "rewards/margins": 2.5711605548858643,
      "rewards/rejected": 0.0,
      "step": 1703
    },
    {
      "epoch": 9.519553072625698,
      "grad_norm": 6.304169270515268,
      "learning_rate": 2.1099674430607326e-06,
      "logits/chosen": -3.5876543521881104,
      "logits/rejected": -3.5124869346618652,
      "logps/chosen": -7.320429801940918,
      "logps/rejected": -68.33543395996094,
      "loss": 0.1399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9297529458999634,
      "rewards/margins": 1.9297529458999634,
      "rewards/rejected": 0.0,
      "step": 1704
    },
    {
      "epoch": 9.525139664804469,
      "grad_norm": 2.3013904035041977,
      "learning_rate": 2.106957330214386e-06,
      "logits/chosen": -3.6286487579345703,
      "logits/rejected": -3.701052665710449,
      "logps/chosen": -5.0448079109191895,
      "logps/rejected": -68.00605010986328,
      "loss": 0.1454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8659031391143799,
      "rewards/margins": 1.8659031391143799,
      "rewards/rejected": 0.0,
      "step": 1705
    },
    {
      "epoch": 9.53072625698324,
      "grad_norm": 2.4717045242503257,
      "learning_rate": 2.1039478014994445e-06,
      "logits/chosen": -3.5894932746887207,
      "logits/rejected": -3.555544137954712,
      "logps/chosen": -3.456618309020996,
      "logps/rejected": -88.71438598632812,
      "loss": 0.1401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3850841522216797,
      "rewards/margins": 2.3850841522216797,
      "rewards/rejected": 0.0,
      "step": 1706
    },
    {
      "epoch": 9.53631284916201,
      "grad_norm": 1.3313068383718942,
      "learning_rate": 2.1009388613886017e-06,
      "logits/chosen": -3.3233022689819336,
      "logits/rejected": -3.2706940174102783,
      "logps/chosen": -9.891947746276855,
      "logps/rejected": -40.292625427246094,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.648688316345215,
      "rewards/margins": 4.648688316345215,
      "rewards/rejected": 0.0,
      "step": 1707
    },
    {
      "epoch": 9.541899441340782,
      "grad_norm": 3.051122044200691,
      "learning_rate": 2.0979305143536797e-06,
      "logits/chosen": -3.4839072227478027,
      "logits/rejected": -3.5375120639801025,
      "logps/chosen": -0.9682869911193848,
      "logps/rejected": -51.149539947509766,
      "loss": 0.0896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.055271863937378,
      "rewards/margins": 3.055271863937378,
      "rewards/rejected": 0.0,
      "step": 1708
    },
    {
      "epoch": 9.547486033519553,
      "grad_norm": 1.9769537440995444,
      "learning_rate": 2.0949227648656194e-06,
      "logits/chosen": -3.527759313583374,
      "logits/rejected": -3.5296452045440674,
      "logps/chosen": -3.5549027919769287,
      "logps/rejected": -52.07046890258789,
      "loss": 0.0989,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.024549722671509,
      "rewards/margins": 3.024549722671509,
      "rewards/rejected": 0.0,
      "step": 1709
    },
    {
      "epoch": 9.553072625698324,
      "grad_norm": 2.371600924750159,
      "learning_rate": 2.0919156173944692e-06,
      "logits/chosen": -3.421851873397827,
      "logits/rejected": -3.414170503616333,
      "logps/chosen": -0.533899188041687,
      "logps/rejected": -90.62651062011719,
      "loss": 0.1133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2438912391662598,
      "rewards/margins": 2.2438912391662598,
      "rewards/rejected": 0.0,
      "step": 1710
    },
    {
      "epoch": 9.558659217877095,
      "grad_norm": 0.6024824407711961,
      "learning_rate": 2.088909076409388e-06,
      "logits/chosen": -3.6028342247009277,
      "logits/rejected": -3.6175148487091064,
      "logps/chosen": -13.611695289611816,
      "logps/rejected": -54.930084228515625,
      "loss": 0.1058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.657881736755371,
      "rewards/margins": 3.657881736755371,
      "rewards/rejected": 0.0,
      "step": 1711
    },
    {
      "epoch": 9.564245810055866,
      "grad_norm": 0.7136233285268685,
      "learning_rate": 2.0859031463786313e-06,
      "logits/chosen": -3.4779891967773438,
      "logits/rejected": -3.4374020099639893,
      "logps/chosen": -0.20293329656124115,
      "logps/rejected": -63.979331970214844,
      "loss": 0.0755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8592374324798584,
      "rewards/margins": 1.8592374324798584,
      "rewards/rejected": 0.0,
      "step": 1712
    },
    {
      "epoch": 9.569832402234637,
      "grad_norm": 1.014063505994714,
      "learning_rate": 2.0828978317695445e-06,
      "logits/chosen": -3.75144362449646,
      "logits/rejected": -3.8971214294433594,
      "logps/chosen": -1.0857157707214355,
      "logps/rejected": -61.45513916015625,
      "loss": 0.0846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.926173686981201,
      "rewards/margins": 2.926173686981201,
      "rewards/rejected": 0.0,
      "step": 1713
    },
    {
      "epoch": 9.575418994413408,
      "grad_norm": 0.9934295363934995,
      "learning_rate": 2.0798931370485627e-06,
      "logits/chosen": -3.6290013790130615,
      "logits/rejected": -3.7686238288879395,
      "logps/chosen": -1.0601145029067993,
      "logps/rejected": -66.40705108642578,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2547264099121094,
      "rewards/margins": 2.2547264099121094,
      "rewards/rejected": 0.0,
      "step": 1714
    },
    {
      "epoch": 9.581005586592179,
      "grad_norm": 1.1543883861921764,
      "learning_rate": 2.0768890666811957e-06,
      "logits/chosen": -3.2339441776275635,
      "logits/rejected": -3.50221848487854,
      "logps/chosen": -0.3189660310745239,
      "logps/rejected": -83.36631774902344,
      "loss": 0.1215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.726385474205017,
      "rewards/margins": 1.726385474205017,
      "rewards/rejected": 0.0,
      "step": 1715
    },
    {
      "epoch": 9.58659217877095,
      "grad_norm": 0.7702424796905194,
      "learning_rate": 2.0738856251320275e-06,
      "logits/chosen": -3.4078786373138428,
      "logits/rejected": -3.603174924850464,
      "logps/chosen": -1.271202564239502,
      "logps/rejected": -45.70873260498047,
      "loss": 0.0785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6816346645355225,
      "rewards/margins": 2.6816346645355225,
      "rewards/rejected": 0.0,
      "step": 1716
    },
    {
      "epoch": 9.59217877094972,
      "grad_norm": 0.8577078215213326,
      "learning_rate": 2.0708828168647066e-06,
      "logits/chosen": -3.4804952144622803,
      "logits/rejected": -3.5177009105682373,
      "logps/chosen": -3.0122082233428955,
      "logps/rejected": -64.20695495605469,
      "loss": 0.073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.139097690582275,
      "rewards/margins": 4.139097690582275,
      "rewards/rejected": 0.0,
      "step": 1717
    },
    {
      "epoch": 9.597765363128492,
      "grad_norm": 3.239322368252329,
      "learning_rate": 2.067880646341943e-06,
      "logits/chosen": -3.2165372371673584,
      "logits/rejected": -3.063971757888794,
      "logps/chosen": -0.5406145453453064,
      "logps/rejected": -62.86500549316406,
      "loss": 0.1113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6647865772247314,
      "rewards/margins": 1.6647865772247314,
      "rewards/rejected": 0.0,
      "step": 1718
    },
    {
      "epoch": 9.603351955307263,
      "grad_norm": 12.704337112471158,
      "learning_rate": 2.0648791180254933e-06,
      "logits/chosen": -3.7271459102630615,
      "logits/rejected": -3.5687432289123535,
      "logps/chosen": -0.9953093528747559,
      "logps/rejected": -44.918601989746094,
      "loss": 0.2204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2698493003845215,
      "rewards/margins": 2.2698493003845215,
      "rewards/rejected": 0.0,
      "step": 1719
    },
    {
      "epoch": 9.608938547486034,
      "grad_norm": 0.973908164162983,
      "learning_rate": 2.061878236376166e-06,
      "logits/chosen": -3.5811591148376465,
      "logits/rejected": -3.5550012588500977,
      "logps/chosen": -0.1348135769367218,
      "logps/rejected": -53.40940856933594,
      "loss": 0.1136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7503291368484497,
      "rewards/margins": 1.7503291368484497,
      "rewards/rejected": 0.0,
      "step": 1720
    },
    {
      "epoch": 9.608938547486034,
      "eval_logits/chosen": -3.3831043243408203,
      "eval_logits/rejected": -3.501887559890747,
      "eval_logps/chosen": -40.792747497558594,
      "eval_logps/rejected": -61.952392578125,
      "eval_loss": 1.7273166179656982,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -1.0722706317901611,
      "eval_rewards/margins": -1.0722706317901611,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5708,
      "eval_samples_per_second": 12.123,
      "eval_steps_per_second": 0.391,
      "step": 1720
    },
    {
      "epoch": 9.614525139664805,
      "grad_norm": 1.45136982757908,
      "learning_rate": 2.058878005853805e-06,
      "logits/chosen": -3.237309217453003,
      "logits/rejected": -3.1556177139282227,
      "logps/chosen": -0.5508144497871399,
      "logps/rejected": -47.702064514160156,
      "loss": 0.1037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.141294002532959,
      "rewards/margins": 2.141294002532959,
      "rewards/rejected": 0.0,
      "step": 1721
    },
    {
      "epoch": 9.620111731843576,
      "grad_norm": 2.291436903184443,
      "learning_rate": 2.055878430917286e-06,
      "logits/chosen": -3.9006667137145996,
      "logits/rejected": -3.8129847049713135,
      "logps/chosen": -5.1265082359313965,
      "logps/rejected": -53.741661071777344,
      "loss": 0.1439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9590446949005127,
      "rewards/margins": 1.9590446949005127,
      "rewards/rejected": 0.0,
      "step": 1722
    },
    {
      "epoch": 9.625698324022347,
      "grad_norm": 4.709701215037108,
      "learning_rate": 2.052879516024513e-06,
      "logits/chosen": -3.344435691833496,
      "logits/rejected": -3.186570405960083,
      "logps/chosen": -3.365818977355957,
      "logps/rejected": -39.984657287597656,
      "loss": 0.1217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.007880210876465,
      "rewards/margins": 3.007880210876465,
      "rewards/rejected": 0.0,
      "step": 1723
    },
    {
      "epoch": 9.631284916201118,
      "grad_norm": 3.0867915580899554,
      "learning_rate": 2.0498812656324064e-06,
      "logits/chosen": -3.3926563262939453,
      "logits/rejected": -3.3077445030212402,
      "logps/chosen": -2.011924982070923,
      "logps/rejected": -109.23329162597656,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.308654308319092,
      "rewards/margins": 2.308654308319092,
      "rewards/rejected": 0.0,
      "step": 1724
    },
    {
      "epoch": 9.636871508379889,
      "grad_norm": 0.8456232792979613,
      "learning_rate": 2.0468836841969012e-06,
      "logits/chosen": -3.153433322906494,
      "logits/rejected": -3.042431354522705,
      "logps/chosen": -7.5671868324279785,
      "logps/rejected": -88.5760498046875,
      "loss": 0.0719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.965087652206421,
      "rewards/margins": 2.965087652206421,
      "rewards/rejected": 0.0,
      "step": 1725
    },
    {
      "epoch": 9.64245810055866,
      "grad_norm": 0.6448877153896351,
      "learning_rate": 2.0438867761729365e-06,
      "logits/chosen": -3.390744209289551,
      "logits/rejected": -3.6386048793792725,
      "logps/chosen": -8.086464881896973,
      "logps/rejected": -88.62616729736328,
      "loss": 0.0863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.785282611846924,
      "rewards/margins": 3.785282611846924,
      "rewards/rejected": 0.0,
      "step": 1726
    },
    {
      "epoch": 9.64804469273743,
      "grad_norm": 1.3263803746630856,
      "learning_rate": 2.040890546014453e-06,
      "logits/chosen": -3.686408758163452,
      "logits/rejected": -3.481718063354492,
      "logps/chosen": -6.189998149871826,
      "logps/rejected": -62.560916900634766,
      "loss": 0.1007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.946810007095337,
      "rewards/margins": 2.946810007095337,
      "rewards/rejected": 0.0,
      "step": 1727
    },
    {
      "epoch": 9.653631284916202,
      "grad_norm": 1.68258027456716,
      "learning_rate": 2.03789499817438e-06,
      "logits/chosen": -3.58300518989563,
      "logits/rejected": -3.73553729057312,
      "logps/chosen": -5.234183311462402,
      "logps/rejected": -39.91857147216797,
      "loss": 0.0816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.087094783782959,
      "rewards/margins": 3.087094783782959,
      "rewards/rejected": 0.0,
      "step": 1728
    },
    {
      "epoch": 9.659217877094973,
      "grad_norm": 2.591727338240659,
      "learning_rate": 2.0349001371046354e-06,
      "logits/chosen": -3.6171152591705322,
      "logits/rejected": -3.598151922225952,
      "logps/chosen": -32.50074005126953,
      "logps/rejected": -74.1661148071289,
      "loss": 0.0672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.045294284820557,
      "rewards/margins": 4.045294284820557,
      "rewards/rejected": 0.0,
      "step": 1729
    },
    {
      "epoch": 9.664804469273744,
      "grad_norm": 0.5434985335832541,
      "learning_rate": 2.0319059672561183e-06,
      "logits/chosen": -3.2739815711975098,
      "logits/rejected": -3.326582908630371,
      "logps/chosen": -3.908116340637207,
      "logps/rejected": -80.24676513671875,
      "loss": 0.0885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6464459896087646,
      "rewards/margins": 3.6464459896087646,
      "rewards/rejected": 0.0,
      "step": 1730
    },
    {
      "epoch": 9.670391061452515,
      "grad_norm": 1.1534564458069194,
      "learning_rate": 2.028912493078696e-06,
      "logits/chosen": -3.5845532417297363,
      "logits/rejected": -3.839960813522339,
      "logps/chosen": -1.0789796113967896,
      "logps/rejected": -94.9745101928711,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8952884674072266,
      "rewards/margins": 2.8952884674072266,
      "rewards/rejected": 0.0,
      "step": 1731
    },
    {
      "epoch": 9.675977653631286,
      "grad_norm": 1.1294877420345548,
      "learning_rate": 2.0259197190212055e-06,
      "logits/chosen": -3.626410961151123,
      "logits/rejected": -3.4565577507019043,
      "logps/chosen": -2.1818482875823975,
      "logps/rejected": -114.11453247070312,
      "loss": 0.1187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.458317279815674,
      "rewards/margins": 2.458317279815674,
      "rewards/rejected": 0.0,
      "step": 1732
    },
    {
      "epoch": 9.681564245810057,
      "grad_norm": 0.6883587169048202,
      "learning_rate": 2.0229276495314416e-06,
      "logits/chosen": -3.4596238136291504,
      "logits/rejected": -3.6664371490478516,
      "logps/chosen": -2.5048840045928955,
      "logps/rejected": -58.94891357421875,
      "loss": 0.0726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1560089588165283,
      "rewards/margins": 3.1560089588165283,
      "rewards/rejected": 0.0,
      "step": 1733
    },
    {
      "epoch": 9.687150837988828,
      "grad_norm": 4.286830813498387,
      "learning_rate": 2.019936289056152e-06,
      "logits/chosen": -3.6720523834228516,
      "logits/rejected": -3.845510721206665,
      "logps/chosen": -8.020297050476074,
      "logps/rejected": -86.55177307128906,
      "loss": 0.123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.4591331481933594,
      "rewards/margins": 2.4591331481933594,
      "rewards/rejected": 0.0,
      "step": 1734
    },
    {
      "epoch": 9.692737430167599,
      "grad_norm": 1.2184284671817447,
      "learning_rate": 2.0169456420410323e-06,
      "logits/chosen": -3.878877878189087,
      "logits/rejected": -3.627819299697876,
      "logps/chosen": -0.8005980849266052,
      "logps/rejected": -105.78583526611328,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.057176113128662,
      "rewards/margins": 2.057176113128662,
      "rewards/rejected": 0.0,
      "step": 1735
    },
    {
      "epoch": 9.69832402234637,
      "grad_norm": 1.4195724424157423,
      "learning_rate": 2.013955712930715e-06,
      "logits/chosen": -3.5256600379943848,
      "logits/rejected": -3.655320405960083,
      "logps/chosen": -17.05972671508789,
      "logps/rejected": -70.8842544555664,
      "loss": 0.0911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8564741611480713,
      "rewards/margins": 3.8564741611480713,
      "rewards/rejected": 0.0,
      "step": 1736
    },
    {
      "epoch": 9.703910614525139,
      "grad_norm": 1.5252554806849477,
      "learning_rate": 2.010966506168768e-06,
      "logits/chosen": -3.546708822250366,
      "logits/rejected": -3.403407335281372,
      "logps/chosen": -0.395435094833374,
      "logps/rejected": -59.337799072265625,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.083359718322754,
      "rewards/margins": 2.083359718322754,
      "rewards/rejected": 0.0,
      "step": 1737
    },
    {
      "epoch": 9.70949720670391,
      "grad_norm": 2.135814087575283,
      "learning_rate": 2.007978026197686e-06,
      "logits/chosen": -3.562342882156372,
      "logits/rejected": -3.537172317504883,
      "logps/chosen": -3.149043560028076,
      "logps/rejected": -49.85601043701172,
      "loss": 0.1083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.195807933807373,
      "rewards/margins": 4.195807933807373,
      "rewards/rejected": 0.0,
      "step": 1738
    },
    {
      "epoch": 9.71508379888268,
      "grad_norm": 1.5003925688302089,
      "learning_rate": 2.00499027745888e-06,
      "logits/chosen": -3.8844969272613525,
      "logits/rejected": -3.8249549865722656,
      "logps/chosen": -7.079092025756836,
      "logps/rejected": -61.77274703979492,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.394224643707275,
      "rewards/margins": 4.394224643707275,
      "rewards/rejected": 0.0,
      "step": 1739
    },
    {
      "epoch": 9.720670391061452,
      "grad_norm": 2.736656216135218,
      "learning_rate": 2.0020032643926786e-06,
      "logits/chosen": -3.6104753017425537,
      "logits/rejected": -3.6250834465026855,
      "logps/chosen": -2.3800580501556396,
      "logps/rejected": -108.77660369873047,
      "loss": 0.1528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7459142208099365,
      "rewards/margins": 1.7459142208099365,
      "rewards/rejected": 0.0,
      "step": 1740
    },
    {
      "epoch": 9.720670391061452,
      "eval_logits/chosen": -3.395005702972412,
      "eval_logits/rejected": -3.516618251800537,
      "eval_logps/chosen": -39.49401092529297,
      "eval_logps/rejected": -61.44636154174805,
      "eval_loss": 1.72057044506073,
      "eval_rewards/accuracies": 0.22499999403953552,
      "eval_rewards/chosen": -0.942396342754364,
      "eval_rewards/margins": -0.942396342754364,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5888,
      "eval_samples_per_second": 12.115,
      "eval_steps_per_second": 0.391,
      "step": 1740
    },
    {
      "epoch": 9.726256983240223,
      "grad_norm": 1.754533234384826,
      "learning_rate": 1.999016991438315e-06,
      "logits/chosen": -3.5466787815093994,
      "logits/rejected": -3.703982353210449,
      "logps/chosen": -0.35999536514282227,
      "logps/rejected": -73.69461059570312,
      "loss": 0.0879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6924054622650146,
      "rewards/margins": 1.6924054622650146,
      "rewards/rejected": 0.0,
      "step": 1741
    },
    {
      "epoch": 9.731843575418994,
      "grad_norm": 13.539464213968925,
      "learning_rate": 1.996031463033922e-06,
      "logits/chosen": -3.4392919540405273,
      "logits/rejected": -3.603217363357544,
      "logps/chosen": -1.5499508380889893,
      "logps/rejected": -70.57502746582031,
      "loss": 0.1334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.445608139038086,
      "rewards/margins": 3.445608139038086,
      "rewards/rejected": 0.0,
      "step": 1742
    },
    {
      "epoch": 9.737430167597765,
      "grad_norm": 0.7492920220529853,
      "learning_rate": 1.993046683616527e-06,
      "logits/chosen": -3.147693157196045,
      "logits/rejected": -3.1780762672424316,
      "logps/chosen": -7.214130401611328,
      "logps/rejected": -50.16831588745117,
      "loss": 0.0867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.166717767715454,
      "rewards/margins": 3.166717767715454,
      "rewards/rejected": 0.0,
      "step": 1743
    },
    {
      "epoch": 9.743016759776536,
      "grad_norm": 5.4172658404680005,
      "learning_rate": 1.990062657622044e-06,
      "logits/chosen": -3.473858594894409,
      "logits/rejected": -3.3615047931671143,
      "logps/chosen": -5.392370223999023,
      "logps/rejected": -59.0214958190918,
      "loss": 0.1367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1769955158233643,
      "rewards/margins": 3.1769955158233643,
      "rewards/rejected": 0.0,
      "step": 1744
    },
    {
      "epoch": 9.748603351955307,
      "grad_norm": 0.7165973879879303,
      "learning_rate": 1.987079389485266e-06,
      "logits/chosen": -3.643000841140747,
      "logits/rejected": -3.66642427444458,
      "logps/chosen": -12.778093338012695,
      "logps/rejected": -61.354278564453125,
      "loss": 0.0965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6815385818481445,
      "rewards/margins": 3.6815385818481445,
      "rewards/rejected": 0.0,
      "step": 1745
    },
    {
      "epoch": 9.754189944134078,
      "grad_norm": 1.5801868072912815,
      "learning_rate": 1.984096883639861e-06,
      "logits/chosen": -3.5251827239990234,
      "logits/rejected": -3.499509811401367,
      "logps/chosen": -0.5946885943412781,
      "logps/rejected": -60.661865234375,
      "loss": 0.0936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.060838222503662,
      "rewards/margins": 2.060838222503662,
      "rewards/rejected": 0.0,
      "step": 1746
    },
    {
      "epoch": 9.759776536312849,
      "grad_norm": 1.136618517911191,
      "learning_rate": 1.981115144518366e-06,
      "logits/chosen": -3.564676284790039,
      "logits/rejected": -3.488727331161499,
      "logps/chosen": -1.7267844676971436,
      "logps/rejected": -61.6003532409668,
      "loss": 0.1053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.702542543411255,
      "rewards/margins": 2.702542543411255,
      "rewards/rejected": 0.0,
      "step": 1747
    },
    {
      "epoch": 9.76536312849162,
      "grad_norm": 2.8560291018620556,
      "learning_rate": 1.9781341765521728e-06,
      "logits/chosen": -3.6216495037078857,
      "logits/rejected": -3.6126089096069336,
      "logps/chosen": -5.482651710510254,
      "logps/rejected": -73.98180389404297,
      "loss": 0.1128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5165343284606934,
      "rewards/margins": 2.5165343284606934,
      "rewards/rejected": 0.0,
      "step": 1748
    },
    {
      "epoch": 9.77094972067039,
      "grad_norm": 0.5893631333958703,
      "learning_rate": 1.9751539841715323e-06,
      "logits/chosen": -3.7940385341644287,
      "logits/rejected": -3.829829454421997,
      "logps/chosen": -3.2233493328094482,
      "logps/rejected": -66.42029571533203,
      "loss": 0.0966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9632086753845215,
      "rewards/margins": 2.9632086753845215,
      "rewards/rejected": 0.0,
      "step": 1749
    },
    {
      "epoch": 9.776536312849162,
      "grad_norm": 4.229470747121671,
      "learning_rate": 1.972174571805543e-06,
      "logits/chosen": -3.690490484237671,
      "logits/rejected": -3.7901103496551514,
      "logps/chosen": -2.556723117828369,
      "logps/rejected": -64.4729995727539,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1672425270080566,
      "rewards/margins": 3.1672425270080566,
      "rewards/rejected": 0.0,
      "step": 1750
    },
    {
      "epoch": 9.782122905027933,
      "grad_norm": 2.938115874503743,
      "learning_rate": 1.969195943882139e-06,
      "logits/chosen": -3.4906649589538574,
      "logits/rejected": -3.3678529262542725,
      "logps/chosen": -9.666050910949707,
      "logps/rejected": -61.878997802734375,
      "loss": 0.1162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.979750633239746,
      "rewards/margins": 4.979750633239746,
      "rewards/rejected": 0.0,
      "step": 1751
    },
    {
      "epoch": 9.787709497206704,
      "grad_norm": 0.7417775574236315,
      "learning_rate": 1.966218104828094e-06,
      "logits/chosen": -3.4690492153167725,
      "logits/rejected": -3.4976236820220947,
      "logps/chosen": -14.973042488098145,
      "logps/rejected": -64.50141906738281,
      "loss": 0.083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6077113151550293,
      "rewards/margins": 3.6077113151550293,
      "rewards/rejected": 0.0,
      "step": 1752
    },
    {
      "epoch": 9.793296089385475,
      "grad_norm": 1.3761083105068592,
      "learning_rate": 1.963241059069007e-06,
      "logits/chosen": -3.4732108116149902,
      "logits/rejected": -3.375981330871582,
      "logps/chosen": -0.33565008640289307,
      "logps/rejected": -69.4214096069336,
      "loss": 0.1253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4838576316833496,
      "rewards/margins": 1.4838576316833496,
      "rewards/rejected": 0.0,
      "step": 1753
    },
    {
      "epoch": 9.798882681564246,
      "grad_norm": 1.1241608799658702,
      "learning_rate": 1.9602648110292972e-06,
      "logits/chosen": -3.311899423599243,
      "logits/rejected": -3.437587261199951,
      "logps/chosen": -29.829452514648438,
      "logps/rejected": -47.62665939331055,
      "loss": 0.0992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.0469865798950195,
      "rewards/margins": 4.0469865798950195,
      "rewards/rejected": 0.0,
      "step": 1754
    },
    {
      "epoch": 9.804469273743017,
      "grad_norm": 0.3677985349006169,
      "learning_rate": 1.9572893651321996e-06,
      "logits/chosen": -3.3179409503936768,
      "logits/rejected": -3.630552291870117,
      "logps/chosen": -0.3840661644935608,
      "logps/rejected": -71.98883056640625,
      "loss": 0.0687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3411030769348145,
      "rewards/margins": 2.3411030769348145,
      "rewards/rejected": 0.0,
      "step": 1755
    },
    {
      "epoch": 9.810055865921788,
      "grad_norm": 3.9730953445225508,
      "learning_rate": 1.954314725799758e-06,
      "logits/chosen": -3.4780025482177734,
      "logits/rejected": -3.270801305770874,
      "logps/chosen": -14.867898941040039,
      "logps/rejected": -50.972503662109375,
      "loss": 0.1298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.489175319671631,
      "rewards/margins": 3.489175319671631,
      "rewards/rejected": 0.0,
      "step": 1756
    },
    {
      "epoch": 9.815642458100559,
      "grad_norm": 1.7712170368183942,
      "learning_rate": 1.951340897452814e-06,
      "logits/chosen": -3.494570732116699,
      "logits/rejected": -3.4363961219787598,
      "logps/chosen": -1.788813829421997,
      "logps/rejected": -104.58145141601562,
      "loss": 0.1372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.576051712036133,
      "rewards/margins": 2.576051712036133,
      "rewards/rejected": 0.0,
      "step": 1757
    },
    {
      "epoch": 9.82122905027933,
      "grad_norm": 1.696198284073317,
      "learning_rate": 1.948367884511007e-06,
      "logits/chosen": -3.7457852363586426,
      "logits/rejected": -3.568553924560547,
      "logps/chosen": -8.227602005004883,
      "logps/rejected": -67.65357971191406,
      "loss": 0.1165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9555673599243164,
      "rewards/margins": 2.9555673599243164,
      "rewards/rejected": 0.0,
      "step": 1758
    },
    {
      "epoch": 9.8268156424581,
      "grad_norm": 0.8789899982739745,
      "learning_rate": 1.945395691392765e-06,
      "logits/chosen": -3.8350160121917725,
      "logits/rejected": -3.6438469886779785,
      "logps/chosen": -0.3920767307281494,
      "logps/rejected": -61.953102111816406,
      "loss": 0.1032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.725175380706787,
      "rewards/margins": 2.725175380706787,
      "rewards/rejected": 0.0,
      "step": 1759
    },
    {
      "epoch": 9.832402234636872,
      "grad_norm": 1.153178349869209,
      "learning_rate": 1.9424243225152944e-06,
      "logits/chosen": -3.688202142715454,
      "logits/rejected": -3.5744245052337646,
      "logps/chosen": -3.919034242630005,
      "logps/rejected": -56.5631103515625,
      "loss": 0.1409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2140867710113525,
      "rewards/margins": 3.2140867710113525,
      "rewards/rejected": 0.0,
      "step": 1760
    },
    {
      "epoch": 9.832402234636872,
      "eval_logits/chosen": -3.372711181640625,
      "eval_logits/rejected": -3.4895870685577393,
      "eval_logps/chosen": -37.82529830932617,
      "eval_logps/rejected": -58.75041961669922,
      "eval_loss": 1.539973497390747,
      "eval_rewards/accuracies": 0.22499999403953552,
      "eval_rewards/chosen": -0.7755261063575745,
      "eval_rewards/margins": -0.7755261063575745,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5222,
      "eval_samples_per_second": 12.146,
      "eval_steps_per_second": 0.392,
      "step": 1760
    },
    {
      "epoch": 9.837988826815643,
      "grad_norm": 1.9574521887081868,
      "learning_rate": 1.939453782294579e-06,
      "logits/chosen": -3.5007829666137695,
      "logits/rejected": -3.473370313644409,
      "logps/chosen": -1.6824532747268677,
      "logps/rejected": -48.41215133666992,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.111269474029541,
      "rewards/margins": 2.111269474029541,
      "rewards/rejected": 0.0,
      "step": 1761
    },
    {
      "epoch": 9.843575418994414,
      "grad_norm": 1.186206354635935,
      "learning_rate": 1.9364840751453724e-06,
      "logits/chosen": -3.4539520740509033,
      "logits/rejected": -3.5242929458618164,
      "logps/chosen": -2.308391571044922,
      "logps/rejected": -112.94505310058594,
      "loss": 0.0867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6572604179382324,
      "rewards/margins": 2.6572604179382324,
      "rewards/rejected": 0.0,
      "step": 1762
    },
    {
      "epoch": 9.849162011173185,
      "grad_norm": 0.6740216875976098,
      "learning_rate": 1.933515205481186e-06,
      "logits/chosen": -3.19785213470459,
      "logits/rejected": -2.9665911197662354,
      "logps/chosen": -19.810211181640625,
      "logps/rejected": -36.555355072021484,
      "loss": 0.0878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.291547775268555,
      "rewards/margins": 4.291547775268555,
      "rewards/rejected": 0.0,
      "step": 1763
    },
    {
      "epoch": 9.854748603351956,
      "grad_norm": 0.8858783617075582,
      "learning_rate": 1.93054717771429e-06,
      "logits/chosen": -3.580690383911133,
      "logits/rejected": -3.4249980449676514,
      "logps/chosen": -5.24107027053833,
      "logps/rejected": -61.088504791259766,
      "loss": 0.092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.881971836090088,
      "rewards/margins": 3.881971836090088,
      "rewards/rejected": 0.0,
      "step": 1764
    },
    {
      "epoch": 9.860335195530727,
      "grad_norm": 1.0283643960545774,
      "learning_rate": 1.9275799962557017e-06,
      "logits/chosen": -3.2603044509887695,
      "logits/rejected": -3.3185908794403076,
      "logps/chosen": -3.30397629737854,
      "logps/rejected": -80.71976470947266,
      "loss": 0.1284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0929181575775146,
      "rewards/margins": 2.0929181575775146,
      "rewards/rejected": 0.0,
      "step": 1765
    },
    {
      "epoch": 9.865921787709498,
      "grad_norm": 0.9545784378054664,
      "learning_rate": 1.9246136655151808e-06,
      "logits/chosen": -3.55584454536438,
      "logits/rejected": -3.598327398300171,
      "logps/chosen": -0.30297279357910156,
      "logps/rejected": -94.37016296386719,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.790191888809204,
      "rewards/margins": 1.790191888809204,
      "rewards/rejected": 0.0,
      "step": 1766
    },
    {
      "epoch": 9.871508379888269,
      "grad_norm": 2.9942625651557937,
      "learning_rate": 1.921648189901225e-06,
      "logits/chosen": -3.567455768585205,
      "logits/rejected": -3.5214009284973145,
      "logps/chosen": -8.146893501281738,
      "logps/rejected": -69.43839263916016,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2009024620056152,
      "rewards/margins": 3.2009024620056152,
      "rewards/rejected": 0.0,
      "step": 1767
    },
    {
      "epoch": 9.87709497206704,
      "grad_norm": 2.377560471557753,
      "learning_rate": 1.918683573821056e-06,
      "logits/chosen": -3.4094650745391846,
      "logits/rejected": -3.4492907524108887,
      "logps/chosen": -0.72724848985672,
      "logps/rejected": -53.80411148071289,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9964818954467773,
      "rewards/margins": 1.9964818954467773,
      "rewards/rejected": 0.0,
      "step": 1768
    },
    {
      "epoch": 9.88268156424581,
      "grad_norm": 3.8283593931444644,
      "learning_rate": 1.915719821680624e-06,
      "logits/chosen": -3.3991925716400146,
      "logits/rejected": -3.4163691997528076,
      "logps/chosen": -7.066792011260986,
      "logps/rejected": -61.86603927612305,
      "loss": 0.1223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7736358642578125,
      "rewards/margins": 2.7736358642578125,
      "rewards/rejected": 0.0,
      "step": 1769
    },
    {
      "epoch": 9.888268156424582,
      "grad_norm": 1.7040477792003483,
      "learning_rate": 1.912756937884592e-06,
      "logits/chosen": -3.569958448410034,
      "logits/rejected": -3.5412685871124268,
      "logps/chosen": -1.6545488834381104,
      "logps/rejected": -64.32032012939453,
      "loss": 0.1042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9841465950012207,
      "rewards/margins": 2.9841465950012207,
      "rewards/rejected": 0.0,
      "step": 1770
    },
    {
      "epoch": 9.893854748603353,
      "grad_norm": 1.3678046046325458,
      "learning_rate": 1.9097949268363323e-06,
      "logits/chosen": -3.5746986865997314,
      "logits/rejected": -3.7211403846740723,
      "logps/chosen": -0.6793075799942017,
      "logps/rejected": -48.69995880126953,
      "loss": 0.0642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.884077548980713,
      "rewards/margins": 2.884077548980713,
      "rewards/rejected": 0.0,
      "step": 1771
    },
    {
      "epoch": 9.899441340782122,
      "grad_norm": 1.8994830295646523,
      "learning_rate": 1.906833792937921e-06,
      "logits/chosen": -3.5501158237457275,
      "logits/rejected": -3.5270187854766846,
      "logps/chosen": -6.7553935050964355,
      "logps/rejected": -70.10858917236328,
      "loss": 0.1312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8363356590270996,
      "rewards/margins": 2.8363356590270996,
      "rewards/rejected": 0.0,
      "step": 1772
    },
    {
      "epoch": 9.905027932960895,
      "grad_norm": 2.258671409025057,
      "learning_rate": 1.9038735405901307e-06,
      "logits/chosen": -3.486774206161499,
      "logits/rejected": -3.4179575443267822,
      "logps/chosen": -3.6594998836517334,
      "logps/rejected": -35.93439483642578,
      "loss": 0.109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4222283363342285,
      "rewards/margins": 3.4222283363342285,
      "rewards/rejected": 0.0,
      "step": 1773
    },
    {
      "epoch": 9.910614525139664,
      "grad_norm": 0.6015861308593023,
      "learning_rate": 1.9009141741924233e-06,
      "logits/chosen": -3.2035207748413086,
      "logits/rejected": -3.5195651054382324,
      "logps/chosen": -0.9899325370788574,
      "logps/rejected": -93.32347869873047,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7646703720092773,
      "rewards/margins": 2.7646703720092773,
      "rewards/rejected": 0.0,
      "step": 1774
    },
    {
      "epoch": 9.916201117318435,
      "grad_norm": 1.0374633094809154,
      "learning_rate": 1.897955698142944e-06,
      "logits/chosen": -3.5002005100250244,
      "logits/rejected": -3.4204864501953125,
      "logps/chosen": -0.8487168550491333,
      "logps/rejected": -55.440406799316406,
      "loss": 0.0968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6739182472229004,
      "rewards/margins": 2.6739182472229004,
      "rewards/rejected": 0.0,
      "step": 1775
    },
    {
      "epoch": 9.921787709497206,
      "grad_norm": 0.6694731999140524,
      "learning_rate": 1.8949981168385163e-06,
      "logits/chosen": -3.4231812953948975,
      "logits/rejected": -3.3564341068267822,
      "logps/chosen": -3.139051675796509,
      "logps/rejected": -49.412841796875,
      "loss": 0.0959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7836532592773438,
      "rewards/margins": 2.7836532592773438,
      "rewards/rejected": 0.0,
      "step": 1776
    },
    {
      "epoch": 9.927374301675977,
      "grad_norm": 1.2303403784757785,
      "learning_rate": 1.892041434674631e-06,
      "logits/chosen": -3.646716356277466,
      "logits/rejected": -3.548978805541992,
      "logps/chosen": -8.4971284866333,
      "logps/rejected": -64.2020263671875,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.17197322845459,
      "rewards/margins": 4.17197322845459,
      "rewards/rejected": 0.0,
      "step": 1777
    },
    {
      "epoch": 9.932960893854748,
      "grad_norm": 1.0851033027631,
      "learning_rate": 1.8890856560454453e-06,
      "logits/chosen": -3.5365021228790283,
      "logits/rejected": -3.6548421382904053,
      "logps/chosen": -1.3907325267791748,
      "logps/rejected": -68.92999267578125,
      "loss": 0.1045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8503189086914062,
      "rewards/margins": 2.8503189086914062,
      "rewards/rejected": 0.0,
      "step": 1778
    },
    {
      "epoch": 9.938547486033519,
      "grad_norm": 5.835574635817555,
      "learning_rate": 1.8861307853437738e-06,
      "logits/chosen": -3.6459155082702637,
      "logits/rejected": -3.6613354682922363,
      "logps/chosen": -0.5251060724258423,
      "logps/rejected": -60.319671630859375,
      "loss": 0.1549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2560856342315674,
      "rewards/margins": 2.2560856342315674,
      "rewards/rejected": 0.0,
      "step": 1779
    },
    {
      "epoch": 9.94413407821229,
      "grad_norm": 2.0650754895530667,
      "learning_rate": 1.8831768269610781e-06,
      "logits/chosen": -2.8045899868011475,
      "logits/rejected": -2.8948328495025635,
      "logps/chosen": -37.12540054321289,
      "logps/rejected": -59.65816116333008,
      "loss": 0.0986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.0843095779418945,
      "rewards/margins": 4.0843095779418945,
      "rewards/rejected": 0.0,
      "step": 1780
    },
    {
      "epoch": 9.94413407821229,
      "eval_logits/chosen": -3.3600845336914062,
      "eval_logits/rejected": -3.475360870361328,
      "eval_logps/chosen": -37.983497619628906,
      "eval_logps/rejected": -59.6567268371582,
      "eval_loss": 1.5906516313552856,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -0.7913460731506348,
      "eval_rewards/margins": -0.7913460731506348,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5506,
      "eval_samples_per_second": 12.133,
      "eval_steps_per_second": 0.391,
      "step": 1780
    },
    {
      "epoch": 9.949720670391061,
      "grad_norm": 0.6812858817552221,
      "learning_rate": 1.8802237852874677e-06,
      "logits/chosen": -3.6178367137908936,
      "logits/rejected": -3.3751797676086426,
      "logps/chosen": -0.414205938577652,
      "logps/rejected": -56.93121337890625,
      "loss": 0.0845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0732007026672363,
      "rewards/margins": 2.0732007026672363,
      "rewards/rejected": 0.0,
      "step": 1781
    },
    {
      "epoch": 9.955307262569832,
      "grad_norm": 1.9346056218865673,
      "learning_rate": 1.8772716647116897e-06,
      "logits/chosen": -3.329305648803711,
      "logits/rejected": -3.472344398498535,
      "logps/chosen": -3.407214641571045,
      "logps/rejected": -94.70446014404297,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.57340669631958,
      "rewards/margins": 2.57340669631958,
      "rewards/rejected": 0.0,
      "step": 1782
    },
    {
      "epoch": 9.960893854748603,
      "grad_norm": 0.8627085788556956,
      "learning_rate": 1.8743204696211193e-06,
      "logits/chosen": -3.5010902881622314,
      "logits/rejected": -3.2746946811676025,
      "logps/chosen": -1.2665964365005493,
      "logps/rejected": -60.75376892089844,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1734437942504883,
      "rewards/margins": 2.1734437942504883,
      "rewards/rejected": 0.0,
      "step": 1783
    },
    {
      "epoch": 9.966480446927374,
      "grad_norm": 0.619333561131409,
      "learning_rate": 1.8713702044017578e-06,
      "logits/chosen": -3.5307931900024414,
      "logits/rejected": -3.3802270889282227,
      "logps/chosen": -2.6427783966064453,
      "logps/rejected": -53.79507064819336,
      "loss": 0.1039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.1284942626953125,
      "rewards/margins": 4.1284942626953125,
      "rewards/rejected": 0.0,
      "step": 1784
    },
    {
      "epoch": 9.972067039106145,
      "grad_norm": 1.3839621567208018,
      "learning_rate": 1.8684208734382265e-06,
      "logits/chosen": -3.4566662311553955,
      "logits/rejected": -3.3718278408050537,
      "logps/chosen": -0.9323084354400635,
      "logps/rejected": -44.53733825683594,
      "loss": 0.1192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.220353364944458,
      "rewards/margins": 2.220353364944458,
      "rewards/rejected": 0.0,
      "step": 1785
    },
    {
      "epoch": 9.977653631284916,
      "grad_norm": 3.221982094884552,
      "learning_rate": 1.8654724811137542e-06,
      "logits/chosen": -3.563037157058716,
      "logits/rejected": -3.421727180480957,
      "logps/chosen": -0.3899504542350769,
      "logps/rejected": -53.110511779785156,
      "loss": 0.1188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9232404232025146,
      "rewards/margins": 1.9232404232025146,
      "rewards/rejected": 0.0,
      "step": 1786
    },
    {
      "epoch": 9.983240223463687,
      "grad_norm": 0.5677778256396898,
      "learning_rate": 1.8625250318101783e-06,
      "logits/chosen": -3.490062952041626,
      "logits/rejected": -3.3280885219573975,
      "logps/chosen": -5.957888603210449,
      "logps/rejected": -58.555747985839844,
      "loss": 0.0831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.737152576446533,
      "rewards/margins": 4.737152576446533,
      "rewards/rejected": 0.0,
      "step": 1787
    },
    {
      "epoch": 9.988826815642458,
      "grad_norm": 2.03821687100002,
      "learning_rate": 1.8595785299079345e-06,
      "logits/chosen": -3.3639016151428223,
      "logits/rejected": -3.3348546028137207,
      "logps/chosen": -1.171812891960144,
      "logps/rejected": -46.033058166503906,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6964056491851807,
      "rewards/margins": 2.6964056491851807,
      "rewards/rejected": 0.0,
      "step": 1788
    },
    {
      "epoch": 9.994413407821229,
      "grad_norm": 0.7854355978130834,
      "learning_rate": 1.8566329797860466e-06,
      "logits/chosen": -3.110532522201538,
      "logits/rejected": -3.2048418521881104,
      "logps/chosen": -0.9708756804466248,
      "logps/rejected": -51.96087646484375,
      "loss": 0.1028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3739547729492188,
      "rewards/margins": 2.3739547729492188,
      "rewards/rejected": 0.0,
      "step": 1789
    },
    {
      "epoch": 10.0,
      "grad_norm": 1.7969949854598435,
      "learning_rate": 1.8536883858221283e-06,
      "logits/chosen": -3.493772506713867,
      "logits/rejected": -3.368607997894287,
      "logps/chosen": -2.662914752960205,
      "logps/rejected": -63.642845153808594,
      "loss": 0.1147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.091583490371704,
      "rewards/margins": 3.091583490371704,
      "rewards/rejected": 0.0,
      "step": 1790
    },
    {
      "epoch": 10.005586592178771,
      "grad_norm": 0.3910063757622164,
      "learning_rate": 1.8507447523923715e-06,
      "logits/chosen": -3.7828547954559326,
      "logits/rejected": -3.753413438796997,
      "logps/chosen": -14.104302406311035,
      "logps/rejected": -89.78163146972656,
      "loss": 0.0996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.048940181732178,
      "rewards/margins": 4.048940181732178,
      "rewards/rejected": 0.0,
      "step": 1791
    },
    {
      "epoch": 10.011173184357542,
      "grad_norm": 1.3395322610789517,
      "learning_rate": 1.8478020838715382e-06,
      "logits/chosen": -3.289055347442627,
      "logits/rejected": -3.1636998653411865,
      "logps/chosen": -2.182673454284668,
      "logps/rejected": -53.46824645996094,
      "loss": 0.1118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.100508213043213,
      "rewards/margins": 3.100508213043213,
      "rewards/rejected": 0.0,
      "step": 1792
    },
    {
      "epoch": 10.016759776536313,
      "grad_norm": 1.2318000727296263,
      "learning_rate": 1.8448603846329583e-06,
      "logits/chosen": -3.3922133445739746,
      "logits/rejected": -3.459946870803833,
      "logps/chosen": -1.4448825120925903,
      "logps/rejected": -51.853485107421875,
      "loss": 0.0762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1097874641418457,
      "rewards/margins": 3.1097874641418457,
      "rewards/rejected": 0.0,
      "step": 1793
    },
    {
      "epoch": 10.022346368715084,
      "grad_norm": 0.7009627232568668,
      "learning_rate": 1.8419196590485207e-06,
      "logits/chosen": -3.8295364379882812,
      "logits/rejected": -3.654109001159668,
      "logps/chosen": -1.2816489934921265,
      "logps/rejected": -73.84138488769531,
      "loss": 0.1125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0791351795196533,
      "rewards/margins": 3.0791351795196533,
      "rewards/rejected": 0.0,
      "step": 1794
    },
    {
      "epoch": 10.027932960893855,
      "grad_norm": 0.7275430257447272,
      "learning_rate": 1.8389799114886672e-06,
      "logits/chosen": -3.5497801303863525,
      "logits/rejected": -3.707500696182251,
      "logps/chosen": -10.870001792907715,
      "logps/rejected": -76.1259765625,
      "loss": 0.0912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.0801191329956055,
      "rewards/margins": 4.0801191329956055,
      "rewards/rejected": 0.0,
      "step": 1795
    },
    {
      "epoch": 10.033519553072626,
      "grad_norm": 0.601524436899315,
      "learning_rate": 1.8360411463223876e-06,
      "logits/chosen": -3.207333564758301,
      "logits/rejected": -3.1272778511047363,
      "logps/chosen": -3.098708152770996,
      "logps/rejected": -54.50117111206055,
      "loss": 0.1165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1771817207336426,
      "rewards/margins": 3.1771817207336426,
      "rewards/rejected": 0.0,
      "step": 1796
    },
    {
      "epoch": 10.039106145251397,
      "grad_norm": 0.3260028509607475,
      "learning_rate": 1.8331033679172078e-06,
      "logits/chosen": -3.341090202331543,
      "logits/rejected": -3.4840996265411377,
      "logps/chosen": -0.1516648828983307,
      "logps/rejected": -88.95285034179688,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.870107650756836,
      "rewards/margins": 1.870107650756836,
      "rewards/rejected": 0.0,
      "step": 1797
    },
    {
      "epoch": 10.044692737430168,
      "grad_norm": 2.3559542734295054,
      "learning_rate": 1.8301665806391911e-06,
      "logits/chosen": -3.6734609603881836,
      "logits/rejected": -3.8021271228790283,
      "logps/chosen": -1.4772058725357056,
      "logps/rejected": -62.111351013183594,
      "loss": 0.1444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4894466400146484,
      "rewards/margins": 3.4894466400146484,
      "rewards/rejected": 0.0,
      "step": 1798
    },
    {
      "epoch": 10.050279329608939,
      "grad_norm": 0.590424892146958,
      "learning_rate": 1.8272307888529276e-06,
      "logits/chosen": -3.749664545059204,
      "logits/rejected": -3.6115610599517822,
      "logps/chosen": -0.20455990731716156,
      "logps/rejected": -66.1418228149414,
      "loss": 0.0882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.339442014694214,
      "rewards/margins": 2.339442014694214,
      "rewards/rejected": 0.0,
      "step": 1799
    },
    {
      "epoch": 10.05586592178771,
      "grad_norm": 1.0890813183040984,
      "learning_rate": 1.8242959969215243e-06,
      "logits/chosen": -3.5653531551361084,
      "logits/rejected": -3.455012321472168,
      "logps/chosen": -0.6495624780654907,
      "logps/rejected": -73.74775695800781,
      "loss": 0.1174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2977352142333984,
      "rewards/margins": 2.2977352142333984,
      "rewards/rejected": 0.0,
      "step": 1800
    },
    {
      "epoch": 10.05586592178771,
      "eval_logits/chosen": -3.3843789100646973,
      "eval_logits/rejected": -3.503326416015625,
      "eval_logps/chosen": -40.432777404785156,
      "eval_logps/rejected": -62.9072265625,
      "eval_loss": 1.8187177181243896,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -1.0362732410430908,
      "eval_rewards/margins": -1.0362732410430908,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6185,
      "eval_samples_per_second": 12.101,
      "eval_steps_per_second": 0.39,
      "step": 1800
    },
    {
      "epoch": 10.061452513966481,
      "grad_norm": 0.26208140182394934,
      "learning_rate": 1.8213622092066058e-06,
      "logits/chosen": -3.2268335819244385,
      "logits/rejected": -3.4109995365142822,
      "logps/chosen": -2.409595012664795,
      "logps/rejected": -112.51763916015625,
      "loss": 0.0804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.580878496170044,
      "rewards/margins": 3.580878496170044,
      "rewards/rejected": 0.0,
      "step": 1801
    },
    {
      "epoch": 10.067039106145252,
      "grad_norm": 1.522543971708162,
      "learning_rate": 1.8184294300683044e-06,
      "logits/chosen": -3.335427761077881,
      "logits/rejected": -3.5729525089263916,
      "logps/chosen": -0.5054720044136047,
      "logps/rejected": -102.11864471435547,
      "loss": 0.1375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1909842491149902,
      "rewards/margins": 2.1909842491149902,
      "rewards/rejected": 0.0,
      "step": 1802
    },
    {
      "epoch": 10.072625698324023,
      "grad_norm": 0.5589144021768444,
      "learning_rate": 1.8154976638652504e-06,
      "logits/chosen": -3.4190707206726074,
      "logits/rejected": -3.3407654762268066,
      "logps/chosen": -0.5011223554611206,
      "logps/rejected": -49.684471130371094,
      "loss": 0.1123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.425100326538086,
      "rewards/margins": 2.425100326538086,
      "rewards/rejected": 0.0,
      "step": 1803
    },
    {
      "epoch": 10.078212290502794,
      "grad_norm": 2.8431296083774567,
      "learning_rate": 1.8125669149545708e-06,
      "logits/chosen": -3.743421792984009,
      "logits/rejected": -3.5671417713165283,
      "logps/chosen": -1.02516508102417,
      "logps/rejected": -55.66260528564453,
      "loss": 0.1061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6816155910491943,
      "rewards/margins": 2.6816155910491943,
      "rewards/rejected": 0.0,
      "step": 1804
    },
    {
      "epoch": 10.083798882681565,
      "grad_norm": 0.31616989472588947,
      "learning_rate": 1.8096371876918816e-06,
      "logits/chosen": -3.62729811668396,
      "logits/rejected": -3.6653366088867188,
      "logps/chosen": -13.438353538513184,
      "logps/rejected": -42.042503356933594,
      "loss": 0.0787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.378164768218994,
      "rewards/margins": 4.378164768218994,
      "rewards/rejected": 0.0,
      "step": 1805
    },
    {
      "epoch": 10.089385474860336,
      "grad_norm": 1.35262109512663,
      "learning_rate": 1.8067084864312773e-06,
      "logits/chosen": -3.2837910652160645,
      "logits/rejected": -3.3387341499328613,
      "logps/chosen": -1.0888484716415405,
      "logps/rejected": -99.297607421875,
      "loss": 0.1006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6988940238952637,
      "rewards/margins": 2.6988940238952637,
      "rewards/rejected": 0.0,
      "step": 1806
    },
    {
      "epoch": 10.094972067039107,
      "grad_norm": 0.3546191410732121,
      "learning_rate": 1.8037808155253306e-06,
      "logits/chosen": -3.4961605072021484,
      "logits/rejected": -3.354398727416992,
      "logps/chosen": -2.3666398525238037,
      "logps/rejected": -40.26435089111328,
      "loss": 0.0928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7270915508270264,
      "rewards/margins": 3.7270915508270264,
      "rewards/rejected": 0.0,
      "step": 1807
    },
    {
      "epoch": 10.100558659217878,
      "grad_norm": 0.2517855402594621,
      "learning_rate": 1.8008541793250827e-06,
      "logits/chosen": -3.615572690963745,
      "logits/rejected": -3.781424045562744,
      "logps/chosen": -6.928436756134033,
      "logps/rejected": -67.60491943359375,
      "loss": 0.1146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1412577629089355,
      "rewards/margins": 3.1412577629089355,
      "rewards/rejected": 0.0,
      "step": 1808
    },
    {
      "epoch": 10.106145251396647,
      "grad_norm": 0.43730250793537745,
      "learning_rate": 1.7979285821800348e-06,
      "logits/chosen": -3.6044921875,
      "logits/rejected": -3.789013624191284,
      "logps/chosen": -0.6970926523208618,
      "logps/rejected": -57.65594482421875,
      "loss": 0.1073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8310821056365967,
      "rewards/margins": 2.8310821056365967,
      "rewards/rejected": 0.0,
      "step": 1809
    },
    {
      "epoch": 10.111731843575418,
      "grad_norm": 3.1707402671858818,
      "learning_rate": 1.795004028438146e-06,
      "logits/chosen": -3.5633649826049805,
      "logits/rejected": -3.3581318855285645,
      "logps/chosen": -39.571475982666016,
      "logps/rejected": -51.94721603393555,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.968894958496094,
      "rewards/margins": 4.968894958496094,
      "rewards/rejected": 0.0,
      "step": 1810
    },
    {
      "epoch": 10.11731843575419,
      "grad_norm": 0.5031221280433621,
      "learning_rate": 1.7920805224458244e-06,
      "logits/chosen": -3.7286195755004883,
      "logits/rejected": -3.643686056137085,
      "logps/chosen": -1.1051347255706787,
      "logps/rejected": -58.75030517578125,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4122815132141113,
      "rewards/margins": 3.4122815132141113,
      "rewards/rejected": 0.0,
      "step": 1811
    },
    {
      "epoch": 10.12290502793296,
      "grad_norm": 1.4768660103014013,
      "learning_rate": 1.789158068547919e-06,
      "logits/chosen": -3.4949514865875244,
      "logits/rejected": -3.6147191524505615,
      "logps/chosen": -0.29397058486938477,
      "logps/rejected": -108.69268798828125,
      "loss": 0.1368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7097421884536743,
      "rewards/margins": 1.7097421884536743,
      "rewards/rejected": 0.0,
      "step": 1812
    },
    {
      "epoch": 10.128491620111731,
      "grad_norm": 0.30804731238343963,
      "learning_rate": 1.7862366710877182e-06,
      "logits/chosen": -3.356867790222168,
      "logits/rejected": -3.4172966480255127,
      "logps/chosen": -0.7839998006820679,
      "logps/rejected": -44.41044235229492,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8078386783599854,
      "rewards/margins": 2.8078386783599854,
      "rewards/rejected": 0.0,
      "step": 1813
    },
    {
      "epoch": 10.134078212290502,
      "grad_norm": 0.5805200100550459,
      "learning_rate": 1.7833163344069393e-06,
      "logits/chosen": -3.54829740524292,
      "logits/rejected": -3.615159034729004,
      "logps/chosen": -15.579971313476562,
      "logps/rejected": -54.72223663330078,
      "loss": 0.0913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5233333110809326,
      "rewards/margins": 3.5233333110809326,
      "rewards/rejected": 0.0,
      "step": 1814
    },
    {
      "epoch": 10.139664804469273,
      "grad_norm": 1.2191438161843848,
      "learning_rate": 1.7803970628457208e-06,
      "logits/chosen": -3.512312650680542,
      "logits/rejected": -3.469557285308838,
      "logps/chosen": -0.6353023648262024,
      "logps/rejected": -57.81452178955078,
      "loss": 0.0819,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1352105140686035,
      "rewards/margins": 3.1352105140686035,
      "rewards/rejected": 0.0,
      "step": 1815
    },
    {
      "epoch": 10.145251396648044,
      "grad_norm": 0.6447252376331342,
      "learning_rate": 1.777478860742621e-06,
      "logits/chosen": -3.2556021213531494,
      "logits/rejected": -3.199376344680786,
      "logps/chosen": -13.679542541503906,
      "logps/rejected": -60.52568817138672,
      "loss": 0.0949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.121767044067383,
      "rewards/margins": 4.121767044067383,
      "rewards/rejected": 0.0,
      "step": 1816
    },
    {
      "epoch": 10.150837988826815,
      "grad_norm": 0.48174202492391055,
      "learning_rate": 1.7745617324346092e-06,
      "logits/chosen": -3.413184642791748,
      "logits/rejected": -3.5721871852874756,
      "logps/chosen": -0.3645296096801758,
      "logps/rejected": -103.21003723144531,
      "loss": 0.0994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.148453712463379,
      "rewards/margins": 2.148453712463379,
      "rewards/rejected": 0.0,
      "step": 1817
    },
    {
      "epoch": 10.156424581005586,
      "grad_norm": 0.2788153870126144,
      "learning_rate": 1.7716456822570555e-06,
      "logits/chosen": -3.4228880405426025,
      "logits/rejected": -3.5375916957855225,
      "logps/chosen": -0.38369378447532654,
      "logps/rejected": -65.49101257324219,
      "loss": 0.0891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4809072017669678,
      "rewards/margins": 2.4809072017669678,
      "rewards/rejected": 0.0,
      "step": 1818
    },
    {
      "epoch": 10.162011173184357,
      "grad_norm": 2.4508847531692517,
      "learning_rate": 1.7687307145437322e-06,
      "logits/chosen": -3.194626569747925,
      "logits/rejected": -3.024061679840088,
      "logps/chosen": -2.1846275329589844,
      "logps/rejected": -52.96369934082031,
      "loss": 0.0833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9406867027282715,
      "rewards/margins": 2.9406867027282715,
      "rewards/rejected": 0.0,
      "step": 1819
    },
    {
      "epoch": 10.167597765363128,
      "grad_norm": 2.8748195204998437,
      "learning_rate": 1.7658168336267973e-06,
      "logits/chosen": -3.600311040878296,
      "logits/rejected": -3.6477460861206055,
      "logps/chosen": -2.6219546794891357,
      "logps/rejected": -107.36520385742188,
      "loss": 0.1089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1325039863586426,
      "rewards/margins": 3.1325039863586426,
      "rewards/rejected": 0.0,
      "step": 1820
    },
    {
      "epoch": 10.167597765363128,
      "eval_logits/chosen": -3.347860813140869,
      "eval_logits/rejected": -3.46952486038208,
      "eval_logps/chosen": -40.451393127441406,
      "eval_logps/rejected": -64.85575103759766,
      "eval_loss": 1.9257804155349731,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -1.0381348133087158,
      "eval_rewards/margins": -1.0381348133087158,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.646,
      "eval_samples_per_second": 12.088,
      "eval_steps_per_second": 0.39,
      "step": 1820
    },
    {
      "epoch": 10.1731843575419,
      "grad_norm": 0.28219501775474176,
      "learning_rate": 1.762904043836799e-06,
      "logits/chosen": -3.7191824913024902,
      "logits/rejected": -3.587934732437134,
      "logps/chosen": -0.6101561784744263,
      "logps/rejected": -53.38368225097656,
      "loss": 0.0754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7611427307128906,
      "rewards/margins": 2.7611427307128906,
      "rewards/rejected": 0.0,
      "step": 1821
    },
    {
      "epoch": 10.17877094972067,
      "grad_norm": 3.8707012858035323,
      "learning_rate": 1.7599923495026617e-06,
      "logits/chosen": -3.744868755340576,
      "logits/rejected": -3.6302597522735596,
      "logps/chosen": -11.502225875854492,
      "logps/rejected": -41.901214599609375,
      "loss": 0.1214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.75807523727417,
      "rewards/margins": 3.75807523727417,
      "rewards/rejected": 0.0,
      "step": 1822
    },
    {
      "epoch": 10.184357541899441,
      "grad_norm": 2.4644004856994237,
      "learning_rate": 1.757081754951681e-06,
      "logits/chosen": -3.453106164932251,
      "logits/rejected": -3.5674312114715576,
      "logps/chosen": -3.0934979915618896,
      "logps/rejected": -45.80261993408203,
      "loss": 0.1069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.920480489730835,
      "rewards/margins": 1.920480489730835,
      "rewards/rejected": 0.0,
      "step": 1823
    },
    {
      "epoch": 10.189944134078212,
      "grad_norm": 0.25831442490760137,
      "learning_rate": 1.7541722645095183e-06,
      "logits/chosen": -3.438068151473999,
      "logits/rejected": -3.460071086883545,
      "logps/chosen": -1.1561459302902222,
      "logps/rejected": -68.99609375,
      "loss": 0.0738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2540786266326904,
      "rewards/margins": 3.2540786266326904,
      "rewards/rejected": 0.0,
      "step": 1824
    },
    {
      "epoch": 10.195530726256983,
      "grad_norm": 0.543464201297547,
      "learning_rate": 1.7512638825001966e-06,
      "logits/chosen": -3.8402299880981445,
      "logits/rejected": -3.7655205726623535,
      "logps/chosen": -0.3638302683830261,
      "logps/rejected": -52.21664810180664,
      "loss": 0.1096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3160061836242676,
      "rewards/margins": 2.3160061836242676,
      "rewards/rejected": 0.0,
      "step": 1825
    },
    {
      "epoch": 10.201117318435754,
      "grad_norm": 0.7258485370292316,
      "learning_rate": 1.7483566132460867e-06,
      "logits/chosen": -3.5556812286376953,
      "logits/rejected": -3.703986644744873,
      "logps/chosen": -1.8845769166946411,
      "logps/rejected": -63.52606201171875,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9624691009521484,
      "rewards/margins": 2.9624691009521484,
      "rewards/rejected": 0.0,
      "step": 1826
    },
    {
      "epoch": 10.206703910614525,
      "grad_norm": 1.3200460030463048,
      "learning_rate": 1.7454504610679103e-06,
      "logits/chosen": -3.491594076156616,
      "logits/rejected": -3.5353875160217285,
      "logps/chosen": -0.9548461437225342,
      "logps/rejected": -70.30261993408203,
      "loss": 0.0888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1432206630706787,
      "rewards/margins": 3.1432206630706787,
      "rewards/rejected": 0.0,
      "step": 1827
    },
    {
      "epoch": 10.212290502793296,
      "grad_norm": 0.4059847412492798,
      "learning_rate": 1.7425454302847273e-06,
      "logits/chosen": -3.612638473510742,
      "logits/rejected": -3.753387212753296,
      "logps/chosen": -4.217129707336426,
      "logps/rejected": -90.61542510986328,
      "loss": 0.0932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9954633712768555,
      "rewards/margins": 3.9954633712768555,
      "rewards/rejected": 0.0,
      "step": 1828
    },
    {
      "epoch": 10.217877094972067,
      "grad_norm": 0.9015012997082222,
      "learning_rate": 1.739641525213929e-06,
      "logits/chosen": -3.6228933334350586,
      "logits/rejected": -3.6046319007873535,
      "logps/chosen": -0.2832130193710327,
      "logps/rejected": -79.41484069824219,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6668143272399902,
      "rewards/margins": 2.6668143272399902,
      "rewards/rejected": 0.0,
      "step": 1829
    },
    {
      "epoch": 10.223463687150838,
      "grad_norm": 0.4231371917341611,
      "learning_rate": 1.7367387501712368e-06,
      "logits/chosen": -3.429401397705078,
      "logits/rejected": -3.567566394805908,
      "logps/chosen": -0.8147755861282349,
      "logps/rejected": -91.83137512207031,
      "loss": 0.0986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.233175277709961,
      "rewards/margins": 2.233175277709961,
      "rewards/rejected": 0.0,
      "step": 1830
    },
    {
      "epoch": 10.22905027932961,
      "grad_norm": 0.8078296315347475,
      "learning_rate": 1.7338371094706925e-06,
      "logits/chosen": -3.5057506561279297,
      "logits/rejected": -3.5160951614379883,
      "logps/chosen": -0.7053801417350769,
      "logps/rejected": -50.547462463378906,
      "loss": 0.0737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4378600120544434,
      "rewards/margins": 2.4378600120544434,
      "rewards/rejected": 0.0,
      "step": 1831
    },
    {
      "epoch": 10.23463687150838,
      "grad_norm": 1.558897141926361,
      "learning_rate": 1.7309366074246487e-06,
      "logits/chosen": -3.3460187911987305,
      "logits/rejected": -3.468597650527954,
      "logps/chosen": -4.833613395690918,
      "logps/rejected": -83.30702209472656,
      "loss": 0.1235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6419501304626465,
      "rewards/margins": 3.6419501304626465,
      "rewards/rejected": 0.0,
      "step": 1832
    },
    {
      "epoch": 10.240223463687151,
      "grad_norm": 0.3287693860945838,
      "learning_rate": 1.728037248343769e-06,
      "logits/chosen": -3.245877742767334,
      "logits/rejected": -3.3541364669799805,
      "logps/chosen": -0.683726966381073,
      "logps/rejected": -73.17742919921875,
      "loss": 0.0999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0730514526367188,
      "rewards/margins": 3.0730514526367188,
      "rewards/rejected": 0.0,
      "step": 1833
    },
    {
      "epoch": 10.245810055865922,
      "grad_norm": 1.2297755721796682,
      "learning_rate": 1.725139036537018e-06,
      "logits/chosen": -3.269057512283325,
      "logits/rejected": -3.2052059173583984,
      "logps/chosen": -3.753692150115967,
      "logps/rejected": -48.83844757080078,
      "loss": 0.1114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.116454601287842,
      "rewards/margins": 3.116454601287842,
      "rewards/rejected": 0.0,
      "step": 1834
    },
    {
      "epoch": 10.251396648044693,
      "grad_norm": 0.7490502204895657,
      "learning_rate": 1.7222419763116533e-06,
      "logits/chosen": -3.253345489501953,
      "logits/rejected": -3.266301155090332,
      "logps/chosen": -8.079336166381836,
      "logps/rejected": -57.24494934082031,
      "loss": 0.0736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.049880027770996,
      "rewards/margins": 4.049880027770996,
      "rewards/rejected": 0.0,
      "step": 1835
    },
    {
      "epoch": 10.256983240223464,
      "grad_norm": 1.9979798645325704,
      "learning_rate": 1.7193460719732224e-06,
      "logits/chosen": -3.5281643867492676,
      "logits/rejected": -3.5558712482452393,
      "logps/chosen": -11.145415306091309,
      "logps/rejected": -59.30460739135742,
      "loss": 0.1275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.312732696533203,
      "rewards/margins": 4.312732696533203,
      "rewards/rejected": 0.0,
      "step": 1836
    },
    {
      "epoch": 10.262569832402235,
      "grad_norm": 1.2532217494238684,
      "learning_rate": 1.7164513278255568e-06,
      "logits/chosen": -3.363966703414917,
      "logits/rejected": -3.3901379108428955,
      "logps/chosen": -2.2889344692230225,
      "logps/rejected": -97.7537841796875,
      "loss": 0.1267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.202756404876709,
      "rewards/margins": 3.202756404876709,
      "rewards/rejected": 0.0,
      "step": 1837
    },
    {
      "epoch": 10.268156424581006,
      "grad_norm": 2.1583791697772394,
      "learning_rate": 1.7135577481707591e-06,
      "logits/chosen": -3.442660093307495,
      "logits/rejected": -3.489664316177368,
      "logps/chosen": -5.15022087097168,
      "logps/rejected": -65.05433654785156,
      "loss": 0.0927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8715643882751465,
      "rewards/margins": 3.8715643882751465,
      "rewards/rejected": 0.0,
      "step": 1838
    },
    {
      "epoch": 10.273743016759777,
      "grad_norm": 1.2325507223261105,
      "learning_rate": 1.7106653373092055e-06,
      "logits/chosen": -3.682332992553711,
      "logits/rejected": -3.7565603256225586,
      "logps/chosen": -0.14939497411251068,
      "logps/rejected": -57.32945251464844,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7746797800064087,
      "rewards/margins": 1.7746797800064087,
      "rewards/rejected": 0.0,
      "step": 1839
    },
    {
      "epoch": 10.279329608938548,
      "grad_norm": 2.20105039740324,
      "learning_rate": 1.7077740995395345e-06,
      "logits/chosen": -3.4352569580078125,
      "logits/rejected": -3.436605453491211,
      "logps/chosen": -0.6781608462333679,
      "logps/rejected": -82.12535095214844,
      "loss": 0.0871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.452923059463501,
      "rewards/margins": 2.452923059463501,
      "rewards/rejected": 0.0,
      "step": 1840
    },
    {
      "epoch": 10.279329608938548,
      "eval_logits/chosen": -3.350893020629883,
      "eval_logits/rejected": -3.465007781982422,
      "eval_logps/chosen": -40.778221130371094,
      "eval_logps/rejected": -64.04618835449219,
      "eval_loss": 1.8663278818130493,
      "eval_rewards/accuracies": 0.22499999403953552,
      "eval_rewards/chosen": -1.0708177089691162,
      "eval_rewards/margins": -1.0708177089691162,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5238,
      "eval_samples_per_second": 12.146,
      "eval_steps_per_second": 0.392,
      "step": 1840
    },
    {
      "epoch": 10.28491620111732,
      "grad_norm": 0.3578135374291303,
      "learning_rate": 1.7048840391586383e-06,
      "logits/chosen": -3.400324821472168,
      "logits/rejected": -3.591071128845215,
      "logps/chosen": -0.09189988672733307,
      "logps/rejected": -69.76734924316406,
      "loss": 0.1029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5655603408813477,
      "rewards/margins": 1.5655603408813477,
      "rewards/rejected": 0.0,
      "step": 1841
    },
    {
      "epoch": 10.29050279329609,
      "grad_norm": 0.574738937321941,
      "learning_rate": 1.7019951604616628e-06,
      "logits/chosen": -3.600079298019409,
      "logits/rejected": -3.589775323867798,
      "logps/chosen": -0.40405401587486267,
      "logps/rejected": -61.542869567871094,
      "loss": 0.0854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5387191772460938,
      "rewards/margins": 2.5387191772460938,
      "rewards/rejected": 0.0,
      "step": 1842
    },
    {
      "epoch": 10.296089385474861,
      "grad_norm": 7.029673460512235,
      "learning_rate": 1.6991074677419956e-06,
      "logits/chosen": -3.6048526763916016,
      "logits/rejected": -3.390634298324585,
      "logps/chosen": -1.1218451261520386,
      "logps/rejected": -99.34815979003906,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6451737880706787,
      "rewards/margins": 3.6451737880706787,
      "rewards/rejected": 0.0,
      "step": 1843
    },
    {
      "epoch": 10.30167597765363,
      "grad_norm": 0.8156140179364133,
      "learning_rate": 1.6962209652912625e-06,
      "logits/chosen": -3.5025954246520996,
      "logits/rejected": -3.3814918994903564,
      "logps/chosen": -1.6926242113113403,
      "logps/rejected": -63.874080657958984,
      "loss": 0.1239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.648773670196533,
      "rewards/margins": 2.648773670196533,
      "rewards/rejected": 0.0,
      "step": 1844
    },
    {
      "epoch": 10.307262569832401,
      "grad_norm": 0.5072480036124378,
      "learning_rate": 1.6933356573993197e-06,
      "logits/chosen": -3.7013661861419678,
      "logits/rejected": -3.628377914428711,
      "logps/chosen": -0.6945739984512329,
      "logps/rejected": -60.01018524169922,
      "loss": 0.1137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8152918815612793,
      "rewards/margins": 2.8152918815612793,
      "rewards/rejected": 0.0,
      "step": 1845
    },
    {
      "epoch": 10.312849162011172,
      "grad_norm": 0.4262599733286907,
      "learning_rate": 1.6904515483542505e-06,
      "logits/chosen": -3.6400747299194336,
      "logits/rejected": -3.4745278358459473,
      "logps/chosen": -2.0223262310028076,
      "logps/rejected": -42.495601654052734,
      "loss": 0.0511,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.674062728881836,
      "rewards/margins": 3.674062728881836,
      "rewards/rejected": 0.0,
      "step": 1846
    },
    {
      "epoch": 10.318435754189943,
      "grad_norm": 1.3378441876599612,
      "learning_rate": 1.6875686424423514e-06,
      "logits/chosen": -3.513801336288452,
      "logits/rejected": -3.478991746902466,
      "logps/chosen": -1.6428406238555908,
      "logps/rejected": -65.45657348632812,
      "loss": 0.1091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3928122520446777,
      "rewards/margins": 2.3928122520446777,
      "rewards/rejected": 0.0,
      "step": 1847
    },
    {
      "epoch": 10.324022346368714,
      "grad_norm": 1.0312479382420698,
      "learning_rate": 1.6846869439481372e-06,
      "logits/chosen": -3.521578311920166,
      "logits/rejected": -3.498636484146118,
      "logps/chosen": -0.6204990148544312,
      "logps/rejected": -49.47021484375,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.444119930267334,
      "rewards/margins": 2.444119930267334,
      "rewards/rejected": 0.0,
      "step": 1848
    },
    {
      "epoch": 10.329608938547485,
      "grad_norm": 1.5220054873511482,
      "learning_rate": 1.6818064571543228e-06,
      "logits/chosen": -3.4338176250457764,
      "logits/rejected": -3.4452130794525146,
      "logps/chosen": -0.18792295455932617,
      "logps/rejected": -60.47420120239258,
      "loss": 0.1128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1142687797546387,
      "rewards/margins": 2.1142687797546387,
      "rewards/rejected": 0.0,
      "step": 1849
    },
    {
      "epoch": 10.335195530726256,
      "grad_norm": 0.6470139962680361,
      "learning_rate": 1.6789271863418249e-06,
      "logits/chosen": -2.8642873764038086,
      "logits/rejected": -3.093078136444092,
      "logps/chosen": -1.8640989065170288,
      "logps/rejected": -119.30538177490234,
      "loss": 0.1149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4951186180114746,
      "rewards/margins": 2.4951186180114746,
      "rewards/rejected": 0.0,
      "step": 1850
    },
    {
      "epoch": 10.340782122905027,
      "grad_norm": 1.7549800431541003,
      "learning_rate": 1.6760491357897546e-06,
      "logits/chosen": -3.5150465965270996,
      "logits/rejected": -3.558497428894043,
      "logps/chosen": -0.6309880614280701,
      "logps/rejected": -75.69943237304688,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7070000171661377,
      "rewards/margins": 2.7070000171661377,
      "rewards/rejected": 0.0,
      "step": 1851
    },
    {
      "epoch": 10.346368715083798,
      "grad_norm": 0.8365342476692098,
      "learning_rate": 1.6731723097754048e-06,
      "logits/chosen": -3.4361815452575684,
      "logits/rejected": -3.56164813041687,
      "logps/chosen": -0.430088609457016,
      "logps/rejected": -64.4599380493164,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4657974243164062,
      "rewards/margins": 2.4657974243164062,
      "rewards/rejected": 0.0,
      "step": 1852
    },
    {
      "epoch": 10.35195530726257,
      "grad_norm": 0.8516075427463982,
      "learning_rate": 1.6702967125742535e-06,
      "logits/chosen": -3.5554122924804688,
      "logits/rejected": -3.7351601123809814,
      "logps/chosen": -2.4928183555603027,
      "logps/rejected": -78.86543273925781,
      "loss": 0.0969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9029293060302734,
      "rewards/margins": 2.9029293060302734,
      "rewards/rejected": 0.0,
      "step": 1853
    },
    {
      "epoch": 10.35754189944134,
      "grad_norm": 0.410386950824953,
      "learning_rate": 1.6674223484599494e-06,
      "logits/chosen": -3.485375165939331,
      "logits/rejected": -3.540364980697632,
      "logps/chosen": -1.2648731470108032,
      "logps/rejected": -54.26875686645508,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.953575611114502,
      "rewards/margins": 2.953575611114502,
      "rewards/rejected": 0.0,
      "step": 1854
    },
    {
      "epoch": 10.363128491620111,
      "grad_norm": 0.7358588691621964,
      "learning_rate": 1.6645492217043086e-06,
      "logits/chosen": -3.017261505126953,
      "logits/rejected": -3.1352293491363525,
      "logps/chosen": -6.2246994972229,
      "logps/rejected": -67.81883239746094,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.575960636138916,
      "rewards/margins": 3.575960636138916,
      "rewards/rejected": 0.0,
      "step": 1855
    },
    {
      "epoch": 10.368715083798882,
      "grad_norm": 0.9551264241494065,
      "learning_rate": 1.6616773365773104e-06,
      "logits/chosen": -3.7633469104766846,
      "logits/rejected": -3.6594436168670654,
      "logps/chosen": -4.115633010864258,
      "logps/rejected": -58.29997634887695,
      "loss": 0.0905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.3983564376831055,
      "rewards/margins": 4.3983564376831055,
      "rewards/rejected": 0.0,
      "step": 1856
    },
    {
      "epoch": 10.374301675977653,
      "grad_norm": 0.9043586705329856,
      "learning_rate": 1.6588066973470876e-06,
      "logits/chosen": -3.181940793991089,
      "logits/rejected": -2.9688026905059814,
      "logps/chosen": -0.39109528064727783,
      "logps/rejected": -103.7013931274414,
      "loss": 0.108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7966195344924927,
      "rewards/margins": 1.7966195344924927,
      "rewards/rejected": 0.0,
      "step": 1857
    },
    {
      "epoch": 10.379888268156424,
      "grad_norm": 0.5473818291774216,
      "learning_rate": 1.6559373082799202e-06,
      "logits/chosen": -3.4166197776794434,
      "logits/rejected": -3.2372827529907227,
      "logps/chosen": -0.8360233306884766,
      "logps/rejected": -102.41311645507812,
      "loss": 0.1008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6629834175109863,
      "rewards/margins": 2.6629834175109863,
      "rewards/rejected": 0.0,
      "step": 1858
    },
    {
      "epoch": 10.385474860335195,
      "grad_norm": 1.9218792604991606,
      "learning_rate": 1.6530691736402317e-06,
      "logits/chosen": -3.5467865467071533,
      "logits/rejected": -3.3068745136260986,
      "logps/chosen": -1.0450036525726318,
      "logps/rejected": -61.36396026611328,
      "loss": 0.1242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0461201667785645,
      "rewards/margins": 3.0461201667785645,
      "rewards/rejected": 0.0,
      "step": 1859
    },
    {
      "epoch": 10.391061452513966,
      "grad_norm": 3.0647070410865385,
      "learning_rate": 1.6502022976905826e-06,
      "logits/chosen": -3.3156321048736572,
      "logits/rejected": -3.4328854084014893,
      "logps/chosen": -0.5637732744216919,
      "logps/rejected": -123.31388854980469,
      "loss": 0.0847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.592149257659912,
      "rewards/margins": 2.592149257659912,
      "rewards/rejected": 0.0,
      "step": 1860
    },
    {
      "epoch": 10.391061452513966,
      "eval_logits/chosen": -3.362684965133667,
      "eval_logits/rejected": -3.4794013500213623,
      "eval_logps/chosen": -40.827308654785156,
      "eval_logps/rejected": -65.08045959472656,
      "eval_loss": 1.8988409042358398,
      "eval_rewards/accuracies": 0.2750000059604645,
      "eval_rewards/chosen": -1.075726866722107,
      "eval_rewards/margins": -1.075726866722107,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.551,
      "eval_samples_per_second": 12.133,
      "eval_steps_per_second": 0.391,
      "step": 1860
    },
    {
      "epoch": 10.396648044692737,
      "grad_norm": 0.4063081652283516,
      "learning_rate": 1.647336684691659e-06,
      "logits/chosen": -3.422671318054199,
      "logits/rejected": -3.444976329803467,
      "logps/chosen": -1.7564685344696045,
      "logps/rejected": -71.61003112792969,
      "loss": 0.1363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5000174045562744,
      "rewards/margins": 2.5000174045562744,
      "rewards/rejected": 0.0,
      "step": 1861
    },
    {
      "epoch": 10.402234636871508,
      "grad_norm": 2.6310498079887568,
      "learning_rate": 1.6444723389022732e-06,
      "logits/chosen": -3.43097186088562,
      "logits/rejected": -3.556753635406494,
      "logps/chosen": -3.6233673095703125,
      "logps/rejected": -122.21969604492188,
      "loss": 0.124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4805307388305664,
      "rewards/margins": 3.4805307388305664,
      "rewards/rejected": 0.0,
      "step": 1862
    },
    {
      "epoch": 10.40782122905028,
      "grad_norm": 1.5333273134468899,
      "learning_rate": 1.641609264579353e-06,
      "logits/chosen": -3.456629991531372,
      "logits/rejected": -3.7521748542785645,
      "logps/chosen": -1.832911729812622,
      "logps/rejected": -57.62110900878906,
      "loss": 0.1293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8934412002563477,
      "rewards/margins": 1.8934412002563477,
      "rewards/rejected": 0.0,
      "step": 1863
    },
    {
      "epoch": 10.41340782122905,
      "grad_norm": 2.4766526166448397,
      "learning_rate": 1.6387474659779366e-06,
      "logits/chosen": -3.41951847076416,
      "logits/rejected": -3.3749523162841797,
      "logps/chosen": -2.3636083602905273,
      "logps/rejected": -45.065147399902344,
      "loss": 0.1429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6900641918182373,
      "rewards/margins": 3.6900641918182373,
      "rewards/rejected": 0.0,
      "step": 1864
    },
    {
      "epoch": 10.418994413407821,
      "grad_norm": 1.705989302061274,
      "learning_rate": 1.6358869473511657e-06,
      "logits/chosen": -3.2864882946014404,
      "logits/rejected": -3.4669852256774902,
      "logps/chosen": -0.17609822750091553,
      "logps/rejected": -92.63481903076172,
      "loss": 0.102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1370439529418945,
      "rewards/margins": 2.1370439529418945,
      "rewards/rejected": 0.0,
      "step": 1865
    },
    {
      "epoch": 10.424581005586592,
      "grad_norm": 7.248076273718469,
      "learning_rate": 1.6330277129502825e-06,
      "logits/chosen": -3.4335949420928955,
      "logits/rejected": -3.4054863452911377,
      "logps/chosen": -2.160593032836914,
      "logps/rejected": -65.21601867675781,
      "loss": 0.1374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.158060073852539,
      "rewards/margins": 4.158060073852539,
      "rewards/rejected": 0.0,
      "step": 1866
    },
    {
      "epoch": 10.430167597765363,
      "grad_norm": 0.3113151981640684,
      "learning_rate": 1.6301697670246165e-06,
      "logits/chosen": -3.670243263244629,
      "logits/rejected": -3.651503562927246,
      "logps/chosen": -9.485980987548828,
      "logps/rejected": -61.38079071044922,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6884853839874268,
      "rewards/margins": 3.6884853839874268,
      "rewards/rejected": 0.0,
      "step": 1867
    },
    {
      "epoch": 10.435754189944134,
      "grad_norm": 2.3442086634703476,
      "learning_rate": 1.6273131138215842e-06,
      "logits/chosen": -3.655951738357544,
      "logits/rejected": -3.7810728549957275,
      "logps/chosen": -4.808541297912598,
      "logps/rejected": -34.22176742553711,
      "loss": 0.1132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9262030124664307,
      "rewards/margins": 3.9262030124664307,
      "rewards/rejected": 0.0,
      "step": 1868
    },
    {
      "epoch": 10.441340782122905,
      "grad_norm": 1.366880042273438,
      "learning_rate": 1.6244577575866832e-06,
      "logits/chosen": -3.6834006309509277,
      "logits/rejected": -3.7787554264068604,
      "logps/chosen": -2.7333266735076904,
      "logps/rejected": -75.77883911132812,
      "loss": 0.1144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.538832664489746,
      "rewards/margins": 4.538832664489746,
      "rewards/rejected": 0.0,
      "step": 1869
    },
    {
      "epoch": 10.446927374301676,
      "grad_norm": 3.172389020180136,
      "learning_rate": 1.621603702563479e-06,
      "logits/chosen": -3.6309397220611572,
      "logits/rejected": -3.6633141040802,
      "logps/chosen": -2.9052274227142334,
      "logps/rejected": -36.97993087768555,
      "loss": 0.119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4872827529907227,
      "rewards/margins": 2.4872827529907227,
      "rewards/rejected": 0.0,
      "step": 1870
    },
    {
      "epoch": 10.452513966480447,
      "grad_norm": 1.0059531554300896,
      "learning_rate": 1.6187509529936072e-06,
      "logits/chosen": -2.7962522506713867,
      "logits/rejected": -2.748729705810547,
      "logps/chosen": -18.829723358154297,
      "logps/rejected": -94.3329849243164,
      "loss": 0.0963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.623750686645508,
      "rewards/margins": 3.623750686645508,
      "rewards/rejected": 0.0,
      "step": 1871
    },
    {
      "epoch": 10.458100558659218,
      "grad_norm": 0.20959865892903914,
      "learning_rate": 1.615899513116761e-06,
      "logits/chosen": -3.641450881958008,
      "logits/rejected": -3.532335042953491,
      "logps/chosen": -5.933674335479736,
      "logps/rejected": -70.68711853027344,
      "loss": 0.0824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.610124111175537,
      "rewards/margins": 3.610124111175537,
      "rewards/rejected": 0.0,
      "step": 1872
    },
    {
      "epoch": 10.46368715083799,
      "grad_norm": 2.005513275243233,
      "learning_rate": 1.6130493871706879e-06,
      "logits/chosen": -3.5202832221984863,
      "logits/rejected": -3.5226643085479736,
      "logps/chosen": -1.0708931684494019,
      "logps/rejected": -61.156768798828125,
      "loss": 0.1169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2096800804138184,
      "rewards/margins": 2.2096800804138184,
      "rewards/rejected": 0.0,
      "step": 1873
    },
    {
      "epoch": 10.46927374301676,
      "grad_norm": 0.4855580203918627,
      "learning_rate": 1.6102005793911824e-06,
      "logits/chosen": -3.4138941764831543,
      "logits/rejected": -3.3274762630462646,
      "logps/chosen": -5.163848876953125,
      "logps/rejected": -62.811370849609375,
      "loss": 0.0881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.053403377532959,
      "rewards/margins": 3.053403377532959,
      "rewards/rejected": 0.0,
      "step": 1874
    },
    {
      "epoch": 10.474860335195531,
      "grad_norm": 0.4304813877116644,
      "learning_rate": 1.6073530940120812e-06,
      "logits/chosen": -3.354595899581909,
      "logits/rejected": -3.5515668392181396,
      "logps/chosen": -1.393860101699829,
      "logps/rejected": -75.31915283203125,
      "loss": 0.1015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5663681030273438,
      "rewards/margins": 2.5663681030273438,
      "rewards/rejected": 0.0,
      "step": 1875
    },
    {
      "epoch": 10.480446927374302,
      "grad_norm": 2.2640872636861715,
      "learning_rate": 1.6045069352652534e-06,
      "logits/chosen": -3.554974317550659,
      "logits/rejected": -3.658646583557129,
      "logps/chosen": -0.6181744337081909,
      "logps/rejected": -62.618934631347656,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6945502758026123,
      "rewards/margins": 2.6945502758026123,
      "rewards/rejected": 0.0,
      "step": 1876
    },
    {
      "epoch": 10.486033519553073,
      "grad_norm": 0.9752466947867896,
      "learning_rate": 1.6016621073805994e-06,
      "logits/chosen": -3.6678943634033203,
      "logits/rejected": -3.45599365234375,
      "logps/chosen": -1.1707990169525146,
      "logps/rejected": -62.23036193847656,
      "loss": 0.1257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.966729164123535,
      "rewards/margins": 2.966729164123535,
      "rewards/rejected": 0.0,
      "step": 1877
    },
    {
      "epoch": 10.491620111731844,
      "grad_norm": 1.6657835145290567,
      "learning_rate": 1.598818614586038e-06,
      "logits/chosen": -3.610893726348877,
      "logits/rejected": -3.564939022064209,
      "logps/chosen": -1.9941030740737915,
      "logps/rejected": -60.29414367675781,
      "loss": 0.0899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.246305227279663,
      "rewards/margins": 3.246305227279663,
      "rewards/rejected": 0.0,
      "step": 1878
    },
    {
      "epoch": 10.497206703910614,
      "grad_norm": 1.2377114453573732,
      "learning_rate": 1.5959764611075068e-06,
      "logits/chosen": -3.4910871982574463,
      "logits/rejected": -3.258188009262085,
      "logps/chosen": -13.615832328796387,
      "logps/rejected": -77.15228271484375,
      "loss": 0.0776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4139256477355957,
      "rewards/margins": 3.4139256477355957,
      "rewards/rejected": 0.0,
      "step": 1879
    },
    {
      "epoch": 10.502793296089386,
      "grad_norm": 1.3061622542877818,
      "learning_rate": 1.593135651168953e-06,
      "logits/chosen": -3.371587038040161,
      "logits/rejected": -3.471658945083618,
      "logps/chosen": -3.805263042449951,
      "logps/rejected": -79.71598052978516,
      "loss": 0.0911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7864508628845215,
      "rewards/margins": 2.7864508628845215,
      "rewards/rejected": 0.0,
      "step": 1880
    },
    {
      "epoch": 10.502793296089386,
      "eval_logits/chosen": -3.364687442779541,
      "eval_logits/rejected": -3.481327533721924,
      "eval_logps/chosen": -40.244415283203125,
      "eval_logps/rejected": -62.86834716796875,
      "eval_loss": 1.8230210542678833,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -1.017437219619751,
      "eval_rewards/margins": -1.017437219619751,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5357,
      "eval_samples_per_second": 12.14,
      "eval_steps_per_second": 0.392,
      "step": 1880
    },
    {
      "epoch": 10.508379888268156,
      "grad_norm": 2.7230274439026676,
      "learning_rate": 1.5902961889923247e-06,
      "logits/chosen": -3.5305335521698,
      "logits/rejected": -3.6255202293395996,
      "logps/chosen": -5.221205711364746,
      "logps/rejected": -59.32157897949219,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.514647960662842,
      "rewards/margins": 2.514647960662842,
      "rewards/rejected": 0.0,
      "step": 1881
    },
    {
      "epoch": 10.513966480446927,
      "grad_norm": 2.0811685610558417,
      "learning_rate": 1.587458078797569e-06,
      "logits/chosen": -3.0037684440612793,
      "logits/rejected": -3.2358644008636475,
      "logps/chosen": -5.044809818267822,
      "logps/rejected": -49.875091552734375,
      "loss": 0.1251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.493228912353516,
      "rewards/margins": 4.493228912353516,
      "rewards/rejected": 0.0,
      "step": 1882
    },
    {
      "epoch": 10.519553072625698,
      "grad_norm": 0.4178822669443709,
      "learning_rate": 1.5846213248026231e-06,
      "logits/chosen": -3.3992927074432373,
      "logits/rejected": -3.4551165103912354,
      "logps/chosen": -0.3054536283016205,
      "logps/rejected": -47.781700134277344,
      "loss": 0.1074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.444098949432373,
      "rewards/margins": 2.444098949432373,
      "rewards/rejected": 0.0,
      "step": 1883
    },
    {
      "epoch": 10.525139664804469,
      "grad_norm": 0.5758918687202611,
      "learning_rate": 1.5817859312234091e-06,
      "logits/chosen": -3.751081705093384,
      "logits/rejected": -3.746689796447754,
      "logps/chosen": -5.557926654815674,
      "logps/rejected": -40.501766204833984,
      "loss": 0.0964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0660886764526367,
      "rewards/margins": 3.0660886764526367,
      "rewards/rejected": 0.0,
      "step": 1884
    },
    {
      "epoch": 10.53072625698324,
      "grad_norm": 0.5175655536951084,
      "learning_rate": 1.5789519022738253e-06,
      "logits/chosen": -3.4938604831695557,
      "logits/rejected": -3.408787250518799,
      "logps/chosen": -0.7572718858718872,
      "logps/rejected": -48.24372863769531,
      "loss": 0.0959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.208012104034424,
      "rewards/margins": 3.208012104034424,
      "rewards/rejected": 0.0,
      "step": 1885
    },
    {
      "epoch": 10.53631284916201,
      "grad_norm": 0.5686564525126403,
      "learning_rate": 1.5761192421657457e-06,
      "logits/chosen": -3.509002923965454,
      "logits/rejected": -3.720712661743164,
      "logps/chosen": -3.1385693550109863,
      "logps/rejected": -110.26823425292969,
      "loss": 0.0921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1413142681121826,
      "rewards/margins": 3.1413142681121826,
      "rewards/rejected": 0.0,
      "step": 1886
    },
    {
      "epoch": 10.541899441340782,
      "grad_norm": 0.6797282803101673,
      "learning_rate": 1.5732879551090052e-06,
      "logits/chosen": -3.2379982471466064,
      "logits/rejected": -3.247185707092285,
      "logps/chosen": -1.746943473815918,
      "logps/rejected": -83.60470581054688,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.450948715209961,
      "rewards/margins": 2.450948715209961,
      "rewards/rejected": 0.0,
      "step": 1887
    },
    {
      "epoch": 10.547486033519553,
      "grad_norm": 0.7269345698266132,
      "learning_rate": 1.570458045311402e-06,
      "logits/chosen": -3.5910613536834717,
      "logits/rejected": -3.6486520767211914,
      "logps/chosen": -0.26958921551704407,
      "logps/rejected": -63.68284606933594,
      "loss": 0.1028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4603090286254883,
      "rewards/margins": 2.4603090286254883,
      "rewards/rejected": 0.0,
      "step": 1888
    },
    {
      "epoch": 10.553072625698324,
      "grad_norm": 1.6496936217778004,
      "learning_rate": 1.5676295169786864e-06,
      "logits/chosen": -3.4306914806365967,
      "logits/rejected": -3.658581256866455,
      "logps/chosen": -3.8959736824035645,
      "logps/rejected": -63.35631561279297,
      "loss": 0.1273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.040558815002441,
      "rewards/margins": 4.040558815002441,
      "rewards/rejected": 0.0,
      "step": 1889
    },
    {
      "epoch": 10.558659217877095,
      "grad_norm": 0.3749406447653732,
      "learning_rate": 1.5648023743145541e-06,
      "logits/chosen": -3.3140451908111572,
      "logits/rejected": -3.3312809467315674,
      "logps/chosen": -0.3791493773460388,
      "logps/rejected": -81.39501953125,
      "loss": 0.1136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1511335372924805,
      "rewards/margins": 2.1511335372924805,
      "rewards/rejected": 0.0,
      "step": 1890
    },
    {
      "epoch": 10.564245810055866,
      "grad_norm": 0.5623322526341141,
      "learning_rate": 1.5619766215206432e-06,
      "logits/chosen": -3.2355260848999023,
      "logits/rejected": -3.434135675430298,
      "logps/chosen": -0.9931041598320007,
      "logps/rejected": -78.95352935791016,
      "loss": 0.1034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.929457664489746,
      "rewards/margins": 2.929457664489746,
      "rewards/rejected": 0.0,
      "step": 1891
    },
    {
      "epoch": 10.569832402234637,
      "grad_norm": 1.81123413301526,
      "learning_rate": 1.5591522627965262e-06,
      "logits/chosen": -3.6742045879364014,
      "logits/rejected": -3.575147867202759,
      "logps/chosen": -0.15574723482131958,
      "logps/rejected": -105.9889144897461,
      "loss": 0.0871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7524991035461426,
      "rewards/margins": 1.7524991035461426,
      "rewards/rejected": 0.0,
      "step": 1892
    },
    {
      "epoch": 10.575418994413408,
      "grad_norm": 6.342348813382397,
      "learning_rate": 1.556329302339702e-06,
      "logits/chosen": -3.173370599746704,
      "logits/rejected": -3.5865557193756104,
      "logps/chosen": -0.47268104553222656,
      "logps/rejected": -71.72222137451172,
      "loss": 0.1558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8899283409118652,
      "rewards/margins": 1.8899283409118652,
      "rewards/rejected": 0.0,
      "step": 1893
    },
    {
      "epoch": 10.581005586592179,
      "grad_norm": 0.6138069919868998,
      "learning_rate": 1.5535077443455926e-06,
      "logits/chosen": -2.7694449424743652,
      "logits/rejected": -2.794915199279785,
      "logps/chosen": -14.91230583190918,
      "logps/rejected": -71.91459655761719,
      "loss": 0.1119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.68274188041687,
      "rewards/margins": 2.68274188041687,
      "rewards/rejected": 0.0,
      "step": 1894
    },
    {
      "epoch": 10.58659217877095,
      "grad_norm": 1.4478130919439438,
      "learning_rate": 1.5506875930075374e-06,
      "logits/chosen": -3.637510061264038,
      "logits/rejected": -3.520637035369873,
      "logps/chosen": -4.761631011962891,
      "logps/rejected": -43.39603805541992,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.377077102661133,
      "rewards/margins": 4.377077102661133,
      "rewards/rejected": 0.0,
      "step": 1895
    },
    {
      "epoch": 10.59217877094972,
      "grad_norm": 0.4740867581553351,
      "learning_rate": 1.5478688525167813e-06,
      "logits/chosen": -3.548502206802368,
      "logits/rejected": -3.7091736793518066,
      "logps/chosen": -0.6988756656646729,
      "logps/rejected": -116.71294403076172,
      "loss": 0.0718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.945897340774536,
      "rewards/margins": 2.945897340774536,
      "rewards/rejected": 0.0,
      "step": 1896
    },
    {
      "epoch": 10.597765363128492,
      "grad_norm": 0.6529084465996916,
      "learning_rate": 1.5450515270624755e-06,
      "logits/chosen": -3.705659866333008,
      "logits/rejected": -3.5849828720092773,
      "logps/chosen": -2.9455525875091553,
      "logps/rejected": -55.67366027832031,
      "loss": 0.0831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.709583282470703,
      "rewards/margins": 3.709583282470703,
      "rewards/rejected": 0.0,
      "step": 1897
    },
    {
      "epoch": 10.603351955307263,
      "grad_norm": 1.6255529426117932,
      "learning_rate": 1.5422356208316685e-06,
      "logits/chosen": -3.588135242462158,
      "logits/rejected": -3.581897258758545,
      "logps/chosen": -2.3905189037323,
      "logps/rejected": -55.12580490112305,
      "loss": 0.0807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.709306001663208,
      "rewards/margins": 3.709306001663208,
      "rewards/rejected": 0.0,
      "step": 1898
    },
    {
      "epoch": 10.608938547486034,
      "grad_norm": 0.41653018851292983,
      "learning_rate": 1.5394211380092962e-06,
      "logits/chosen": -3.6193394660949707,
      "logits/rejected": -3.7073936462402344,
      "logps/chosen": -0.349814772605896,
      "logps/rejected": -39.51415252685547,
      "loss": 0.1271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1231887340545654,
      "rewards/margins": 2.1231887340545654,
      "rewards/rejected": 0.0,
      "step": 1899
    },
    {
      "epoch": 10.614525139664805,
      "grad_norm": 0.9071932577399633,
      "learning_rate": 1.5366080827781835e-06,
      "logits/chosen": -3.516650676727295,
      "logits/rejected": -3.519280195236206,
      "logps/chosen": -0.5557469725608826,
      "logps/rejected": -61.2393798828125,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.97426438331604,
      "rewards/margins": 2.97426438331604,
      "rewards/rejected": 0.0,
      "step": 1900
    },
    {
      "epoch": 10.614525139664805,
      "eval_logits/chosen": -3.380796432495117,
      "eval_logits/rejected": -3.501640796661377,
      "eval_logps/chosen": -40.92212677001953,
      "eval_logps/rejected": -63.96775436401367,
      "eval_loss": 1.8811070919036865,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.0852086544036865,
      "eval_rewards/margins": -1.0852086544036865,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5984,
      "eval_samples_per_second": 12.11,
      "eval_steps_per_second": 0.391,
      "step": 1900
    },
    {
      "epoch": 10.620111731843576,
      "grad_norm": 0.9573639890815385,
      "learning_rate": 1.5337964593190303e-06,
      "logits/chosen": -3.470519781112671,
      "logits/rejected": -3.4611704349517822,
      "logps/chosen": -0.8330127000808716,
      "logps/rejected": -63.18382263183594,
      "loss": 0.0615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4841697216033936,
      "rewards/margins": 2.4841697216033936,
      "rewards/rejected": 0.0,
      "step": 1901
    },
    {
      "epoch": 10.625698324022347,
      "grad_norm": 1.4834787652152417,
      "learning_rate": 1.53098627181041e-06,
      "logits/chosen": -3.361020565032959,
      "logits/rejected": -3.526202917098999,
      "logps/chosen": -3.656935930252075,
      "logps/rejected": -92.43724060058594,
      "loss": 0.1037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.119581699371338,
      "rewards/margins": 3.119581699371338,
      "rewards/rejected": 0.0,
      "step": 1902
    },
    {
      "epoch": 10.631284916201118,
      "grad_norm": 0.5314639770539274,
      "learning_rate": 1.5281775244287611e-06,
      "logits/chosen": -3.8416459560394287,
      "logits/rejected": -3.552985906600952,
      "logps/chosen": -0.3053224980831146,
      "logps/rejected": -88.59835052490234,
      "loss": 0.0797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6074740886688232,
      "rewards/margins": 2.6074740886688232,
      "rewards/rejected": 0.0,
      "step": 1903
    },
    {
      "epoch": 10.636871508379889,
      "grad_norm": 0.46177488043668974,
      "learning_rate": 1.5253702213483843e-06,
      "logits/chosen": -3.482520580291748,
      "logits/rejected": -3.2466533184051514,
      "logps/chosen": -0.20741355419158936,
      "logps/rejected": -88.4041748046875,
      "loss": 0.1199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2101762294769287,
      "rewards/margins": 2.2101762294769287,
      "rewards/rejected": 0.0,
      "step": 1904
    },
    {
      "epoch": 10.64245810055866,
      "grad_norm": 0.2640460270107916,
      "learning_rate": 1.5225643667414295e-06,
      "logits/chosen": -3.7150557041168213,
      "logits/rejected": -3.7463040351867676,
      "logps/chosen": -0.1099075898528099,
      "logps/rejected": -71.5216064453125,
      "loss": 0.0943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7626874446868896,
      "rewards/margins": 1.7626874446868896,
      "rewards/rejected": 0.0,
      "step": 1905
    },
    {
      "epoch": 10.64804469273743,
      "grad_norm": 1.0792401500123097,
      "learning_rate": 1.5197599647778982e-06,
      "logits/chosen": -3.5274102687835693,
      "logits/rejected": -3.502525568008423,
      "logps/chosen": -23.357486724853516,
      "logps/rejected": -69.50971221923828,
      "loss": 0.114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.76220965385437,
      "rewards/margins": 3.76220965385437,
      "rewards/rejected": 0.0,
      "step": 1906
    },
    {
      "epoch": 10.653631284916202,
      "grad_norm": 0.6840136933433377,
      "learning_rate": 1.51695701962563e-06,
      "logits/chosen": -3.582507371902466,
      "logits/rejected": -3.268921375274658,
      "logps/chosen": -0.2917966842651367,
      "logps/rejected": -70.78060913085938,
      "loss": 0.0938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.278756618499756,
      "rewards/margins": 2.278756618499756,
      "rewards/rejected": 0.0,
      "step": 1907
    },
    {
      "epoch": 10.659217877094973,
      "grad_norm": 1.8047179699841405,
      "learning_rate": 1.5141555354503007e-06,
      "logits/chosen": -3.5529603958129883,
      "logits/rejected": -3.3822736740112305,
      "logps/chosen": -1.1932954788208008,
      "logps/rejected": -66.62885284423828,
      "loss": 0.1007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9169023036956787,
      "rewards/margins": 1.9169023036956787,
      "rewards/rejected": 0.0,
      "step": 1908
    },
    {
      "epoch": 10.664804469273744,
      "grad_norm": 1.5210878876146494,
      "learning_rate": 1.511355516415416e-06,
      "logits/chosen": -3.4758920669555664,
      "logits/rejected": -3.5420684814453125,
      "logps/chosen": -13.395991325378418,
      "logps/rejected": -56.96251678466797,
      "loss": 0.1054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.511204242706299,
      "rewards/margins": 2.511204242706299,
      "rewards/rejected": 0.0,
      "step": 1909
    },
    {
      "epoch": 10.670391061452515,
      "grad_norm": 4.450142605992978,
      "learning_rate": 1.5085569666823007e-06,
      "logits/chosen": -3.5957298278808594,
      "logits/rejected": -3.474027633666992,
      "logps/chosen": -5.622009754180908,
      "logps/rejected": -57.08566665649414,
      "loss": 0.0823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.321436882019043,
      "rewards/margins": 2.321436882019043,
      "rewards/rejected": 0.0,
      "step": 1910
    },
    {
      "epoch": 10.675977653631286,
      "grad_norm": 1.431955799623685,
      "learning_rate": 1.5057598904101001e-06,
      "logits/chosen": -3.5819008350372314,
      "logits/rejected": -3.375476360321045,
      "logps/chosen": -0.8728051781654358,
      "logps/rejected": -59.26123046875,
      "loss": 0.1239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4677889347076416,
      "rewards/margins": 3.4677889347076416,
      "rewards/rejected": 0.0,
      "step": 1911
    },
    {
      "epoch": 10.681564245810057,
      "grad_norm": 0.20567871276367017,
      "learning_rate": 1.5029642917557666e-06,
      "logits/chosen": -3.270430564880371,
      "logits/rejected": -3.4546468257904053,
      "logps/chosen": -20.708711624145508,
      "logps/rejected": -101.58991241455078,
      "loss": 0.1014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.744950771331787,
      "rewards/margins": 3.744950771331787,
      "rewards/rejected": 0.0,
      "step": 1912
    },
    {
      "epoch": 10.687150837988828,
      "grad_norm": 0.7606774754472898,
      "learning_rate": 1.5001701748740576e-06,
      "logits/chosen": -3.3331940174102783,
      "logits/rejected": -3.1430776119232178,
      "logps/chosen": -1.3847193717956543,
      "logps/rejected": -44.281402587890625,
      "loss": 0.1115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2289862632751465,
      "rewards/margins": 3.2289862632751465,
      "rewards/rejected": 0.0,
      "step": 1913
    },
    {
      "epoch": 10.692737430167599,
      "grad_norm": 1.0921551271410324,
      "learning_rate": 1.4973775439175285e-06,
      "logits/chosen": -3.554081678390503,
      "logits/rejected": -3.7551686763763428,
      "logps/chosen": -1.5026917457580566,
      "logps/rejected": -83.34381103515625,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.135389804840088,
      "rewards/margins": 2.135389804840088,
      "rewards/rejected": 0.0,
      "step": 1914
    },
    {
      "epoch": 10.69832402234637,
      "grad_norm": 0.5609449640621393,
      "learning_rate": 1.4945864030365278e-06,
      "logits/chosen": -3.462705373764038,
      "logits/rejected": -3.535687208175659,
      "logps/chosen": -1.9013279676437378,
      "logps/rejected": -53.33708953857422,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9594054222106934,
      "rewards/margins": 3.9594054222106934,
      "rewards/rejected": 0.0,
      "step": 1915
    },
    {
      "epoch": 10.703910614525139,
      "grad_norm": 1.148052228041579,
      "learning_rate": 1.4917967563791852e-06,
      "logits/chosen": -3.344198226928711,
      "logits/rejected": -3.2460873126983643,
      "logps/chosen": -14.071776390075684,
      "logps/rejected": -63.42417907714844,
      "loss": 0.073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.377337455749512,
      "rewards/margins": 4.377337455749512,
      "rewards/rejected": 0.0,
      "step": 1916
    },
    {
      "epoch": 10.70949720670391,
      "grad_norm": 0.2706742437627774,
      "learning_rate": 1.489008608091414e-06,
      "logits/chosen": -3.8853940963745117,
      "logits/rejected": -3.7629876136779785,
      "logps/chosen": -0.2183942198753357,
      "logps/rejected": -64.00030517578125,
      "loss": 0.0839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.804091215133667,
      "rewards/margins": 1.804091215133667,
      "rewards/rejected": 0.0,
      "step": 1917
    },
    {
      "epoch": 10.71508379888268,
      "grad_norm": 0.7624107066997272,
      "learning_rate": 1.4862219623168995e-06,
      "logits/chosen": -3.605376958847046,
      "logits/rejected": -3.529445171356201,
      "logps/chosen": -21.671852111816406,
      "logps/rejected": -90.73119354248047,
      "loss": 0.1053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.235976219177246,
      "rewards/margins": 5.235976219177246,
      "rewards/rejected": 0.0,
      "step": 1918
    },
    {
      "epoch": 10.720670391061452,
      "grad_norm": 1.0582577738602597,
      "learning_rate": 1.4834368231970922e-06,
      "logits/chosen": -3.328507900238037,
      "logits/rejected": -3.3908731937408447,
      "logps/chosen": -2.112903356552124,
      "logps/rejected": -78.47604370117188,
      "loss": 0.0968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.420705795288086,
      "rewards/margins": 2.420705795288086,
      "rewards/rejected": 0.0,
      "step": 1919
    },
    {
      "epoch": 10.726256983240223,
      "grad_norm": 1.0385421312943666,
      "learning_rate": 1.4806531948712061e-06,
      "logits/chosen": -3.434699535369873,
      "logits/rejected": -3.579760789871216,
      "logps/chosen": -1.1138752698898315,
      "logps/rejected": -48.459442138671875,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.51025390625,
      "rewards/margins": 2.51025390625,
      "rewards/rejected": 0.0,
      "step": 1920
    },
    {
      "epoch": 10.726256983240223,
      "eval_logits/chosen": -3.3875300884246826,
      "eval_logits/rejected": -3.5099034309387207,
      "eval_logps/chosen": -41.4708366394043,
      "eval_logps/rejected": -65.34963989257812,
      "eval_loss": 1.9305604696273804,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.1400792598724365,
      "eval_rewards/margins": -1.1400792598724365,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5273,
      "eval_samples_per_second": 12.144,
      "eval_steps_per_second": 0.392,
      "step": 1920
    },
    {
      "epoch": 10.731843575418994,
      "grad_norm": 0.30721791834320944,
      "learning_rate": 1.4778710814762082e-06,
      "logits/chosen": -3.4270122051239014,
      "logits/rejected": -3.537722110748291,
      "logps/chosen": -0.10599761456251144,
      "logps/rejected": -83.46339416503906,
      "loss": 0.094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.491112470626831,
      "rewards/margins": 1.491112470626831,
      "rewards/rejected": 0.0,
      "step": 1921
    },
    {
      "epoch": 10.737430167597765,
      "grad_norm": 1.6299539073923892,
      "learning_rate": 1.4750904871468147e-06,
      "logits/chosen": -3.5827889442443848,
      "logits/rejected": -3.5397393703460693,
      "logps/chosen": -0.32963454723358154,
      "logps/rejected": -46.82147216796875,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.13735294342041,
      "rewards/margins": 2.13735294342041,
      "rewards/rejected": 0.0,
      "step": 1922
    },
    {
      "epoch": 10.743016759776536,
      "grad_norm": 0.38429197988254976,
      "learning_rate": 1.4723114160154827e-06,
      "logits/chosen": -3.719639539718628,
      "logits/rejected": -3.681619167327881,
      "logps/chosen": -0.3326700031757355,
      "logps/rejected": -129.16024780273438,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.252685308456421,
      "rewards/margins": 2.252685308456421,
      "rewards/rejected": 0.0,
      "step": 1923
    },
    {
      "epoch": 10.748603351955307,
      "grad_norm": 5.218088547710772,
      "learning_rate": 1.469533872212409e-06,
      "logits/chosen": -3.4839463233947754,
      "logits/rejected": -3.4495553970336914,
      "logps/chosen": -2.5414867401123047,
      "logps/rejected": -81.22103118896484,
      "loss": 0.1201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5661559104919434,
      "rewards/margins": 2.5661559104919434,
      "rewards/rejected": 0.0,
      "step": 1924
    },
    {
      "epoch": 10.754189944134078,
      "grad_norm": 3.8651495543988856,
      "learning_rate": 1.4667578598655163e-06,
      "logits/chosen": -3.7414140701293945,
      "logits/rejected": -3.5447230339050293,
      "logps/chosen": -0.4117240309715271,
      "logps/rejected": -66.62950134277344,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2728543281555176,
      "rewards/margins": 2.2728543281555176,
      "rewards/rejected": 0.0,
      "step": 1925
    },
    {
      "epoch": 10.759776536312849,
      "grad_norm": 1.9663976410412252,
      "learning_rate": 1.4639833831004541e-06,
      "logits/chosen": -3.509573459625244,
      "logits/rejected": -3.6938788890838623,
      "logps/chosen": -3.7568588256835938,
      "logps/rejected": -87.111328125,
      "loss": 0.1132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2137937545776367,
      "rewards/margins": 2.2137937545776367,
      "rewards/rejected": 0.0,
      "step": 1926
    },
    {
      "epoch": 10.76536312849162,
      "grad_norm": 1.5893021187902323,
      "learning_rate": 1.4612104460405901e-06,
      "logits/chosen": -3.756633758544922,
      "logits/rejected": -3.6647486686706543,
      "logps/chosen": -14.113752365112305,
      "logps/rejected": -101.93325805664062,
      "loss": 0.1085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.299509048461914,
      "rewards/margins": 4.299509048461914,
      "rewards/rejected": 0.0,
      "step": 1927
    },
    {
      "epoch": 10.77094972067039,
      "grad_norm": 1.067414060449884,
      "learning_rate": 1.4584390528070012e-06,
      "logits/chosen": -3.5315799713134766,
      "logits/rejected": -3.607389211654663,
      "logps/chosen": -1.062852382659912,
      "logps/rejected": -64.84236907958984,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4649295806884766,
      "rewards/margins": 1.4649295806884766,
      "rewards/rejected": 0.0,
      "step": 1928
    },
    {
      "epoch": 10.776536312849162,
      "grad_norm": 0.43832360052201713,
      "learning_rate": 1.4556692075184702e-06,
      "logits/chosen": -3.755368709564209,
      "logits/rejected": -3.682917356491089,
      "logps/chosen": -0.8227950930595398,
      "logps/rejected": -47.70490646362305,
      "loss": 0.1042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.519676923751831,
      "rewards/margins": 2.519676923751831,
      "rewards/rejected": 0.0,
      "step": 1929
    },
    {
      "epoch": 10.782122905027933,
      "grad_norm": 1.6121942458378709,
      "learning_rate": 1.4529009142914846e-06,
      "logits/chosen": -3.47988224029541,
      "logits/rejected": -3.4414000511169434,
      "logps/chosen": -1.703910231590271,
      "logps/rejected": -71.94161987304688,
      "loss": 0.086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.371534824371338,
      "rewards/margins": 2.371534824371338,
      "rewards/rejected": 0.0,
      "step": 1930
    },
    {
      "epoch": 10.787709497206704,
      "grad_norm": 0.9427619649118614,
      "learning_rate": 1.450134177240216e-06,
      "logits/chosen": -3.7063114643096924,
      "logits/rejected": -3.73846435546875,
      "logps/chosen": -1.3462764024734497,
      "logps/rejected": -53.64207458496094,
      "loss": 0.1105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2511160373687744,
      "rewards/margins": 3.2511160373687744,
      "rewards/rejected": 0.0,
      "step": 1931
    },
    {
      "epoch": 10.793296089385475,
      "grad_norm": 0.38837490889484666,
      "learning_rate": 1.4473690004765324e-06,
      "logits/chosen": -3.71903920173645,
      "logits/rejected": -3.49131178855896,
      "logps/chosen": -1.0754673480987549,
      "logps/rejected": -55.298797607421875,
      "loss": 0.1053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.519463539123535,
      "rewards/margins": 2.519463539123535,
      "rewards/rejected": 0.0,
      "step": 1932
    },
    {
      "epoch": 10.798882681564246,
      "grad_norm": 1.1754802561891664,
      "learning_rate": 1.444605388109977e-06,
      "logits/chosen": -3.59006667137146,
      "logits/rejected": -3.6125619411468506,
      "logps/chosen": -0.7061415910720825,
      "logps/rejected": -75.16734313964844,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9788613319396973,
      "rewards/margins": 2.9788613319396973,
      "rewards/rejected": 0.0,
      "step": 1933
    },
    {
      "epoch": 10.804469273743017,
      "grad_norm": 0.7749208980013307,
      "learning_rate": 1.4418433442477705e-06,
      "logits/chosen": -3.3704445362091064,
      "logits/rejected": -3.611788272857666,
      "logps/chosen": -4.384029388427734,
      "logps/rejected": -62.92270278930664,
      "loss": 0.1027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.831284523010254,
      "rewards/margins": 2.831284523010254,
      "rewards/rejected": 0.0,
      "step": 1934
    },
    {
      "epoch": 10.810055865921788,
      "grad_norm": 5.234704463328495,
      "learning_rate": 1.439082872994802e-06,
      "logits/chosen": -3.4347050189971924,
      "logits/rejected": -3.4902803897857666,
      "logps/chosen": -8.174182891845703,
      "logps/rejected": -93.80488586425781,
      "loss": 0.1188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.504265785217285,
      "rewards/margins": 3.504265785217285,
      "rewards/rejected": 0.0,
      "step": 1935
    },
    {
      "epoch": 10.815642458100559,
      "grad_norm": 1.0234277918564951,
      "learning_rate": 1.436323978453623e-06,
      "logits/chosen": -3.965768814086914,
      "logits/rejected": -3.9412920475006104,
      "logps/chosen": -0.33470287919044495,
      "logps/rejected": -94.09967041015625,
      "loss": 0.0863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2408151626586914,
      "rewards/margins": 2.2408151626586914,
      "rewards/rejected": 0.0,
      "step": 1936
    },
    {
      "epoch": 10.82122905027933,
      "grad_norm": 1.1831169696550294,
      "learning_rate": 1.4335666647244426e-06,
      "logits/chosen": -3.5080182552337646,
      "logits/rejected": -3.2419207096099854,
      "logps/chosen": -0.28059545159339905,
      "logps/rejected": -95.88370513916016,
      "loss": 0.1248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.458085775375366,
      "rewards/margins": 2.458085775375366,
      "rewards/rejected": 0.0,
      "step": 1937
    },
    {
      "epoch": 10.8268156424581,
      "grad_norm": 1.9841139682125835,
      "learning_rate": 1.4308109359051196e-06,
      "logits/chosen": -3.3940176963806152,
      "logits/rejected": -3.4868674278259277,
      "logps/chosen": -12.880878448486328,
      "logps/rejected": -56.56982421875,
      "loss": 0.1049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.371760845184326,
      "rewards/margins": 3.371760845184326,
      "rewards/rejected": 0.0,
      "step": 1938
    },
    {
      "epoch": 10.832402234636872,
      "grad_norm": 1.8145631141804979,
      "learning_rate": 1.4280567960911578e-06,
      "logits/chosen": -3.4762253761291504,
      "logits/rejected": -3.3363311290740967,
      "logps/chosen": -1.268075704574585,
      "logps/rejected": -98.85761260986328,
      "loss": 0.1023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.667719841003418,
      "rewards/margins": 1.667719841003418,
      "rewards/rejected": 0.0,
      "step": 1939
    },
    {
      "epoch": 10.837988826815643,
      "grad_norm": 0.8389981919913592,
      "learning_rate": 1.4253042493756991e-06,
      "logits/chosen": -3.350085496902466,
      "logits/rejected": -3.482414484024048,
      "logps/chosen": -0.4391435980796814,
      "logps/rejected": -41.90592956542969,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.651862621307373,
      "rewards/margins": 2.651862621307373,
      "rewards/rejected": 0.0,
      "step": 1940
    },
    {
      "epoch": 10.837988826815643,
      "eval_logits/chosen": -3.380390167236328,
      "eval_logits/rejected": -3.497129440307617,
      "eval_logps/chosen": -41.27619934082031,
      "eval_logps/rejected": -63.53928756713867,
      "eval_loss": 1.8204180002212524,
      "eval_rewards/accuracies": 0.15000000596046448,
      "eval_rewards/chosen": -1.1206157207489014,
      "eval_rewards/margins": -1.1206157207489014,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5816,
      "eval_samples_per_second": 12.118,
      "eval_steps_per_second": 0.391,
      "step": 1940
    },
    {
      "epoch": 10.843575418994414,
      "grad_norm": 0.3697955858688798,
      "learning_rate": 1.4225532998495205e-06,
      "logits/chosen": -3.3763725757598877,
      "logits/rejected": -3.4832074642181396,
      "logps/chosen": -1.7104624509811401,
      "logps/rejected": -47.716064453125,
      "loss": 0.1068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.341691017150879,
      "rewards/margins": 2.341691017150879,
      "rewards/rejected": 0.0,
      "step": 1941
    },
    {
      "epoch": 10.849162011173185,
      "grad_norm": 1.8953945870403675,
      "learning_rate": 1.4198039516010192e-06,
      "logits/chosen": -3.462455987930298,
      "logits/rejected": -3.399484395980835,
      "logps/chosen": -1.4690604209899902,
      "logps/rejected": -66.32952880859375,
      "loss": 0.0829,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.145908832550049,
      "rewards/margins": 3.145908832550049,
      "rewards/rejected": 0.0,
      "step": 1942
    },
    {
      "epoch": 10.854748603351956,
      "grad_norm": 1.6242784446013638,
      "learning_rate": 1.4170562087162188e-06,
      "logits/chosen": -2.86747145652771,
      "logits/rejected": -2.7883448600769043,
      "logps/chosen": -2.224454402923584,
      "logps/rejected": -48.64619445800781,
      "loss": 0.0772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4717113971710205,
      "rewards/margins": 3.4717113971710205,
      "rewards/rejected": 0.0,
      "step": 1943
    },
    {
      "epoch": 10.860335195530727,
      "grad_norm": 0.43734624783546483,
      "learning_rate": 1.4143100752787543e-06,
      "logits/chosen": -3.468783378601074,
      "logits/rejected": -3.496511459350586,
      "logps/chosen": -0.4575924873352051,
      "logps/rejected": -65.26058197021484,
      "loss": 0.0808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6470730304718018,
      "rewards/margins": 2.6470730304718018,
      "rewards/rejected": 0.0,
      "step": 1944
    },
    {
      "epoch": 10.865921787709498,
      "grad_norm": 2.8712715031681513,
      "learning_rate": 1.4115655553698686e-06,
      "logits/chosen": -3.587819814682007,
      "logits/rejected": -3.2320334911346436,
      "logps/chosen": -0.33169567584991455,
      "logps/rejected": -104.62492370605469,
      "loss": 0.1195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0345301628112793,
      "rewards/margins": 2.0345301628112793,
      "rewards/rejected": 0.0,
      "step": 1945
    },
    {
      "epoch": 10.871508379888269,
      "grad_norm": 1.255371676194468,
      "learning_rate": 1.4088226530684073e-06,
      "logits/chosen": -3.4530553817749023,
      "logits/rejected": -3.4478769302368164,
      "logps/chosen": -3.292661428451538,
      "logps/rejected": -52.60951232910156,
      "loss": 0.0997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.943061113357544,
      "rewards/margins": 2.943061113357544,
      "rewards/rejected": 0.0,
      "step": 1946
    },
    {
      "epoch": 10.87709497206704,
      "grad_norm": 0.5492287760155764,
      "learning_rate": 1.4060813724508116e-06,
      "logits/chosen": -3.6516594886779785,
      "logits/rejected": -3.604886054992676,
      "logps/chosen": -4.540423393249512,
      "logps/rejected": -51.70039749145508,
      "loss": 0.1238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.020253658294678,
      "rewards/margins": 4.020253658294678,
      "rewards/rejected": 0.0,
      "step": 1947
    },
    {
      "epoch": 10.88268156424581,
      "grad_norm": 1.762721631991342,
      "learning_rate": 1.403341717591113e-06,
      "logits/chosen": -3.5054879188537598,
      "logits/rejected": -3.558548927307129,
      "logps/chosen": -1.9523506164550781,
      "logps/rejected": -109.2947769165039,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4929206371307373,
      "rewards/margins": 2.4929206371307373,
      "rewards/rejected": 0.0,
      "step": 1948
    },
    {
      "epoch": 10.888268156424582,
      "grad_norm": 3.6860568156722175,
      "learning_rate": 1.4006036925609245e-06,
      "logits/chosen": -3.5458579063415527,
      "logits/rejected": -3.4672398567199707,
      "logps/chosen": -1.204513430595398,
      "logps/rejected": -70.13800048828125,
      "loss": 0.1002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.981576442718506,
      "rewards/margins": 2.981576442718506,
      "rewards/rejected": 0.0,
      "step": 1949
    },
    {
      "epoch": 10.893854748603353,
      "grad_norm": 0.742774865468539,
      "learning_rate": 1.397867301429443e-06,
      "logits/chosen": -3.4058454036712646,
      "logits/rejected": -3.4225943088531494,
      "logps/chosen": -19.898094177246094,
      "logps/rejected": -71.7255859375,
      "loss": 0.076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.574991226196289,
      "rewards/margins": 3.574991226196289,
      "rewards/rejected": 0.0,
      "step": 1950
    },
    {
      "epoch": 10.899441340782122,
      "grad_norm": 3.827482692929041,
      "learning_rate": 1.3951325482634286e-06,
      "logits/chosen": -3.320505142211914,
      "logits/rejected": -3.419077157974243,
      "logps/chosen": -3.9677786827087402,
      "logps/rejected": -86.2957992553711,
      "loss": 0.1386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6835715770721436,
      "rewards/margins": 1.6835715770721436,
      "rewards/rejected": 0.0,
      "step": 1951
    },
    {
      "epoch": 10.905027932960895,
      "grad_norm": 1.2618890110392729,
      "learning_rate": 1.3923994371272148e-06,
      "logits/chosen": -3.488513708114624,
      "logits/rejected": -3.481642961502075,
      "logps/chosen": -0.7222033143043518,
      "logps/rejected": -66.96025085449219,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6302990913391113,
      "rewards/margins": 2.6302990913391113,
      "rewards/rejected": 0.0,
      "step": 1952
    },
    {
      "epoch": 10.910614525139664,
      "grad_norm": 2.6749902360326514,
      "learning_rate": 1.3896679720826906e-06,
      "logits/chosen": -3.409677267074585,
      "logits/rejected": -3.4608354568481445,
      "logps/chosen": -1.8787163496017456,
      "logps/rejected": -57.18359375,
      "loss": 0.1267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6481831073760986,
      "rewards/margins": 2.6481831073760986,
      "rewards/rejected": 0.0,
      "step": 1953
    },
    {
      "epoch": 10.916201117318435,
      "grad_norm": 1.1323008944472155,
      "learning_rate": 1.3869381571892988e-06,
      "logits/chosen": -3.5333642959594727,
      "logits/rejected": -3.572920083999634,
      "logps/chosen": -14.31783390045166,
      "logps/rejected": -56.976905822753906,
      "loss": 0.0909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.775381088256836,
      "rewards/margins": 4.775381088256836,
      "rewards/rejected": 0.0,
      "step": 1954
    },
    {
      "epoch": 10.921787709497206,
      "grad_norm": 0.7329378633674726,
      "learning_rate": 1.3842099965040306e-06,
      "logits/chosen": -3.8064165115356445,
      "logits/rejected": -3.7237932682037354,
      "logps/chosen": -0.6585365533828735,
      "logps/rejected": -85.82054138183594,
      "loss": 0.1218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9117242097854614,
      "rewards/margins": 1.9117242097854614,
      "rewards/rejected": 0.0,
      "step": 1955
    },
    {
      "epoch": 10.927374301675977,
      "grad_norm": 0.3474960202215662,
      "learning_rate": 1.3814834940814192e-06,
      "logits/chosen": -3.5235888957977295,
      "logits/rejected": -3.6640632152557373,
      "logps/chosen": -0.6185259819030762,
      "logps/rejected": -47.316001892089844,
      "loss": 0.1054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.929185628890991,
      "rewards/margins": 2.929185628890991,
      "rewards/rejected": 0.0,
      "step": 1956
    },
    {
      "epoch": 10.932960893854748,
      "grad_norm": 1.3691782749920565,
      "learning_rate": 1.3787586539735325e-06,
      "logits/chosen": -3.856191635131836,
      "logits/rejected": -3.4850780963897705,
      "logps/chosen": -0.9431565999984741,
      "logps/rejected": -70.31283569335938,
      "loss": 0.0987,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5125434398651123,
      "rewards/margins": 2.5125434398651123,
      "rewards/rejected": 0.0,
      "step": 1957
    },
    {
      "epoch": 10.938547486033519,
      "grad_norm": 0.6205406249095144,
      "learning_rate": 1.3760354802299671e-06,
      "logits/chosen": -3.5989227294921875,
      "logits/rejected": -3.4922947883605957,
      "logps/chosen": -0.43777763843536377,
      "logps/rejected": -39.418968200683594,
      "loss": 0.0936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.013535261154175,
      "rewards/margins": 2.013535261154175,
      "rewards/rejected": 0.0,
      "step": 1958
    },
    {
      "epoch": 10.94413407821229,
      "grad_norm": 2.7878976684032266,
      "learning_rate": 1.3733139768978474e-06,
      "logits/chosen": -3.5515856742858887,
      "logits/rejected": -3.3022613525390625,
      "logps/chosen": -1.2453441619873047,
      "logps/rejected": -62.0585823059082,
      "loss": 0.0889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.508762836456299,
      "rewards/margins": 3.508762836456299,
      "rewards/rejected": 0.0,
      "step": 1959
    },
    {
      "epoch": 10.949720670391061,
      "grad_norm": 0.4185645275787515,
      "learning_rate": 1.3705941480218077e-06,
      "logits/chosen": -3.5387067794799805,
      "logits/rejected": -3.3467071056365967,
      "logps/chosen": -1.7016234397888184,
      "logps/rejected": -51.99770736694336,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.489067554473877,
      "rewards/margins": 2.489067554473877,
      "rewards/rejected": 0.0,
      "step": 1960
    },
    {
      "epoch": 10.949720670391061,
      "eval_logits/chosen": -3.3719730377197266,
      "eval_logits/rejected": -3.4910519123077393,
      "eval_logps/chosen": -40.053382873535156,
      "eval_logps/rejected": -61.68449783325195,
      "eval_loss": 1.7629327774047852,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -0.9983338117599487,
      "eval_rewards/margins": -0.9983338117599487,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5562,
      "eval_samples_per_second": 12.13,
      "eval_steps_per_second": 0.391,
      "step": 1960
    },
    {
      "epoch": 10.955307262569832,
      "grad_norm": 0.30703139089312736,
      "learning_rate": 1.3678759976440012e-06,
      "logits/chosen": -3.6893060207366943,
      "logits/rejected": -3.802746534347534,
      "logps/chosen": -7.111313343048096,
      "logps/rejected": -85.58920288085938,
      "loss": 0.0887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6622395515441895,
      "rewards/margins": 3.6622395515441895,
      "rewards/rejected": 0.0,
      "step": 1961
    },
    {
      "epoch": 10.960893854748603,
      "grad_norm": 0.9757232861832693,
      "learning_rate": 1.3651595298040824e-06,
      "logits/chosen": -3.7464218139648438,
      "logits/rejected": -3.1319453716278076,
      "logps/chosen": -0.4653093218803406,
      "logps/rejected": -125.25736999511719,
      "loss": 0.1123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2709813117980957,
      "rewards/margins": 1.2709813117980957,
      "rewards/rejected": 0.0,
      "step": 1962
    },
    {
      "epoch": 10.966480446927374,
      "grad_norm": 2.0062094806385646,
      "learning_rate": 1.3624447485392065e-06,
      "logits/chosen": -3.4685118198394775,
      "logits/rejected": -3.5160629749298096,
      "logps/chosen": -0.44039735198020935,
      "logps/rejected": -129.36048889160156,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2523200511932373,
      "rewards/margins": 2.2523200511932373,
      "rewards/rejected": 0.0,
      "step": 1963
    },
    {
      "epoch": 10.972067039106145,
      "grad_norm": 0.8587637775341056,
      "learning_rate": 1.3597316578840216e-06,
      "logits/chosen": -3.527029037475586,
      "logits/rejected": -3.247988700866699,
      "logps/chosen": -1.413446307182312,
      "logps/rejected": -57.140480041503906,
      "loss": 0.0818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.653505802154541,
      "rewards/margins": 2.653505802154541,
      "rewards/rejected": 0.0,
      "step": 1964
    },
    {
      "epoch": 10.977653631284916,
      "grad_norm": 3.524009642805855,
      "learning_rate": 1.3570202618706641e-06,
      "logits/chosen": -3.6982839107513428,
      "logits/rejected": -3.646101951599121,
      "logps/chosen": -0.358418345451355,
      "logps/rejected": -57.208351135253906,
      "loss": 0.1097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.733020782470703,
      "rewards/margins": 2.733020782470703,
      "rewards/rejected": 0.0,
      "step": 1965
    },
    {
      "epoch": 10.983240223463687,
      "grad_norm": 0.6180492404524871,
      "learning_rate": 1.3543105645287505e-06,
      "logits/chosen": -3.4708380699157715,
      "logits/rejected": -3.505613088607788,
      "logps/chosen": -10.452714920043945,
      "logps/rejected": -94.30807495117188,
      "loss": 0.0948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.015303611755371,
      "rewards/margins": 4.015303611755371,
      "rewards/rejected": 0.0,
      "step": 1966
    },
    {
      "epoch": 10.988826815642458,
      "grad_norm": 0.7473328563142749,
      "learning_rate": 1.3516025698853745e-06,
      "logits/chosen": -3.4737179279327393,
      "logits/rejected": -3.3661861419677734,
      "logps/chosen": -4.149295806884766,
      "logps/rejected": -83.95880889892578,
      "loss": 0.0863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1812515258789062,
      "rewards/margins": 3.1812515258789062,
      "rewards/rejected": 0.0,
      "step": 1967
    },
    {
      "epoch": 10.994413407821229,
      "grad_norm": 2.2948053970584104,
      "learning_rate": 1.348896281965098e-06,
      "logits/chosen": -3.671900749206543,
      "logits/rejected": -3.521353244781494,
      "logps/chosen": -0.8432856202125549,
      "logps/rejected": -75.6719970703125,
      "loss": 0.1029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0071892738342285,
      "rewards/margins": 3.0071892738342285,
      "rewards/rejected": 0.0,
      "step": 1968
    },
    {
      "epoch": 11.0,
      "grad_norm": 0.2992588373393029,
      "learning_rate": 1.3461917047899458e-06,
      "logits/chosen": -3.4457528591156006,
      "logits/rejected": -3.4958903789520264,
      "logps/chosen": -0.8118683695793152,
      "logps/rejected": -54.36821746826172,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.332993507385254,
      "rewards/margins": 2.332993507385254,
      "rewards/rejected": 0.0,
      "step": 1969
    },
    {
      "epoch": 11.005586592178771,
      "grad_norm": 1.0217706897277232,
      "learning_rate": 1.343488842379404e-06,
      "logits/chosen": -3.2717134952545166,
      "logits/rejected": -3.187403917312622,
      "logps/chosen": -2.606665849685669,
      "logps/rejected": -44.91965866088867,
      "loss": 0.1173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0489940643310547,
      "rewards/margins": 3.0489940643310547,
      "rewards/rejected": 0.0,
      "step": 1970
    },
    {
      "epoch": 11.011173184357542,
      "grad_norm": 1.6046952621116972,
      "learning_rate": 1.3407876987504043e-06,
      "logits/chosen": -3.583498954772949,
      "logits/rejected": -3.642273187637329,
      "logps/chosen": -8.27937126159668,
      "logps/rejected": -63.010772705078125,
      "loss": 0.1104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.594078540802002,
      "rewards/margins": 3.594078540802002,
      "rewards/rejected": 0.0,
      "step": 1971
    },
    {
      "epoch": 11.016759776536313,
      "grad_norm": 0.3041284204250699,
      "learning_rate": 1.3380882779173294e-06,
      "logits/chosen": -3.8382058143615723,
      "logits/rejected": -3.658895254135132,
      "logps/chosen": -3.5260868072509766,
      "logps/rejected": -51.575496673583984,
      "loss": 0.0836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.281654357910156,
      "rewards/margins": 4.281654357910156,
      "rewards/rejected": 0.0,
      "step": 1972
    },
    {
      "epoch": 11.022346368715084,
      "grad_norm": 0.4491315745870193,
      "learning_rate": 1.335390583891999e-06,
      "logits/chosen": -3.111011028289795,
      "logits/rejected": -3.1003363132476807,
      "logps/chosen": -0.4808184504508972,
      "logps/rejected": -64.08306884765625,
      "loss": 0.0968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2796454429626465,
      "rewards/margins": 2.2796454429626465,
      "rewards/rejected": 0.0,
      "step": 1973
    },
    {
      "epoch": 11.027932960893855,
      "grad_norm": 1.193865672562742,
      "learning_rate": 1.332694620683666e-06,
      "logits/chosen": -3.68479323387146,
      "logits/rejected": -3.7551462650299072,
      "logps/chosen": -1.5174839496612549,
      "logps/rejected": -60.55691909790039,
      "loss": 0.0869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5974979400634766,
      "rewards/margins": 3.5974979400634766,
      "rewards/rejected": 0.0,
      "step": 1974
    },
    {
      "epoch": 11.033519553072626,
      "grad_norm": 0.21316353381747052,
      "learning_rate": 1.330000392299013e-06,
      "logits/chosen": -3.4833590984344482,
      "logits/rejected": -3.4203081130981445,
      "logps/chosen": -22.37731170654297,
      "logps/rejected": -50.88762664794922,
      "loss": 0.0787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4333529472351074,
      "rewards/margins": 3.4333529472351074,
      "rewards/rejected": 0.0,
      "step": 1975
    },
    {
      "epoch": 11.039106145251397,
      "grad_norm": 0.6177208162237641,
      "learning_rate": 1.327307902742142e-06,
      "logits/chosen": -3.663641929626465,
      "logits/rejected": -3.6082499027252197,
      "logps/chosen": -7.348990440368652,
      "logps/rejected": -51.60411071777344,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.974184036254883,
      "rewards/margins": 3.974184036254883,
      "rewards/rejected": 0.0,
      "step": 1976
    },
    {
      "epoch": 11.044692737430168,
      "grad_norm": 0.14052905228071041,
      "learning_rate": 1.3246171560145725e-06,
      "logits/chosen": -3.4699089527130127,
      "logits/rejected": -3.602429151535034,
      "logps/chosen": -1.354343295097351,
      "logps/rejected": -78.89804077148438,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.870638608932495,
      "rewards/margins": 2.870638608932495,
      "rewards/rejected": 0.0,
      "step": 1977
    },
    {
      "epoch": 11.050279329608939,
      "grad_norm": 0.34879090563121246,
      "learning_rate": 1.321928156115233e-06,
      "logits/chosen": -3.4804749488830566,
      "logits/rejected": -3.473572254180908,
      "logps/chosen": -5.32487154006958,
      "logps/rejected": -68.65103149414062,
      "loss": 0.075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.599088668823242,
      "rewards/margins": 3.599088668823242,
      "rewards/rejected": 0.0,
      "step": 1978
    },
    {
      "epoch": 11.05586592178771,
      "grad_norm": 1.1759566344165213,
      "learning_rate": 1.3192409070404582e-06,
      "logits/chosen": -3.580338478088379,
      "logits/rejected": -3.4270260334014893,
      "logps/chosen": -0.20871601998806,
      "logps/rejected": -55.795005798339844,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5261716842651367,
      "rewards/margins": 2.5261716842651367,
      "rewards/rejected": 0.0,
      "step": 1979
    },
    {
      "epoch": 11.061452513966481,
      "grad_norm": 1.2460271373377116,
      "learning_rate": 1.3165554127839754e-06,
      "logits/chosen": -3.626626968383789,
      "logits/rejected": -3.6134192943573,
      "logps/chosen": -8.5318021774292,
      "logps/rejected": -82.13290405273438,
      "loss": 0.1322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0537943840026855,
      "rewards/margins": 3.0537943840026855,
      "rewards/rejected": 0.0,
      "step": 1980
    },
    {
      "epoch": 11.061452513966481,
      "eval_logits/chosen": -3.3757095336914062,
      "eval_logits/rejected": -3.494736433029175,
      "eval_logps/chosen": -41.00090026855469,
      "eval_logps/rejected": -62.60357666015625,
      "eval_loss": 1.8170552253723145,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -1.093085765838623,
      "eval_rewards/margins": -1.093085765838623,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5175,
      "eval_samples_per_second": 12.149,
      "eval_steps_per_second": 0.392,
      "step": 1980
    },
    {
      "epoch": 11.067039106145252,
      "grad_norm": 1.218432207352182,
      "learning_rate": 1.3138716773369104e-06,
      "logits/chosen": -3.323758840560913,
      "logits/rejected": -2.9491257667541504,
      "logps/chosen": -0.4414094090461731,
      "logps/rejected": -71.10795593261719,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1178158521652222,
      "rewards/margins": 1.1178158521652222,
      "rewards/rejected": 0.0,
      "step": 1981
    },
    {
      "epoch": 11.072625698324023,
      "grad_norm": 0.2512365599964245,
      "learning_rate": 1.3111897046877704e-06,
      "logits/chosen": -3.698585033416748,
      "logits/rejected": -3.660135269165039,
      "logps/chosen": -1.5125116109848022,
      "logps/rejected": -48.46613311767578,
      "loss": 0.0907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4537014961242676,
      "rewards/margins": 3.4537014961242676,
      "rewards/rejected": 0.0,
      "step": 1982
    },
    {
      "epoch": 11.078212290502794,
      "grad_norm": 0.15242549854036247,
      "learning_rate": 1.3085094988224448e-06,
      "logits/chosen": -3.228538751602173,
      "logits/rejected": -3.2212462425231934,
      "logps/chosen": -7.33699369430542,
      "logps/rejected": -76.6466064453125,
      "loss": 0.1058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6904194355010986,
      "rewards/margins": 3.6904194355010986,
      "rewards/rejected": 0.0,
      "step": 1983
    },
    {
      "epoch": 11.083798882681565,
      "grad_norm": 1.0181600267029562,
      "learning_rate": 1.3058310637241966e-06,
      "logits/chosen": -3.5749239921569824,
      "logits/rejected": -3.579148769378662,
      "logps/chosen": -11.650062561035156,
      "logps/rejected": -76.98095703125,
      "loss": 0.1109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0073089599609375,
      "rewards/margins": 3.0073089599609375,
      "rewards/rejected": 0.0,
      "step": 1984
    },
    {
      "epoch": 11.089385474860336,
      "grad_norm": 2.233511712777928,
      "learning_rate": 1.303154403373658e-06,
      "logits/chosen": -3.207616090774536,
      "logits/rejected": -3.314760684967041,
      "logps/chosen": -5.020147800445557,
      "logps/rejected": -42.78601837158203,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4789881706237793,
      "rewards/margins": 3.4789881706237793,
      "rewards/rejected": 0.0,
      "step": 1985
    },
    {
      "epoch": 11.094972067039107,
      "grad_norm": 0.6097976890940501,
      "learning_rate": 1.3004795217488222e-06,
      "logits/chosen": -3.378551483154297,
      "logits/rejected": -3.534003734588623,
      "logps/chosen": -2.224734306335449,
      "logps/rejected": -99.10171508789062,
      "loss": 0.0849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5509774684906006,
      "rewards/margins": 2.5509774684906006,
      "rewards/rejected": 0.0,
      "step": 1986
    },
    {
      "epoch": 11.100558659217878,
      "grad_norm": 0.18777301801440888,
      "learning_rate": 1.2978064228250392e-06,
      "logits/chosen": -3.559340476989746,
      "logits/rejected": -3.3054168224334717,
      "logps/chosen": -4.793196678161621,
      "logps/rejected": -90.91523742675781,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.173738956451416,
      "rewards/margins": 3.173738956451416,
      "rewards/rejected": 0.0,
      "step": 1987
    },
    {
      "epoch": 11.106145251396647,
      "grad_norm": 0.3017721778620066,
      "learning_rate": 1.2951351105750125e-06,
      "logits/chosen": -3.6008262634277344,
      "logits/rejected": -3.4951438903808594,
      "logps/chosen": -1.304526925086975,
      "logps/rejected": -78.948974609375,
      "loss": 0.085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.508617401123047,
      "rewards/margins": 2.508617401123047,
      "rewards/rejected": 0.0,
      "step": 1988
    },
    {
      "epoch": 11.111731843575418,
      "grad_norm": 0.5466765674024163,
      "learning_rate": 1.2924655889687848e-06,
      "logits/chosen": -3.3460233211517334,
      "logits/rejected": -3.4405462741851807,
      "logps/chosen": -0.30123621225357056,
      "logps/rejected": -77.57840728759766,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.788219928741455,
      "rewards/margins": 2.788219928741455,
      "rewards/rejected": 0.0,
      "step": 1989
    },
    {
      "epoch": 11.11731843575419,
      "grad_norm": 2.140557840110226,
      "learning_rate": 1.2897978619737428e-06,
      "logits/chosen": -3.371725082397461,
      "logits/rejected": -3.438117027282715,
      "logps/chosen": -0.2903643846511841,
      "logps/rejected": -83.59112548828125,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.117947578430176,
      "rewards/margins": 2.117947578430176,
      "rewards/rejected": 0.0,
      "step": 1990
    },
    {
      "epoch": 11.12290502793296,
      "grad_norm": 0.5649912091574911,
      "learning_rate": 1.287131933554603e-06,
      "logits/chosen": -3.500460386276245,
      "logits/rejected": -3.6598591804504395,
      "logps/chosen": -1.1989589929580688,
      "logps/rejected": -67.79438018798828,
      "loss": 0.0985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8182015419006348,
      "rewards/margins": 3.8182015419006348,
      "rewards/rejected": 0.0,
      "step": 1991
    },
    {
      "epoch": 11.128491620111731,
      "grad_norm": 1.2362860398492093,
      "learning_rate": 1.2844678076734096e-06,
      "logits/chosen": -3.2802748680114746,
      "logits/rejected": -3.2406277656555176,
      "logps/chosen": -1.5266366004943848,
      "logps/rejected": -80.24435424804688,
      "loss": 0.0808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8455498218536377,
      "rewards/margins": 3.8455498218536377,
      "rewards/rejected": 0.0,
      "step": 1992
    },
    {
      "epoch": 11.134078212290502,
      "grad_norm": 0.5867358748202203,
      "learning_rate": 1.2818054882895287e-06,
      "logits/chosen": -3.6321635246276855,
      "logits/rejected": -3.721055030822754,
      "logps/chosen": -0.17887702584266663,
      "logps/rejected": -68.75697326660156,
      "loss": 0.1246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.180601119995117,
      "rewards/margins": 2.180601119995117,
      "rewards/rejected": 0.0,
      "step": 1993
    },
    {
      "epoch": 11.139664804469273,
      "grad_norm": 0.7406198565831181,
      "learning_rate": 1.279144979359641e-06,
      "logits/chosen": -3.300584554672241,
      "logits/rejected": -3.1296565532684326,
      "logps/chosen": -0.7058359384536743,
      "logps/rejected": -63.31174087524414,
      "loss": 0.0863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4095675945281982,
      "rewards/margins": 3.4095675945281982,
      "rewards/rejected": 0.0,
      "step": 1994
    },
    {
      "epoch": 11.145251396648044,
      "grad_norm": 1.5420641490879652,
      "learning_rate": 1.2764862848377363e-06,
      "logits/chosen": -3.6271448135375977,
      "logits/rejected": -3.5483219623565674,
      "logps/chosen": -13.053669929504395,
      "logps/rejected": -53.63007736206055,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.471860885620117,
      "rewards/margins": 4.471860885620117,
      "rewards/rejected": 0.0,
      "step": 1995
    },
    {
      "epoch": 11.150837988826815,
      "grad_norm": 0.47769495814745333,
      "learning_rate": 1.273829408675108e-06,
      "logits/chosen": -3.590665102005005,
      "logits/rejected": -3.4244651794433594,
      "logps/chosen": -6.671918869018555,
      "logps/rejected": -95.93168640136719,
      "loss": 0.0789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.704322338104248,
      "rewards/margins": 3.704322338104248,
      "rewards/rejected": 0.0,
      "step": 1996
    },
    {
      "epoch": 11.156424581005586,
      "grad_norm": 0.3203389830238645,
      "learning_rate": 1.2711743548203475e-06,
      "logits/chosen": -3.6140873432159424,
      "logits/rejected": -3.5000052452087402,
      "logps/chosen": -0.7334027290344238,
      "logps/rejected": -59.23403549194336,
      "loss": 0.0904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4092206954956055,
      "rewards/margins": 2.4092206954956055,
      "rewards/rejected": 0.0,
      "step": 1997
    },
    {
      "epoch": 11.162011173184357,
      "grad_norm": 0.14982743075401792,
      "learning_rate": 1.2685211272193365e-06,
      "logits/chosen": -3.40109920501709,
      "logits/rejected": -3.602478504180908,
      "logps/chosen": -0.12563037872314453,
      "logps/rejected": -81.82844543457031,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8913253545761108,
      "rewards/margins": 1.8913253545761108,
      "rewards/rejected": 0.0,
      "step": 1998
    },
    {
      "epoch": 11.167597765363128,
      "grad_norm": 0.316545453470076,
      "learning_rate": 1.2658697298152472e-06,
      "logits/chosen": -3.403529644012451,
      "logits/rejected": -3.2880196571350098,
      "logps/chosen": -2.3204259872436523,
      "logps/rejected": -52.657127380371094,
      "loss": 0.0727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9262375831604004,
      "rewards/margins": 3.9262375831604004,
      "rewards/rejected": 0.0,
      "step": 1999
    },
    {
      "epoch": 11.1731843575419,
      "grad_norm": 0.23595729957696468,
      "learning_rate": 1.2632201665485236e-06,
      "logits/chosen": -3.444183111190796,
      "logits/rejected": -3.5231740474700928,
      "logps/chosen": -0.5426976084709167,
      "logps/rejected": -84.52286529541016,
      "loss": 0.096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6543660163879395,
      "rewards/margins": 2.6543660163879395,
      "rewards/rejected": 0.0,
      "step": 2000
    },
    {
      "epoch": 11.1731843575419,
      "eval_logits/chosen": -3.363694667816162,
      "eval_logits/rejected": -3.487701416015625,
      "eval_logps/chosen": -42.42475509643555,
      "eval_logps/rejected": -66.212158203125,
      "eval_loss": 1.9801753759384155,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -1.235471487045288,
      "eval_rewards/margins": -1.235471487045288,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6682,
      "eval_samples_per_second": 12.077,
      "eval_steps_per_second": 0.39,
      "step": 2000
    },
    {
      "epoch": 11.17877094972067,
      "grad_norm": 0.5485664483589889,
      "learning_rate": 1.260572441356892e-06,
      "logits/chosen": -3.7426741123199463,
      "logits/rejected": -3.6024863719940186,
      "logps/chosen": -1.617774248123169,
      "logps/rejected": -63.73008346557617,
      "loss": 0.1114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0671334266662598,
      "rewards/margins": 2.0671334266662598,
      "rewards/rejected": 0.0,
      "step": 2001
    },
    {
      "epoch": 11.184357541899441,
      "grad_norm": 0.8285159256023984,
      "learning_rate": 1.2579265581753429e-06,
      "logits/chosen": -3.749650716781616,
      "logits/rejected": -3.6289634704589844,
      "logps/chosen": -0.5560716390609741,
      "logps/rejected": -90.3453140258789,
      "loss": 0.1002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1482648849487305,
      "rewards/margins": 2.1482648849487305,
      "rewards/rejected": 0.0,
      "step": 2002
    },
    {
      "epoch": 11.189944134078212,
      "grad_norm": 0.3163890349773343,
      "learning_rate": 1.2552825209361292e-06,
      "logits/chosen": -3.5729429721832275,
      "logits/rejected": -3.406898260116577,
      "logps/chosen": -3.87595796585083,
      "logps/rejected": -48.53721618652344,
      "loss": 0.1029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8156652450561523,
      "rewards/margins": 3.8156652450561523,
      "rewards/rejected": 0.0,
      "step": 2003
    },
    {
      "epoch": 11.195530726256983,
      "grad_norm": 0.21532167733629315,
      "learning_rate": 1.2526403335687614e-06,
      "logits/chosen": -3.697666645050049,
      "logits/rejected": -3.8748302459716797,
      "logps/chosen": -0.05199631676077843,
      "logps/rejected": -45.24409484863281,
      "loss": 0.1066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4362645149230957,
      "rewards/margins": 1.4362645149230957,
      "rewards/rejected": 0.0,
      "step": 2004
    },
    {
      "epoch": 11.201117318435754,
      "grad_norm": 1.0388518406393341,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": -3.3242759704589844,
      "logits/rejected": -3.409313440322876,
      "logps/chosen": -0.4138806462287903,
      "logps/rejected": -70.34220886230469,
      "loss": 0.0963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2606801986694336,
      "rewards/margins": 2.2606801986694336,
      "rewards/rejected": 0.0,
      "step": 2005
    },
    {
      "epoch": 11.206703910614525,
      "grad_norm": 0.6211228758497738,
      "learning_rate": 1.2473615241538526e-06,
      "logits/chosen": -3.8346052169799805,
      "logits/rejected": -3.662896156311035,
      "logps/chosen": -0.7217550277709961,
      "logps/rejected": -40.622283935546875,
      "loss": 0.1028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7094969749450684,
      "rewards/margins": 2.7094969749450684,
      "rewards/rejected": 0.0,
      "step": 2006
    },
    {
      "epoch": 11.212290502793296,
      "grad_norm": 0.20782944487037594,
      "learning_rate": 1.2447249099515614e-06,
      "logits/chosen": -3.596827745437622,
      "logits/rejected": -3.5618011951446533,
      "logps/chosen": -2.40287709236145,
      "logps/rejected": -57.84024429321289,
      "loss": 0.076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.826413154602051,
      "rewards/margins": 3.826413154602051,
      "rewards/rejected": 0.0,
      "step": 2007
    },
    {
      "epoch": 11.217877094972067,
      "grad_norm": 0.3753760801718792,
      "learning_rate": 1.2420901613116085e-06,
      "logits/chosen": -3.511869192123413,
      "logits/rejected": -3.4096953868865967,
      "logps/chosen": -0.5595200657844543,
      "logps/rejected": -81.786376953125,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.323298454284668,
      "rewards/margins": 2.323298454284668,
      "rewards/rejected": 0.0,
      "step": 2008
    },
    {
      "epoch": 11.223463687150838,
      "grad_norm": 0.2005920434655265,
      "learning_rate": 1.2394572821496953e-06,
      "logits/chosen": -3.7069270610809326,
      "logits/rejected": -3.6378891468048096,
      "logps/chosen": -0.14432762563228607,
      "logps/rejected": -64.90775299072266,
      "loss": 0.0835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4250612258911133,
      "rewards/margins": 2.4250612258911133,
      "rewards/rejected": 0.0,
      "step": 2009
    },
    {
      "epoch": 11.22905027932961,
      "grad_norm": 0.7259347318145596,
      "learning_rate": 1.236826276378752e-06,
      "logits/chosen": -3.5983476638793945,
      "logits/rejected": -3.5774810314178467,
      "logps/chosen": -0.1632506251335144,
      "logps/rejected": -72.68748474121094,
      "loss": 0.1046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0105669498443604,
      "rewards/margins": 2.0105669498443604,
      "rewards/rejected": 0.0,
      "step": 2010
    },
    {
      "epoch": 11.23463687150838,
      "grad_norm": 0.2928745613193596,
      "learning_rate": 1.2341971479089209e-06,
      "logits/chosen": -3.329390048980713,
      "logits/rejected": -3.36487078666687,
      "logps/chosen": -1.0230234861373901,
      "logps/rejected": -46.777950286865234,
      "loss": 0.0828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6884872913360596,
      "rewards/margins": 2.6884872913360596,
      "rewards/rejected": 0.0,
      "step": 2011
    },
    {
      "epoch": 11.240223463687151,
      "grad_norm": 0.14860147632535856,
      "learning_rate": 1.2315699006475555e-06,
      "logits/chosen": -3.415602684020996,
      "logits/rejected": -3.4792656898498535,
      "logps/chosen": -11.695252418518066,
      "logps/rejected": -72.0348892211914,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.691225290298462,
      "rewards/margins": 3.691225290298462,
      "rewards/rejected": 0.0,
      "step": 2012
    },
    {
      "epoch": 11.245810055865922,
      "grad_norm": 0.777653269481215,
      "learning_rate": 1.2289445384992116e-06,
      "logits/chosen": -3.06030011177063,
      "logits/rejected": -3.1394362449645996,
      "logps/chosen": -0.8841867446899414,
      "logps/rejected": -60.745460510253906,
      "loss": 0.086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4885358810424805,
      "rewards/margins": 3.4885358810424805,
      "rewards/rejected": 0.0,
      "step": 2013
    },
    {
      "epoch": 11.251396648044693,
      "grad_norm": 0.4715058294136999,
      "learning_rate": 1.226321065365648e-06,
      "logits/chosen": -3.3867573738098145,
      "logits/rejected": -3.2592625617980957,
      "logps/chosen": -4.840304374694824,
      "logps/rejected": -61.1673698425293,
      "loss": 0.114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.482936382293701,
      "rewards/margins": 3.482936382293701,
      "rewards/rejected": 0.0,
      "step": 2014
    },
    {
      "epoch": 11.256983240223464,
      "grad_norm": 1.1965551830871173,
      "learning_rate": 1.2236994851458101e-06,
      "logits/chosen": -3.679734468460083,
      "logits/rejected": -3.7000327110290527,
      "logps/chosen": -0.628045916557312,
      "logps/rejected": -55.733436584472656,
      "loss": 0.1173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7780635356903076,
      "rewards/margins": 2.7780635356903076,
      "rewards/rejected": 0.0,
      "step": 2015
    },
    {
      "epoch": 11.262569832402235,
      "grad_norm": 0.1990450205299003,
      "learning_rate": 1.221079801735833e-06,
      "logits/chosen": -3.2515339851379395,
      "logits/rejected": -3.2742526531219482,
      "logps/chosen": -5.051718235015869,
      "logps/rejected": -66.07675170898438,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.468876361846924,
      "rewards/margins": 3.468876361846924,
      "rewards/rejected": 0.0,
      "step": 2016
    },
    {
      "epoch": 11.268156424581006,
      "grad_norm": 0.12113217647597457,
      "learning_rate": 1.218462019029036e-06,
      "logits/chosen": -3.7115538120269775,
      "logits/rejected": -3.5704989433288574,
      "logps/chosen": -0.3689883053302765,
      "logps/rejected": -87.79774475097656,
      "loss": 0.0742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6640357971191406,
      "rewards/margins": 2.6640357971191406,
      "rewards/rejected": 0.0,
      "step": 2017
    },
    {
      "epoch": 11.273743016759777,
      "grad_norm": 2.420975143573792,
      "learning_rate": 1.2158461409159058e-06,
      "logits/chosen": -3.4564201831817627,
      "logits/rejected": -3.4792425632476807,
      "logps/chosen": -0.8609983325004578,
      "logps/rejected": -83.78791809082031,
      "loss": 0.1337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1490585803985596,
      "rewards/margins": 3.1490585803985596,
      "rewards/rejected": 0.0,
      "step": 2018
    },
    {
      "epoch": 11.279329608938548,
      "grad_norm": 0.5047132279646139,
      "learning_rate": 1.2132321712841067e-06,
      "logits/chosen": -3.5586676597595215,
      "logits/rejected": -3.493823528289795,
      "logps/chosen": -0.19511762261390686,
      "logps/rejected": -102.56202697753906,
      "loss": 0.0992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3996028900146484,
      "rewards/margins": 2.3996028900146484,
      "rewards/rejected": 0.0,
      "step": 2019
    },
    {
      "epoch": 11.28491620111732,
      "grad_norm": 3.668856546111845,
      "learning_rate": 1.2106201140184617e-06,
      "logits/chosen": -3.6005795001983643,
      "logits/rejected": -3.485306978225708,
      "logps/chosen": -5.675540924072266,
      "logps/rejected": -74.52142333984375,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3155651092529297,
      "rewards/margins": 3.3155651092529297,
      "rewards/rejected": 0.0,
      "step": 2020
    },
    {
      "epoch": 11.28491620111732,
      "eval_logits/chosen": -3.36620831489563,
      "eval_logits/rejected": -3.489704132080078,
      "eval_logps/chosen": -43.62875747680664,
      "eval_logps/rejected": -68.3808822631836,
      "eval_loss": 2.094752311706543,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.3558714389801025,
      "eval_rewards/margins": -1.3558714389801025,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5213,
      "eval_samples_per_second": 12.147,
      "eval_steps_per_second": 0.392,
      "step": 2020
    },
    {
      "epoch": 11.29050279329609,
      "grad_norm": 0.7730506762985184,
      "learning_rate": 1.2080099730009533e-06,
      "logits/chosen": -3.207867383956909,
      "logits/rejected": -3.4357550144195557,
      "logps/chosen": -0.06953571736812592,
      "logps/rejected": -74.87329864501953,
      "loss": 0.0932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4149808883666992,
      "rewards/margins": 1.4149808883666992,
      "rewards/rejected": 0.0,
      "step": 2021
    },
    {
      "epoch": 11.296089385474861,
      "grad_norm": 0.3569267689098826,
      "learning_rate": 1.2054017521107158e-06,
      "logits/chosen": -3.516810894012451,
      "logits/rejected": -3.4730916023254395,
      "logps/chosen": -3.3637492656707764,
      "logps/rejected": -95.37886047363281,
      "loss": 0.1248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.350259780883789,
      "rewards/margins": 3.350259780883789,
      "rewards/rejected": 0.0,
      "step": 2022
    },
    {
      "epoch": 11.30167597765363,
      "grad_norm": 0.3563805302657981,
      "learning_rate": 1.2027954552240301e-06,
      "logits/chosen": -3.475813627243042,
      "logits/rejected": -3.5504002571105957,
      "logps/chosen": -0.670828104019165,
      "logps/rejected": -53.31642532348633,
      "loss": 0.0964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.729583263397217,
      "rewards/margins": 2.729583263397217,
      "rewards/rejected": 0.0,
      "step": 2023
    },
    {
      "epoch": 11.307262569832401,
      "grad_norm": 0.15144188929045269,
      "learning_rate": 1.2001910862143176e-06,
      "logits/chosen": -3.6179404258728027,
      "logits/rejected": -3.697195053100586,
      "logps/chosen": -2.534743309020996,
      "logps/rejected": -79.23489379882812,
      "loss": 0.0805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.025665760040283,
      "rewards/margins": 3.025665760040283,
      "rewards/rejected": 0.0,
      "step": 2024
    },
    {
      "epoch": 11.312849162011172,
      "grad_norm": 0.5390509998050078,
      "learning_rate": 1.1975886489521342e-06,
      "logits/chosen": -3.783571720123291,
      "logits/rejected": -3.0726187229156494,
      "logps/chosen": -0.18711137771606445,
      "logps/rejected": -109.8524169921875,
      "loss": 0.1065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2568883895874023,
      "rewards/margins": 2.2568883895874023,
      "rewards/rejected": 0.0,
      "step": 2025
    },
    {
      "epoch": 11.318435754189943,
      "grad_norm": 0.8069456953224077,
      "learning_rate": 1.194988147305165e-06,
      "logits/chosen": -3.1587204933166504,
      "logits/rejected": -3.2754621505737305,
      "logps/chosen": -11.931844711303711,
      "logps/rejected": -61.38832473754883,
      "loss": 0.0666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.723265171051025,
      "rewards/margins": 4.723265171051025,
      "rewards/rejected": 0.0,
      "step": 2026
    },
    {
      "epoch": 11.324022346368714,
      "grad_norm": 0.35692376728527364,
      "learning_rate": 1.192389585138218e-06,
      "logits/chosen": -3.2549374103546143,
      "logits/rejected": -3.394542694091797,
      "logps/chosen": -0.06766582280397415,
      "logps/rejected": -125.23938751220703,
      "loss": 0.108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6696641445159912,
      "rewards/margins": 1.6696641445159912,
      "rewards/rejected": 0.0,
      "step": 2027
    },
    {
      "epoch": 11.329608938547485,
      "grad_norm": 0.14349266156910412,
      "learning_rate": 1.1897929663132221e-06,
      "logits/chosen": -3.3487205505371094,
      "logits/rejected": -3.607750415802002,
      "logps/chosen": -18.503292083740234,
      "logps/rejected": -94.28089904785156,
      "loss": 0.0735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.142282485961914,
      "rewards/margins": 4.142282485961914,
      "rewards/rejected": 0.0,
      "step": 2028
    },
    {
      "epoch": 11.335195530726256,
      "grad_norm": 0.1500402263471141,
      "learning_rate": 1.1871982946892112e-06,
      "logits/chosen": -3.1887173652648926,
      "logits/rejected": -3.2744178771972656,
      "logps/chosen": -25.47255516052246,
      "logps/rejected": -70.01998901367188,
      "loss": 0.0723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.359230995178223,
      "rewards/margins": 4.359230995178223,
      "rewards/rejected": 0.0,
      "step": 2029
    },
    {
      "epoch": 11.340782122905027,
      "grad_norm": 0.9861694966392585,
      "learning_rate": 1.1846055741223331e-06,
      "logits/chosen": -3.359739303588867,
      "logits/rejected": -3.52128529548645,
      "logps/chosen": -0.4845837354660034,
      "logps/rejected": -56.89478302001953,
      "loss": 0.109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.876415729522705,
      "rewards/margins": 2.876415729522705,
      "rewards/rejected": 0.0,
      "step": 2030
    },
    {
      "epoch": 11.346368715083798,
      "grad_norm": 0.3845386869268309,
      "learning_rate": 1.1820148084658304e-06,
      "logits/chosen": -3.230508804321289,
      "logits/rejected": -3.2582647800445557,
      "logps/chosen": -2.5865018367767334,
      "logps/rejected": -44.207279205322266,
      "loss": 0.0971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8268954753875732,
      "rewards/margins": 2.8268954753875732,
      "rewards/rejected": 0.0,
      "step": 2031
    },
    {
      "epoch": 11.35195530726257,
      "grad_norm": 0.30118546298779014,
      "learning_rate": 1.1794260015700427e-06,
      "logits/chosen": -3.4633073806762695,
      "logits/rejected": -3.5913784503936768,
      "logps/chosen": -22.585433959960938,
      "logps/rejected": -73.2200927734375,
      "loss": 0.0545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 6.3427629470825195,
      "rewards/margins": 6.3427629470825195,
      "rewards/rejected": 0.0,
      "step": 2032
    },
    {
      "epoch": 11.35754189944134,
      "grad_norm": 0.7041383472962439,
      "learning_rate": 1.176839157282397e-06,
      "logits/chosen": -3.292330503463745,
      "logits/rejected": -3.3754420280456543,
      "logps/chosen": -0.6523759365081787,
      "logps/rejected": -50.204345703125,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4499597549438477,
      "rewards/margins": 2.4499597549438477,
      "rewards/rejected": 0.0,
      "step": 2033
    },
    {
      "epoch": 11.363128491620111,
      "grad_norm": 0.17645793752466102,
      "learning_rate": 1.1742542794474077e-06,
      "logits/chosen": -3.4403226375579834,
      "logits/rejected": -3.3124616146087646,
      "logps/chosen": -0.40641266107559204,
      "logps/rejected": -54.53502655029297,
      "loss": 0.075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.399075508117676,
      "rewards/margins": 2.399075508117676,
      "rewards/rejected": 0.0,
      "step": 2034
    },
    {
      "epoch": 11.368715083798882,
      "grad_norm": 0.11817853732849412,
      "learning_rate": 1.1716713719066606e-06,
      "logits/chosen": -3.6661393642425537,
      "logits/rejected": -3.4843428134918213,
      "logps/chosen": -0.15770237147808075,
      "logps/rejected": -62.941741943359375,
      "loss": 0.0883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5786004066467285,
      "rewards/margins": 2.5786004066467285,
      "rewards/rejected": 0.0,
      "step": 2035
    },
    {
      "epoch": 11.374301675977653,
      "grad_norm": 0.2826211298099155,
      "learning_rate": 1.169090438498816e-06,
      "logits/chosen": -3.4977152347564697,
      "logits/rejected": -3.616786479949951,
      "logps/chosen": -0.9014302492141724,
      "logps/rejected": -90.75349426269531,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.282579183578491,
      "rewards/margins": 3.282579183578491,
      "rewards/rejected": 0.0,
      "step": 2036
    },
    {
      "epoch": 11.379888268156424,
      "grad_norm": 0.36197424559813046,
      "learning_rate": 1.1665114830596038e-06,
      "logits/chosen": -3.6331145763397217,
      "logits/rejected": -3.449481725692749,
      "logps/chosen": -0.5883764624595642,
      "logps/rejected": -72.79119110107422,
      "loss": 0.0805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.193502902984619,
      "rewards/margins": 3.193502902984619,
      "rewards/rejected": 0.0,
      "step": 2037
    },
    {
      "epoch": 11.385474860335195,
      "grad_norm": 0.21782969520477852,
      "learning_rate": 1.1639345094218075e-06,
      "logits/chosen": -3.5120177268981934,
      "logits/rejected": -3.746638298034668,
      "logps/chosen": -0.07049018144607544,
      "logps/rejected": -70.62564849853516,
      "loss": 0.1122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.586543083190918,
      "rewards/margins": 1.586543083190918,
      "rewards/rejected": 0.0,
      "step": 2038
    },
    {
      "epoch": 11.391061452513966,
      "grad_norm": 0.3259715342240604,
      "learning_rate": 1.1613595214152713e-06,
      "logits/chosen": -3.574512004852295,
      "logits/rejected": -3.4965298175811768,
      "logps/chosen": -3.674063205718994,
      "logps/rejected": -73.70630645751953,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.203405380249023,
      "rewards/margins": 4.203405380249023,
      "rewards/rejected": 0.0,
      "step": 2039
    },
    {
      "epoch": 11.396648044692737,
      "grad_norm": 0.7058785682305136,
      "learning_rate": 1.158786522866885e-06,
      "logits/chosen": -3.4144365787506104,
      "logits/rejected": -3.036867380142212,
      "logps/chosen": -9.30002212524414,
      "logps/rejected": -57.00837326049805,
      "loss": 0.1074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.414975166320801,
      "rewards/margins": 3.414975166320801,
      "rewards/rejected": 0.0,
      "step": 2040
    },
    {
      "epoch": 11.396648044692737,
      "eval_logits/chosen": -3.3663361072540283,
      "eval_logits/rejected": -3.492006301879883,
      "eval_logps/chosen": -43.989891052246094,
      "eval_logps/rejected": -68.42874908447266,
      "eval_loss": 2.1274282932281494,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -1.3919849395751953,
      "eval_rewards/margins": -1.3919849395751953,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5509,
      "eval_samples_per_second": 12.133,
      "eval_steps_per_second": 0.391,
      "step": 2040
    },
    {
      "epoch": 11.402234636871508,
      "grad_norm": 0.28378673381278235,
      "learning_rate": 1.1562155176005827e-06,
      "logits/chosen": -3.5120849609375,
      "logits/rejected": -3.6463894844055176,
      "logps/chosen": -16.93041229248047,
      "logps/rejected": -72.14543151855469,
      "loss": 0.0917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.939737796783447,
      "rewards/margins": 4.939737796783447,
      "rewards/rejected": 0.0,
      "step": 2041
    },
    {
      "epoch": 11.40782122905028,
      "grad_norm": 0.35640522663297686,
      "learning_rate": 1.153646509437335e-06,
      "logits/chosen": -3.7153375148773193,
      "logits/rejected": -3.6829428672790527,
      "logps/chosen": -12.69172477722168,
      "logps/rejected": -72.83932495117188,
      "loss": 0.1305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3849239349365234,
      "rewards/margins": 3.3849239349365234,
      "rewards/rejected": 0.0,
      "step": 2042
    },
    {
      "epoch": 11.41340782122905,
      "grad_norm": 0.7852538039968189,
      "learning_rate": 1.151079502195149e-06,
      "logits/chosen": -3.671967029571533,
      "logits/rejected": -3.6483962535858154,
      "logps/chosen": -0.7687722444534302,
      "logps/rejected": -60.67894744873047,
      "loss": 0.098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.42402720451355,
      "rewards/margins": 3.42402720451355,
      "rewards/rejected": 0.0,
      "step": 2043
    },
    {
      "epoch": 11.418994413407821,
      "grad_norm": 1.3952085703162076,
      "learning_rate": 1.1485144996890512e-06,
      "logits/chosen": -3.6112420558929443,
      "logits/rejected": -3.527771472930908,
      "logps/chosen": -22.264448165893555,
      "logps/rejected": -88.17578125,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4124882221221924,
      "rewards/margins": 3.4124882221221924,
      "rewards/rejected": 0.0,
      "step": 2044
    },
    {
      "epoch": 11.424581005586592,
      "grad_norm": 0.6003100796941231,
      "learning_rate": 1.1459515057310924e-06,
      "logits/chosen": -3.4060487747192383,
      "logits/rejected": -3.3833277225494385,
      "logps/chosen": -1.869299054145813,
      "logps/rejected": -71.9408187866211,
      "loss": 0.0955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.536369800567627,
      "rewards/margins": 3.536369800567627,
      "rewards/rejected": 0.0,
      "step": 2045
    },
    {
      "epoch": 11.430167597765363,
      "grad_norm": 0.20965254113225715,
      "learning_rate": 1.1433905241303409e-06,
      "logits/chosen": -3.5424232482910156,
      "logits/rejected": -3.588852882385254,
      "logps/chosen": -2.846301555633545,
      "logps/rejected": -88.60520935058594,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.112240791320801,
      "rewards/margins": 4.112240791320801,
      "rewards/rejected": 0.0,
      "step": 2046
    },
    {
      "epoch": 11.435754189944134,
      "grad_norm": 0.1829098868978377,
      "learning_rate": 1.1408315586928675e-06,
      "logits/chosen": -3.2333359718322754,
      "logits/rejected": -3.2307612895965576,
      "logps/chosen": -0.31394442915916443,
      "logps/rejected": -69.28108215332031,
      "loss": 0.1027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.959980010986328,
      "rewards/margins": 2.959980010986328,
      "rewards/rejected": 0.0,
      "step": 2047
    },
    {
      "epoch": 11.441340782122905,
      "grad_norm": 1.3391615047967287,
      "learning_rate": 1.1382746132217546e-06,
      "logits/chosen": -3.594850540161133,
      "logits/rejected": -3.7689173221588135,
      "logps/chosen": -0.17619812488555908,
      "logps/rejected": -88.36469268798828,
      "loss": 0.1027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.593599796295166,
      "rewards/margins": 2.593599796295166,
      "rewards/rejected": 0.0,
      "step": 2048
    },
    {
      "epoch": 11.446927374301676,
      "grad_norm": 0.23440469197571406,
      "learning_rate": 1.1357196915170744e-06,
      "logits/chosen": -3.207235336303711,
      "logits/rejected": -3.3562963008880615,
      "logps/chosen": -0.9255645275115967,
      "logps/rejected": -84.56532287597656,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5304999351501465,
      "rewards/margins": 2.5304999351501465,
      "rewards/rejected": 0.0,
      "step": 2049
    },
    {
      "epoch": 11.452513966480447,
      "grad_norm": 0.14216154791281962,
      "learning_rate": 1.1331667973758986e-06,
      "logits/chosen": -3.619515895843506,
      "logits/rejected": -2.9845082759857178,
      "logps/chosen": -0.07501231133937836,
      "logps/rejected": -118.23396301269531,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8474912643432617,
      "rewards/margins": 1.8474912643432617,
      "rewards/rejected": 0.0,
      "step": 2050
    },
    {
      "epoch": 11.458100558659218,
      "grad_norm": 2.8172582481596082,
      "learning_rate": 1.1306159345922815e-06,
      "logits/chosen": -3.4893620014190674,
      "logits/rejected": -3.6972763538360596,
      "logps/chosen": -0.3193190097808838,
      "logps/rejected": -76.88438415527344,
      "loss": 0.1521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0219225883483887,
      "rewards/margins": 2.0219225883483887,
      "rewards/rejected": 0.0,
      "step": 2051
    },
    {
      "epoch": 11.46368715083799,
      "grad_norm": 0.18042766020992987,
      "learning_rate": 1.1280671069572596e-06,
      "logits/chosen": -3.5585415363311768,
      "logits/rejected": -3.669483184814453,
      "logps/chosen": -2.361182451248169,
      "logps/rejected": -56.60914611816406,
      "loss": 0.1121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4479427337646484,
      "rewards/margins": 3.4479427337646484,
      "rewards/rejected": 0.0,
      "step": 2052
    },
    {
      "epoch": 11.46927374301676,
      "grad_norm": 1.1729903374140136,
      "learning_rate": 1.125520318258844e-06,
      "logits/chosen": -3.716301441192627,
      "logits/rejected": -3.6839730739593506,
      "logps/chosen": -5.205162048339844,
      "logps/rejected": -43.59601593017578,
      "loss": 0.0928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4449119567871094,
      "rewards/margins": 3.4449119567871094,
      "rewards/rejected": 0.0,
      "step": 2053
    },
    {
      "epoch": 11.474860335195531,
      "grad_norm": 0.29695706683987294,
      "learning_rate": 1.1229755722820181e-06,
      "logits/chosen": -3.240823268890381,
      "logits/rejected": -3.409513235092163,
      "logps/chosen": -0.21283794939517975,
      "logps/rejected": -76.42710876464844,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5360255241394043,
      "rewards/margins": 2.5360255241394043,
      "rewards/rejected": 0.0,
      "step": 2054
    },
    {
      "epoch": 11.480446927374302,
      "grad_norm": 0.4338495141280093,
      "learning_rate": 1.120432872808726e-06,
      "logits/chosen": -3.2456319332122803,
      "logits/rejected": -3.243220090866089,
      "logps/chosen": -14.994654655456543,
      "logps/rejected": -110.94085693359375,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6381173133850098,
      "rewards/margins": 3.6381173133850098,
      "rewards/rejected": 0.0,
      "step": 2055
    },
    {
      "epoch": 11.486033519553073,
      "grad_norm": 0.4361978585705121,
      "learning_rate": 1.1178922236178715e-06,
      "logits/chosen": -3.515784740447998,
      "logits/rejected": -3.3833765983581543,
      "logps/chosen": -0.2808058559894562,
      "logps/rejected": -102.72672271728516,
      "loss": 0.1349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3861265182495117,
      "rewards/margins": 2.3861265182495117,
      "rewards/rejected": 0.0,
      "step": 2056
    },
    {
      "epoch": 11.491620111731844,
      "grad_norm": 0.270704798049981,
      "learning_rate": 1.1153536284853144e-06,
      "logits/chosen": -3.3684651851654053,
      "logits/rejected": -3.281177043914795,
      "logps/chosen": -0.4776436984539032,
      "logps/rejected": -55.18301773071289,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3707613945007324,
      "rewards/margins": 2.3707613945007324,
      "rewards/rejected": 0.0,
      "step": 2057
    },
    {
      "epoch": 11.497206703910614,
      "grad_norm": 0.5982623661317277,
      "learning_rate": 1.1128170911838553e-06,
      "logits/chosen": -3.6087872982025146,
      "logits/rejected": -3.431091785430908,
      "logps/chosen": -0.15964321792125702,
      "logps/rejected": -72.39300537109375,
      "loss": 0.091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.50662899017334,
      "rewards/margins": 2.50662899017334,
      "rewards/rejected": 0.0,
      "step": 2058
    },
    {
      "epoch": 11.502793296089386,
      "grad_norm": 0.1901204245519802,
      "learning_rate": 1.1102826154832439e-06,
      "logits/chosen": -3.4885616302490234,
      "logits/rejected": -3.6892285346984863,
      "logps/chosen": -0.5527942776679993,
      "logps/rejected": -115.32758331298828,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2907745838165283,
      "rewards/margins": 2.2907745838165283,
      "rewards/rejected": 0.0,
      "step": 2059
    },
    {
      "epoch": 11.508379888268156,
      "grad_norm": 0.8106204577386887,
      "learning_rate": 1.1077502051501604e-06,
      "logits/chosen": -3.6276888847351074,
      "logits/rejected": -3.5567421913146973,
      "logps/chosen": -0.528810977935791,
      "logps/rejected": -64.58943176269531,
      "loss": 0.1362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9575989246368408,
      "rewards/margins": 1.9575989246368408,
      "rewards/rejected": 0.0,
      "step": 2060
    },
    {
      "epoch": 11.508379888268156,
      "eval_logits/chosen": -3.3600082397460938,
      "eval_logits/rejected": -3.4860317707061768,
      "eval_logps/chosen": -43.56682586669922,
      "eval_logps/rejected": -67.80416107177734,
      "eval_loss": 2.0924324989318848,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.34967839717865,
      "eval_rewards/margins": -1.34967839717865,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5481,
      "eval_samples_per_second": 12.134,
      "eval_steps_per_second": 0.391,
      "step": 2060
    },
    {
      "epoch": 11.513966480446927,
      "grad_norm": 0.49113321959587003,
      "learning_rate": 1.1052198639482186e-06,
      "logits/chosen": -3.4717705249786377,
      "logits/rejected": -3.510472536087036,
      "logps/chosen": -0.10664243996143341,
      "logps/rejected": -41.75074005126953,
      "loss": 0.0914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1781668663024902,
      "rewards/margins": 2.1781668663024902,
      "rewards/rejected": 0.0,
      "step": 2061
    },
    {
      "epoch": 11.519553072625698,
      "grad_norm": 0.5152080736737282,
      "learning_rate": 1.1026915956379547e-06,
      "logits/chosen": -3.4027798175811768,
      "logits/rejected": -3.55617356300354,
      "logps/chosen": -5.086245059967041,
      "logps/rejected": -66.29646301269531,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.285412311553955,
      "rewards/margins": 3.285412311553955,
      "rewards/rejected": 0.0,
      "step": 2062
    },
    {
      "epoch": 11.525139664804469,
      "grad_norm": 0.23043445202229368,
      "learning_rate": 1.1001654039768287e-06,
      "logits/chosen": -3.543848752975464,
      "logits/rejected": -3.441652774810791,
      "logps/chosen": -0.3873385190963745,
      "logps/rejected": -88.67133331298828,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8881816864013672,
      "rewards/margins": 1.8881816864013672,
      "rewards/rejected": 0.0,
      "step": 2063
    },
    {
      "epoch": 11.53072625698324,
      "grad_norm": 0.2880363427308498,
      "learning_rate": 1.0976412927192072e-06,
      "logits/chosen": -3.325949192047119,
      "logits/rejected": -3.284792900085449,
      "logps/chosen": -1.4022696018218994,
      "logps/rejected": -71.85545349121094,
      "loss": 0.1005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3758702278137207,
      "rewards/margins": 3.3758702278137207,
      "rewards/rejected": 0.0,
      "step": 2064
    },
    {
      "epoch": 11.53631284916201,
      "grad_norm": 0.23139250406076128,
      "learning_rate": 1.095119265616372e-06,
      "logits/chosen": -3.546360969543457,
      "logits/rejected": -3.4458816051483154,
      "logps/chosen": -1.52488112449646,
      "logps/rejected": -60.87740707397461,
      "loss": 0.0685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.912606716156006,
      "rewards/margins": 3.912606716156006,
      "rewards/rejected": 0.0,
      "step": 2065
    },
    {
      "epoch": 11.541899441340782,
      "grad_norm": 0.37774267119492483,
      "learning_rate": 1.0925993264165048e-06,
      "logits/chosen": -3.3148419857025146,
      "logits/rejected": -3.3550186157226562,
      "logps/chosen": -2.7571189403533936,
      "logps/rejected": -92.48747253417969,
      "loss": 0.086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.348743438720703,
      "rewards/margins": 2.348743438720703,
      "rewards/rejected": 0.0,
      "step": 2066
    },
    {
      "epoch": 11.547486033519553,
      "grad_norm": 0.11678942241312237,
      "learning_rate": 1.090081478864681e-06,
      "logits/chosen": -3.607976198196411,
      "logits/rejected": -3.655184507369995,
      "logps/chosen": -0.5582696199417114,
      "logps/rejected": -77.35395812988281,
      "loss": 0.0861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.066649913787842,
      "rewards/margins": 3.066649913787842,
      "rewards/rejected": 0.0,
      "step": 2067
    },
    {
      "epoch": 11.553072625698324,
      "grad_norm": 0.1350129826549824,
      "learning_rate": 1.087565726702873e-06,
      "logits/chosen": -3.5390470027923584,
      "logits/rejected": -3.5165624618530273,
      "logps/chosen": -0.3333151936531067,
      "logps/rejected": -63.4757080078125,
      "loss": 0.0918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.816955089569092,
      "rewards/margins": 2.816955089569092,
      "rewards/rejected": 0.0,
      "step": 2068
    },
    {
      "epoch": 11.558659217877095,
      "grad_norm": 2.5289156744880112,
      "learning_rate": 1.0850520736699362e-06,
      "logits/chosen": -3.671410083770752,
      "logits/rejected": -3.731820583343506,
      "logps/chosen": -0.5359936952590942,
      "logps/rejected": -81.48534393310547,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.461421489715576,
      "rewards/margins": 2.461421489715576,
      "rewards/rejected": 0.0,
      "step": 2069
    },
    {
      "epoch": 11.564245810055866,
      "grad_norm": 0.1649022280959967,
      "learning_rate": 1.0825405235016065e-06,
      "logits/chosen": -3.647160053253174,
      "logits/rejected": -3.5849385261535645,
      "logps/chosen": -0.2609698474407196,
      "logps/rejected": -76.89021301269531,
      "loss": 0.0739,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.390981435775757,
      "rewards/margins": 2.390981435775757,
      "rewards/rejected": 0.0,
      "step": 2070
    },
    {
      "epoch": 11.569832402234637,
      "grad_norm": 1.4719470246717177,
      "learning_rate": 1.0800310799304932e-06,
      "logits/chosen": -3.421604633331299,
      "logits/rejected": -3.473567247390747,
      "logps/chosen": -0.11275311559438705,
      "logps/rejected": -99.02001190185547,
      "loss": 0.1125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5924322605133057,
      "rewards/margins": 1.5924322605133057,
      "rewards/rejected": 0.0,
      "step": 2071
    },
    {
      "epoch": 11.575418994413408,
      "grad_norm": 0.47858295914817045,
      "learning_rate": 1.0775237466860802e-06,
      "logits/chosen": -3.295795202255249,
      "logits/rejected": -3.272120714187622,
      "logps/chosen": -1.5118334293365479,
      "logps/rejected": -81.84585571289062,
      "loss": 0.092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.625046730041504,
      "rewards/margins": 2.625046730041504,
      "rewards/rejected": 0.0,
      "step": 2072
    },
    {
      "epoch": 11.581005586592179,
      "grad_norm": 2.341464928107241,
      "learning_rate": 1.0750185274947067e-06,
      "logits/chosen": -3.3456509113311768,
      "logits/rejected": -3.2320127487182617,
      "logps/chosen": -0.24334463477134705,
      "logps/rejected": -69.25424194335938,
      "loss": 0.0998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.567368268966675,
      "rewards/margins": 2.567368268966675,
      "rewards/rejected": 0.0,
      "step": 2073
    },
    {
      "epoch": 11.58659217877095,
      "grad_norm": 0.22009391262598826,
      "learning_rate": 1.0725154260795776e-06,
      "logits/chosen": -3.9172523021698,
      "logits/rejected": -3.892346143722534,
      "logps/chosen": -0.035237446427345276,
      "logps/rejected": -50.78630447387695,
      "loss": 0.0825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4381601810455322,
      "rewards/margins": 1.4381601810455322,
      "rewards/rejected": 0.0,
      "step": 2074
    },
    {
      "epoch": 11.59217877094972,
      "grad_norm": 0.23810556497350877,
      "learning_rate": 1.070014446160747e-06,
      "logits/chosen": -3.474716901779175,
      "logits/rejected": -3.4445223808288574,
      "logps/chosen": -0.254489004611969,
      "logps/rejected": -76.31936645507812,
      "loss": 0.1012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6910252571105957,
      "rewards/margins": 2.6910252571105957,
      "rewards/rejected": 0.0,
      "step": 2075
    },
    {
      "epoch": 11.597765363128492,
      "grad_norm": 0.21327742897805063,
      "learning_rate": 1.0675155914551145e-06,
      "logits/chosen": -3.5339014530181885,
      "logits/rejected": -3.5197579860687256,
      "logps/chosen": -0.18966253101825714,
      "logps/rejected": -66.66136932373047,
      "loss": 0.089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6947064399719238,
      "rewards/margins": 1.6947064399719238,
      "rewards/rejected": 0.0,
      "step": 2076
    },
    {
      "epoch": 11.603351955307263,
      "grad_norm": 0.12043932500347757,
      "learning_rate": 1.0650188656764262e-06,
      "logits/chosen": -3.462836980819702,
      "logits/rejected": -3.3858237266540527,
      "logps/chosen": -3.3658440113067627,
      "logps/rejected": -74.22305297851562,
      "loss": 0.09,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1835052967071533,
      "rewards/margins": 3.1835052967071533,
      "rewards/rejected": 0.0,
      "step": 2077
    },
    {
      "epoch": 11.608938547486034,
      "grad_norm": 1.0271914875709658,
      "learning_rate": 1.062524272535258e-06,
      "logits/chosen": -3.6363320350646973,
      "logits/rejected": -3.8121120929718018,
      "logps/chosen": -1.6896469593048096,
      "logps/rejected": -119.82862854003906,
      "loss": 0.0786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.44681978225708,
      "rewards/margins": 3.44681978225708,
      "rewards/rejected": 0.0,
      "step": 2078
    },
    {
      "epoch": 11.614525139664805,
      "grad_norm": 0.21420706922367522,
      "learning_rate": 1.0600318157390222e-06,
      "logits/chosen": -3.608989953994751,
      "logits/rejected": -3.731722116470337,
      "logps/chosen": -4.1234002113342285,
      "logps/rejected": -72.41159057617188,
      "loss": 0.1019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.752960205078125,
      "rewards/margins": 3.752960205078125,
      "rewards/rejected": 0.0,
      "step": 2079
    },
    {
      "epoch": 11.620111731843576,
      "grad_norm": 0.3711578830589583,
      "learning_rate": 1.0575414989919523e-06,
      "logits/chosen": -3.6840550899505615,
      "logits/rejected": -3.334397077560425,
      "logps/chosen": -1.829545497894287,
      "logps/rejected": -56.995582580566406,
      "loss": 0.0648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.641461133956909,
      "rewards/margins": 3.641461133956909,
      "rewards/rejected": 0.0,
      "step": 2080
    },
    {
      "epoch": 11.620111731843576,
      "eval_logits/chosen": -3.3596081733703613,
      "eval_logits/rejected": -3.4855377674102783,
      "eval_logps/chosen": -43.79681396484375,
      "eval_logps/rejected": -68.90438079833984,
      "eval_loss": 2.128615617752075,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.3726770877838135,
      "eval_rewards/margins": -1.3726770877838135,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6382,
      "eval_samples_per_second": 12.091,
      "eval_steps_per_second": 0.39,
      "step": 2080
    },
    {
      "epoch": 11.625698324022347,
      "grad_norm": 0.15797490340395567,
      "learning_rate": 1.0550533259951026e-06,
      "logits/chosen": -3.6676278114318848,
      "logits/rejected": -3.4475128650665283,
      "logps/chosen": -0.14914539456367493,
      "logps/rejected": -133.8074493408203,
      "loss": 0.1013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.231872081756592,
      "rewards/margins": 2.231872081756592,
      "rewards/rejected": 0.0,
      "step": 2081
    },
    {
      "epoch": 11.631284916201118,
      "grad_norm": 2.7562601756190572,
      "learning_rate": 1.0525673004463407e-06,
      "logits/chosen": -3.558908224105835,
      "logits/rejected": -3.6239116191864014,
      "logps/chosen": -3.191877841949463,
      "logps/rejected": -41.19059371948242,
      "loss": 0.1214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.652935028076172,
      "rewards/margins": 3.652935028076172,
      "rewards/rejected": 0.0,
      "step": 2082
    },
    {
      "epoch": 11.636871508379889,
      "grad_norm": 0.3188997746286465,
      "learning_rate": 1.0500834260403458e-06,
      "logits/chosen": -3.4660637378692627,
      "logits/rejected": -3.7934863567352295,
      "logps/chosen": -6.72496223449707,
      "logps/rejected": -80.35829162597656,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.657498359680176,
      "rewards/margins": 3.657498359680176,
      "rewards/rejected": 0.0,
      "step": 2083
    },
    {
      "epoch": 11.64245810055866,
      "grad_norm": 0.20463988647244924,
      "learning_rate": 1.0476017064685943e-06,
      "logits/chosen": -3.687196731567383,
      "logits/rejected": -3.2436025142669678,
      "logps/chosen": -7.795816898345947,
      "logps/rejected": -100.4124984741211,
      "loss": 0.071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9016826152801514,
      "rewards/margins": 3.9016826152801514,
      "rewards/rejected": 0.0,
      "step": 2084
    },
    {
      "epoch": 11.64804469273743,
      "grad_norm": 0.2845596252363511,
      "learning_rate": 1.0451221454193655e-06,
      "logits/chosen": -3.2707607746124268,
      "logits/rejected": -3.7212791442871094,
      "logps/chosen": -0.47181397676467896,
      "logps/rejected": -80.09098815917969,
      "loss": 0.0858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1099698543548584,
      "rewards/margins": 2.1099698543548584,
      "rewards/rejected": 0.0,
      "step": 2085
    },
    {
      "epoch": 11.653631284916202,
      "grad_norm": 0.5352722399005803,
      "learning_rate": 1.0426447465777293e-06,
      "logits/chosen": -3.6874172687530518,
      "logits/rejected": -3.6744444370269775,
      "logps/chosen": -2.849461078643799,
      "logps/rejected": -97.09371185302734,
      "loss": 0.1393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.724611520767212,
      "rewards/margins": 3.724611520767212,
      "rewards/rejected": 0.0,
      "step": 2086
    },
    {
      "epoch": 11.659217877094973,
      "grad_norm": 1.0125903129190787,
      "learning_rate": 1.0401695136255383e-06,
      "logits/chosen": -3.5947346687316895,
      "logits/rejected": -3.052906036376953,
      "logps/chosen": -5.007686614990234,
      "logps/rejected": -78.40203857421875,
      "loss": 0.1085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7514803409576416,
      "rewards/margins": 3.7514803409576416,
      "rewards/rejected": 0.0,
      "step": 2087
    },
    {
      "epoch": 11.664804469273744,
      "grad_norm": 0.3520332772650081,
      "learning_rate": 1.0376964502414326e-06,
      "logits/chosen": -3.36950945854187,
      "logits/rejected": -3.384535551071167,
      "logps/chosen": -0.44833192229270935,
      "logps/rejected": -72.59114837646484,
      "loss": 0.0905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5193448066711426,
      "rewards/margins": 2.5193448066711426,
      "rewards/rejected": 0.0,
      "step": 2088
    },
    {
      "epoch": 11.670391061452515,
      "grad_norm": 0.27508073133223343,
      "learning_rate": 1.0352255601008243e-06,
      "logits/chosen": -3.638601303100586,
      "logits/rejected": -3.3416879177093506,
      "logps/chosen": -0.15703147649765015,
      "logps/rejected": -73.22254943847656,
      "loss": 0.1111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6554811000823975,
      "rewards/margins": 1.6554811000823975,
      "rewards/rejected": 0.0,
      "step": 2089
    },
    {
      "epoch": 11.675977653631286,
      "grad_norm": 0.7582337050023996,
      "learning_rate": 1.0327568468758952e-06,
      "logits/chosen": -3.610506057739258,
      "logits/rejected": -3.564732551574707,
      "logps/chosen": -0.10979451984167099,
      "logps/rejected": -58.58778762817383,
      "loss": 0.0908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5627222061157227,
      "rewards/margins": 2.5627222061157227,
      "rewards/rejected": 0.0,
      "step": 2090
    },
    {
      "epoch": 11.681564245810057,
      "grad_norm": 0.17666767455729754,
      "learning_rate": 1.030290314235593e-06,
      "logits/chosen": -3.512104034423828,
      "logits/rejected": -3.438829183578491,
      "logps/chosen": -0.2256065011024475,
      "logps/rejected": -78.04668426513672,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3830485343933105,
      "rewards/margins": 2.3830485343933105,
      "rewards/rejected": 0.0,
      "step": 2091
    },
    {
      "epoch": 11.687150837988828,
      "grad_norm": 0.5632773177313098,
      "learning_rate": 1.0278259658456262e-06,
      "logits/chosen": -3.6253538131713867,
      "logits/rejected": -3.309044122695923,
      "logps/chosen": -0.6020023226737976,
      "logps/rejected": -75.8205337524414,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2949023246765137,
      "rewards/margins": 3.2949023246765137,
      "rewards/rejected": 0.0,
      "step": 2092
    },
    {
      "epoch": 11.692737430167599,
      "grad_norm": 0.20265462601533027,
      "learning_rate": 1.0253638053684526e-06,
      "logits/chosen": -3.2287769317626953,
      "logits/rejected": -3.412078857421875,
      "logps/chosen": -0.10008642822504044,
      "logps/rejected": -89.72229766845703,
      "loss": 0.1001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9739428758621216,
      "rewards/margins": 1.9739428758621216,
      "rewards/rejected": 0.0,
      "step": 2093
    },
    {
      "epoch": 11.69832402234637,
      "grad_norm": 0.9077192205009595,
      "learning_rate": 1.0229038364632831e-06,
      "logits/chosen": -3.713380813598633,
      "logits/rejected": -3.5552313327789307,
      "logps/chosen": -0.6872936487197876,
      "logps/rejected": -65.89900970458984,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1584928035736084,
      "rewards/margins": 2.1584928035736084,
      "rewards/rejected": 0.0,
      "step": 2094
    },
    {
      "epoch": 11.703910614525139,
      "grad_norm": 0.538032805708107,
      "learning_rate": 1.020446062786069e-06,
      "logits/chosen": -3.478909492492676,
      "logits/rejected": -3.2160627841949463,
      "logps/chosen": -0.24049176275730133,
      "logps/rejected": -66.54315185546875,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0722150802612305,
      "rewards/margins": 2.0722150802612305,
      "rewards/rejected": 0.0,
      "step": 2095
    },
    {
      "epoch": 11.70949720670391,
      "grad_norm": 0.31064470017580953,
      "learning_rate": 1.0179904879895e-06,
      "logits/chosen": -3.3462183475494385,
      "logits/rejected": -3.4453110694885254,
      "logps/chosen": -0.2259645164012909,
      "logps/rejected": -62.6806640625,
      "loss": 0.1002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4421703815460205,
      "rewards/margins": 2.4421703815460205,
      "rewards/rejected": 0.0,
      "step": 2096
    },
    {
      "epoch": 11.71508379888268,
      "grad_norm": 1.032527583042397,
      "learning_rate": 1.0155371157229974e-06,
      "logits/chosen": -3.248305320739746,
      "logits/rejected": -3.3000659942626953,
      "logps/chosen": -9.189111709594727,
      "logps/rejected": -79.42362213134766,
      "loss": 0.0728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0039610862731934,
      "rewards/margins": 3.0039610862731934,
      "rewards/rejected": 0.0,
      "step": 2097
    },
    {
      "epoch": 11.720670391061452,
      "grad_norm": 1.4476988799345978,
      "learning_rate": 1.0130859496327089e-06,
      "logits/chosen": -3.6477084159851074,
      "logits/rejected": -3.791811227798462,
      "logps/chosen": -1.1395684480667114,
      "logps/rejected": -93.91838073730469,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8196749687194824,
      "rewards/margins": 2.8196749687194824,
      "rewards/rejected": 0.0,
      "step": 2098
    },
    {
      "epoch": 11.726256983240223,
      "grad_norm": 0.325824766429487,
      "learning_rate": 1.0106369933615043e-06,
      "logits/chosen": -3.5877373218536377,
      "logits/rejected": -3.622466564178467,
      "logps/chosen": -0.7786736488342285,
      "logps/rejected": -56.567596435546875,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5448122024536133,
      "rewards/margins": 3.5448122024536133,
      "rewards/rejected": 0.0,
      "step": 2099
    },
    {
      "epoch": 11.731843575418994,
      "grad_norm": 0.35376839465762555,
      "learning_rate": 1.0081902505489677e-06,
      "logits/chosen": -3.444740056991577,
      "logits/rejected": -3.372232437133789,
      "logps/chosen": -2.6223764419555664,
      "logps/rejected": -119.43436431884766,
      "loss": 0.119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.222923755645752,
      "rewards/margins": 3.222923755645752,
      "rewards/rejected": 0.0,
      "step": 2100
    },
    {
      "epoch": 11.731843575418994,
      "eval_logits/chosen": -3.3538169860839844,
      "eval_logits/rejected": -3.479052782058716,
      "eval_logps/chosen": -43.69095230102539,
      "eval_logps/rejected": -69.35709381103516,
      "eval_loss": 2.112902879714966,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.3620909452438354,
      "eval_rewards/margins": -1.3620909452438354,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5342,
      "eval_samples_per_second": 12.141,
      "eval_steps_per_second": 0.392,
      "step": 2100
    },
    {
      "epoch": 11.737430167597765,
      "grad_norm": 0.11776529844305574,
      "learning_rate": 1.0057457248313974e-06,
      "logits/chosen": -3.5674898624420166,
      "logits/rejected": -3.3266310691833496,
      "logps/chosen": -0.1973455548286438,
      "logps/rejected": -93.28620147705078,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1996355056762695,
      "rewards/margins": 2.1996355056762695,
      "rewards/rejected": 0.0,
      "step": 2101
    },
    {
      "epoch": 11.743016759776536,
      "grad_norm": 1.4620352865621444,
      "learning_rate": 1.0033034198417902e-06,
      "logits/chosen": -3.430107831954956,
      "logits/rejected": -3.650120496749878,
      "logps/chosen": -0.07732068747282028,
      "logps/rejected": -111.69308471679688,
      "loss": 0.1219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7498666048049927,
      "rewards/margins": 1.7498666048049927,
      "rewards/rejected": 0.0,
      "step": 2102
    },
    {
      "epoch": 11.748603351955307,
      "grad_norm": 0.44937893935177725,
      "learning_rate": 1.0008633392098491e-06,
      "logits/chosen": -3.4860727787017822,
      "logits/rejected": -3.3946802616119385,
      "logps/chosen": -0.18098577857017517,
      "logps/rejected": -70.42391967773438,
      "loss": 0.0918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3767454624176025,
      "rewards/margins": 2.3767454624176025,
      "rewards/rejected": 0.0,
      "step": 2103
    },
    {
      "epoch": 11.754189944134078,
      "grad_norm": 1.3386284913953124,
      "learning_rate": 9.984254865619678e-07,
      "logits/chosen": -3.401736259460449,
      "logits/rejected": -3.3779296875,
      "logps/chosen": -0.2391912192106247,
      "logps/rejected": -74.93659973144531,
      "loss": 0.102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.477368116378784,
      "rewards/margins": 2.477368116378784,
      "rewards/rejected": 0.0,
      "step": 2104
    },
    {
      "epoch": 11.759776536312849,
      "grad_norm": 0.7064831217719927,
      "learning_rate": 9.959898655212292e-07,
      "logits/chosen": -3.2398624420166016,
      "logits/rejected": -3.594224691390991,
      "logps/chosen": -2.8214237689971924,
      "logps/rejected": -187.3613739013672,
      "loss": 0.107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8530759811401367,
      "rewards/margins": 2.8530759811401367,
      "rewards/rejected": 0.0,
      "step": 2105
    },
    {
      "epoch": 11.76536312849162,
      "grad_norm": 0.2649464117024522,
      "learning_rate": 9.935564797074007e-07,
      "logits/chosen": -3.4824416637420654,
      "logits/rejected": -3.4747154712677,
      "logps/chosen": -0.9322139620780945,
      "logps/rejected": -56.4398193359375,
      "loss": 0.1487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4856386184692383,
      "rewards/margins": 2.4856386184692383,
      "rewards/rejected": 0.0,
      "step": 2106
    },
    {
      "epoch": 11.77094972067039,
      "grad_norm": 0.18981541594811632,
      "learning_rate": 9.911253327369245e-07,
      "logits/chosen": -3.2381865978240967,
      "logits/rejected": -3.350369691848755,
      "logps/chosen": -0.18925127387046814,
      "logps/rejected": -65.52972412109375,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.603680372238159,
      "rewards/margins": 2.603680372238159,
      "rewards/rejected": 0.0,
      "step": 2107
    },
    {
      "epoch": 11.776536312849162,
      "grad_norm": 0.6996224058537489,
      "learning_rate": 9.886964282229206e-07,
      "logits/chosen": -3.2665212154388428,
      "logits/rejected": -3.2883708477020264,
      "logps/chosen": -3.059497356414795,
      "logps/rejected": -59.164390563964844,
      "loss": 0.0884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9657537937164307,
      "rewards/margins": 3.9657537937164307,
      "rewards/rejected": 0.0,
      "step": 2108
    },
    {
      "epoch": 11.782122905027933,
      "grad_norm": 2.681406086290604,
      "learning_rate": 9.862697697751731e-07,
      "logits/chosen": -3.472437620162964,
      "logits/rejected": -3.4647719860076904,
      "logps/chosen": -1.7902541160583496,
      "logps/rejected": -50.870479583740234,
      "loss": 0.1211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.138012409210205,
      "rewards/margins": 3.138012409210205,
      "rewards/rejected": 0.0,
      "step": 2109
    },
    {
      "epoch": 11.787709497206704,
      "grad_norm": 0.23747272779717468,
      "learning_rate": 9.83845361000128e-07,
      "logits/chosen": -3.106231451034546,
      "logits/rejected": -3.0281145572662354,
      "logps/chosen": -0.661620020866394,
      "logps/rejected": -65.30532836914062,
      "loss": 0.1063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.92305850982666,
      "rewards/margins": 2.92305850982666,
      "rewards/rejected": 0.0,
      "step": 2110
    },
    {
      "epoch": 11.793296089385475,
      "grad_norm": 0.31801127875572727,
      "learning_rate": 9.814232055008879e-07,
      "logits/chosen": -3.285126209259033,
      "logits/rejected": -3.4112703800201416,
      "logps/chosen": -2.7618467807769775,
      "logps/rejected": -79.00245666503906,
      "loss": 0.1081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8287994861602783,
      "rewards/margins": 2.8287994861602783,
      "rewards/rejected": 0.0,
      "step": 2111
    },
    {
      "epoch": 11.798882681564246,
      "grad_norm": 0.3260354945264033,
      "learning_rate": 9.790033068772104e-07,
      "logits/chosen": -3.392638921737671,
      "logits/rejected": -3.396554708480835,
      "logps/chosen": -0.20374906063079834,
      "logps/rejected": -58.962440490722656,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8707345724105835,
      "rewards/margins": 1.8707345724105835,
      "rewards/rejected": 0.0,
      "step": 2112
    },
    {
      "epoch": 11.804469273743017,
      "grad_norm": 0.3148987866061907,
      "learning_rate": 9.765856687254913e-07,
      "logits/chosen": -3.623311758041382,
      "logits/rejected": -3.565458059310913,
      "logps/chosen": -1.3426401615142822,
      "logps/rejected": -53.824703216552734,
      "loss": 0.0654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8870527744293213,
      "rewards/margins": 2.8870527744293213,
      "rewards/rejected": 0.0,
      "step": 2113
    },
    {
      "epoch": 11.810055865921788,
      "grad_norm": 1.2114223893811196,
      "learning_rate": 9.741702946387748e-07,
      "logits/chosen": -3.570756196975708,
      "logits/rejected": -3.422825336456299,
      "logps/chosen": -0.5120782852172852,
      "logps/rejected": -63.0224494934082,
      "loss": 0.0904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2059507369995117,
      "rewards/margins": 3.2059507369995117,
      "rewards/rejected": 0.0,
      "step": 2114
    },
    {
      "epoch": 11.815642458100559,
      "grad_norm": 0.7052828731807312,
      "learning_rate": 9.717571882067358e-07,
      "logits/chosen": -3.751319646835327,
      "logits/rejected": -3.6667308807373047,
      "logps/chosen": -1.1978087425231934,
      "logps/rejected": -57.41690444946289,
      "loss": 0.1172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.708017349243164,
      "rewards/margins": 2.708017349243164,
      "rewards/rejected": 0.0,
      "step": 2115
    },
    {
      "epoch": 11.82122905027933,
      "grad_norm": 0.7222314530314252,
      "learning_rate": 9.693463530156806e-07,
      "logits/chosen": -3.1767401695251465,
      "logits/rejected": -3.178191661834717,
      "logps/chosen": -4.604262351989746,
      "logps/rejected": -48.9003791809082,
      "loss": 0.1403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.803452730178833,
      "rewards/margins": 3.803452730178833,
      "rewards/rejected": 0.0,
      "step": 2116
    },
    {
      "epoch": 11.8268156424581,
      "grad_norm": 2.915937757952767,
      "learning_rate": 9.669377926485391e-07,
      "logits/chosen": -3.391408681869507,
      "logits/rejected": -3.5301711559295654,
      "logps/chosen": -0.30739492177963257,
      "logps/rejected": -64.79705810546875,
      "loss": 0.1266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4749534130096436,
      "rewards/margins": 2.4749534130096436,
      "rewards/rejected": 0.0,
      "step": 2117
    },
    {
      "epoch": 11.832402234636872,
      "grad_norm": 0.13883943603649204,
      "learning_rate": 9.64531510684861e-07,
      "logits/chosen": -3.5633866786956787,
      "logits/rejected": -3.408674955368042,
      "logps/chosen": -1.2700015306472778,
      "logps/rejected": -55.135616302490234,
      "loss": 0.1034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.847590923309326,
      "rewards/margins": 3.847590923309326,
      "rewards/rejected": 0.0,
      "step": 2118
    },
    {
      "epoch": 11.837988826815643,
      "grad_norm": 0.6330941716166542,
      "learning_rate": 9.621275107008102e-07,
      "logits/chosen": -3.450352191925049,
      "logits/rejected": -3.33793044090271,
      "logps/chosen": -2.1506645679473877,
      "logps/rejected": -35.86470413208008,
      "loss": 0.1165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.902376174926758,
      "rewards/margins": 2.902376174926758,
      "rewards/rejected": 0.0,
      "step": 2119
    },
    {
      "epoch": 11.843575418994414,
      "grad_norm": 0.6749087972617245,
      "learning_rate": 9.597257962691575e-07,
      "logits/chosen": -3.233321189880371,
      "logits/rejected": -3.3388006687164307,
      "logps/chosen": -6.9530029296875,
      "logps/rejected": -65.3197021484375,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0718464851379395,
      "rewards/margins": 3.0718464851379395,
      "rewards/rejected": 0.0,
      "step": 2120
    },
    {
      "epoch": 11.843575418994414,
      "eval_logits/chosen": -3.354063034057617,
      "eval_logits/rejected": -3.4799752235412598,
      "eval_logps/chosen": -42.99395751953125,
      "eval_logps/rejected": -68.5378189086914,
      "eval_loss": 2.0683581829071045,
      "eval_rewards/accuracies": 0.22499999403953552,
      "eval_rewards/chosen": -1.2923917770385742,
      "eval_rewards/margins": -1.2923917770385742,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.4927,
      "eval_samples_per_second": 12.16,
      "eval_steps_per_second": 0.392,
      "step": 2120
    },
    {
      "epoch": 11.849162011173185,
      "grad_norm": 0.7936634577956813,
      "learning_rate": 9.573263709592814e-07,
      "logits/chosen": -3.5605971813201904,
      "logits/rejected": -3.5355732440948486,
      "logps/chosen": -0.49521562457084656,
      "logps/rejected": -78.16862487792969,
      "loss": 0.1036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9077646732330322,
      "rewards/margins": 2.9077646732330322,
      "rewards/rejected": 0.0,
      "step": 2121
    },
    {
      "epoch": 11.854748603351956,
      "grad_norm": 0.14546084488353506,
      "learning_rate": 9.549292383371514e-07,
      "logits/chosen": -3.501621723175049,
      "logits/rejected": -3.4943320751190186,
      "logps/chosen": -0.3626451790332794,
      "logps/rejected": -76.94122314453125,
      "loss": 0.1346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.806244373321533,
      "rewards/margins": 2.806244373321533,
      "rewards/rejected": 0.0,
      "step": 2122
    },
    {
      "epoch": 11.860335195530727,
      "grad_norm": 1.3773157729503929,
      "learning_rate": 9.52534401965336e-07,
      "logits/chosen": -3.4123246669769287,
      "logits/rejected": -3.5216379165649414,
      "logps/chosen": -3.4609341621398926,
      "logps/rejected": -78.91799926757812,
      "loss": 0.0755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.039107084274292,
      "rewards/margins": 3.039107084274292,
      "rewards/rejected": 0.0,
      "step": 2123
    },
    {
      "epoch": 11.865921787709498,
      "grad_norm": 0.14820095812850959,
      "learning_rate": 9.501418654029878e-07,
      "logits/chosen": -3.0319855213165283,
      "logits/rejected": -3.007941961288452,
      "logps/chosen": -4.992859363555908,
      "logps/rejected": -72.39254760742188,
      "loss": 0.1098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.247443199157715,
      "rewards/margins": 3.247443199157715,
      "rewards/rejected": 0.0,
      "step": 2124
    },
    {
      "epoch": 11.871508379888269,
      "grad_norm": 0.5581658806462737,
      "learning_rate": 9.477516322058425e-07,
      "logits/chosen": -3.432847738265991,
      "logits/rejected": -3.5374810695648193,
      "logps/chosen": -1.6510238647460938,
      "logps/rejected": -72.46475219726562,
      "loss": 0.1012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4792754650115967,
      "rewards/margins": 2.4792754650115967,
      "rewards/rejected": 0.0,
      "step": 2125
    },
    {
      "epoch": 11.87709497206704,
      "grad_norm": 0.3471962258364664,
      "learning_rate": 9.453637059262119e-07,
      "logits/chosen": -3.5156214237213135,
      "logits/rejected": -3.409743547439575,
      "logps/chosen": -2.002575635910034,
      "logps/rejected": -66.9073257446289,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.518725872039795,
      "rewards/margins": 3.518725872039795,
      "rewards/rejected": 0.0,
      "step": 2126
    },
    {
      "epoch": 11.88268156424581,
      "grad_norm": 0.11658684414138289,
      "learning_rate": 9.429780901129798e-07,
      "logits/chosen": -3.342656135559082,
      "logits/rejected": -3.473238706588745,
      "logps/chosen": -0.05001457780599594,
      "logps/rejected": -63.23524475097656,
      "loss": 0.1264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7209937572479248,
      "rewards/margins": 1.7209937572479248,
      "rewards/rejected": 0.0,
      "step": 2127
    },
    {
      "epoch": 11.888268156424582,
      "grad_norm": 0.4732273920765925,
      "learning_rate": 9.405947883115965e-07,
      "logits/chosen": -3.4681079387664795,
      "logits/rejected": -3.3706822395324707,
      "logps/chosen": -0.6987142562866211,
      "logps/rejected": -107.07435607910156,
      "loss": 0.0911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.095095634460449,
      "rewards/margins": 2.095095634460449,
      "rewards/rejected": 0.0,
      "step": 2128
    },
    {
      "epoch": 11.893854748603353,
      "grad_norm": 0.2560498897124754,
      "learning_rate": 9.382138040640714e-07,
      "logits/chosen": -3.4451162815093994,
      "logits/rejected": -3.6241812705993652,
      "logps/chosen": -0.7612197399139404,
      "logps/rejected": -78.20520782470703,
      "loss": 0.1098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.891972541809082,
      "rewards/margins": 3.891972541809082,
      "rewards/rejected": 0.0,
      "step": 2129
    },
    {
      "epoch": 11.899441340782122,
      "grad_norm": 0.23099937553713856,
      "learning_rate": 9.358351409089744e-07,
      "logits/chosen": -3.6357643604278564,
      "logits/rejected": -3.364360809326172,
      "logps/chosen": -0.9575303196907043,
      "logps/rejected": -86.79127502441406,
      "loss": 0.0912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8177108764648438,
      "rewards/margins": 2.8177108764648438,
      "rewards/rejected": 0.0,
      "step": 2130
    },
    {
      "epoch": 11.905027932960895,
      "grad_norm": 0.2775083384563687,
      "learning_rate": 9.334588023814187e-07,
      "logits/chosen": -3.5837795734405518,
      "logits/rejected": -3.721532106399536,
      "logps/chosen": -21.173358917236328,
      "logps/rejected": -77.8681411743164,
      "loss": 0.0938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6620640754699707,
      "rewards/margins": 3.6620640754699707,
      "rewards/rejected": 0.0,
      "step": 2131
    },
    {
      "epoch": 11.910614525139664,
      "grad_norm": 0.19404273566827196,
      "learning_rate": 9.310847920130695e-07,
      "logits/chosen": -3.2862868309020996,
      "logits/rejected": -3.4090752601623535,
      "logps/chosen": -0.11985038220882416,
      "logps/rejected": -76.44017791748047,
      "loss": 0.1003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.244511365890503,
      "rewards/margins": 2.244511365890503,
      "rewards/rejected": 0.0,
      "step": 2132
    },
    {
      "epoch": 11.916201117318435,
      "grad_norm": 0.1547899077288386,
      "learning_rate": 9.287131133321276e-07,
      "logits/chosen": -3.667191982269287,
      "logits/rejected": -3.6030125617980957,
      "logps/chosen": -1.2039844989776611,
      "logps/rejected": -53.819759368896484,
      "loss": 0.0983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2384963035583496,
      "rewards/margins": 3.2384963035583496,
      "rewards/rejected": 0.0,
      "step": 2133
    },
    {
      "epoch": 11.921787709497206,
      "grad_norm": 0.4554575047709529,
      "learning_rate": 9.263437698633302e-07,
      "logits/chosen": -3.6230242252349854,
      "logits/rejected": -3.6675920486450195,
      "logps/chosen": -6.2709879875183105,
      "logps/rejected": -130.41665649414062,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0353829860687256,
      "rewards/margins": 3.0353829860687256,
      "rewards/rejected": 0.0,
      "step": 2134
    },
    {
      "epoch": 11.927374301675977,
      "grad_norm": 0.31514892514800286,
      "learning_rate": 9.239767651279435e-07,
      "logits/chosen": -3.207146167755127,
      "logits/rejected": -3.3738033771514893,
      "logps/chosen": -23.250364303588867,
      "logps/rejected": -98.33113098144531,
      "loss": 0.1034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.011957168579102,
      "rewards/margins": 4.011957168579102,
      "rewards/rejected": 0.0,
      "step": 2135
    },
    {
      "epoch": 11.932960893854748,
      "grad_norm": 0.2880291865748024,
      "learning_rate": 9.216121026437582e-07,
      "logits/chosen": -3.370464324951172,
      "logits/rejected": -3.398242950439453,
      "logps/chosen": -2.0058069229125977,
      "logps/rejected": -105.86723327636719,
      "loss": 0.1049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5218749046325684,
      "rewards/margins": 3.5218749046325684,
      "rewards/rejected": 0.0,
      "step": 2136
    },
    {
      "epoch": 11.938547486033519,
      "grad_norm": 0.8056892493708053,
      "learning_rate": 9.192497859250843e-07,
      "logits/chosen": -3.505457639694214,
      "logits/rejected": -3.6586875915527344,
      "logps/chosen": -0.5528593063354492,
      "logps/rejected": -102.23004150390625,
      "loss": 0.1365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.549329161643982,
      "rewards/margins": 1.549329161643982,
      "rewards/rejected": 0.0,
      "step": 2137
    },
    {
      "epoch": 11.94413407821229,
      "grad_norm": 1.695993871384258,
      "learning_rate": 9.168898184827444e-07,
      "logits/chosen": -3.3472237586975098,
      "logits/rejected": -3.30665922164917,
      "logps/chosen": -0.4582473635673523,
      "logps/rejected": -51.77047348022461,
      "loss": 0.1094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6994659900665283,
      "rewards/margins": 2.6994659900665283,
      "rewards/rejected": 0.0,
      "step": 2138
    },
    {
      "epoch": 11.949720670391061,
      "grad_norm": 0.36274283201364554,
      "learning_rate": 9.145322038240706e-07,
      "logits/chosen": -3.432640552520752,
      "logits/rejected": -3.38053297996521,
      "logps/chosen": -0.18456915020942688,
      "logps/rejected": -90.82321166992188,
      "loss": 0.1017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3926563262939453,
      "rewards/margins": 2.3926563262939453,
      "rewards/rejected": 0.0,
      "step": 2139
    },
    {
      "epoch": 11.955307262569832,
      "grad_norm": 0.4070355610146338,
      "learning_rate": 9.121769454528978e-07,
      "logits/chosen": -3.6200287342071533,
      "logits/rejected": -3.561007261276245,
      "logps/chosen": -1.060689091682434,
      "logps/rejected": -95.60116577148438,
      "loss": 0.0731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.457488775253296,
      "rewards/margins": 2.457488775253296,
      "rewards/rejected": 0.0,
      "step": 2140
    },
    {
      "epoch": 11.955307262569832,
      "eval_logits/chosen": -3.3648364543914795,
      "eval_logits/rejected": -3.488814115524292,
      "eval_logps/chosen": -42.87978744506836,
      "eval_logps/rejected": -68.26777648925781,
      "eval_loss": 2.081083059310913,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.2809747457504272,
      "eval_rewards/margins": -1.2809747457504272,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5383,
      "eval_samples_per_second": 12.139,
      "eval_steps_per_second": 0.392,
      "step": 2140
    },
    {
      "epoch": 11.960893854748603,
      "grad_norm": 1.7202468860223303,
      "learning_rate": 9.098240468695613e-07,
      "logits/chosen": -3.435312509536743,
      "logits/rejected": -3.307941198348999,
      "logps/chosen": -4.947483062744141,
      "logps/rejected": -59.53804016113281,
      "loss": 0.0918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.42866325378418,
      "rewards/margins": 4.42866325378418,
      "rewards/rejected": 0.0,
      "step": 2141
    },
    {
      "epoch": 11.966480446927374,
      "grad_norm": 1.2681875306915786,
      "learning_rate": 9.074735115708844e-07,
      "logits/chosen": -3.483518362045288,
      "logits/rejected": -3.4921188354492188,
      "logps/chosen": -1.165252685546875,
      "logps/rejected": -66.79515075683594,
      "loss": 0.1055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0645148754119873,
      "rewards/margins": 2.0645148754119873,
      "rewards/rejected": 0.0,
      "step": 2142
    },
    {
      "epoch": 11.972067039106145,
      "grad_norm": 0.5315916447937143,
      "learning_rate": 9.051253430501836e-07,
      "logits/chosen": -3.6370506286621094,
      "logits/rejected": -3.495640754699707,
      "logps/chosen": -1.5603574514389038,
      "logps/rejected": -58.07075500488281,
      "loss": 0.1098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.853677749633789,
      "rewards/margins": 2.853677749633789,
      "rewards/rejected": 0.0,
      "step": 2143
    },
    {
      "epoch": 11.977653631284916,
      "grad_norm": 0.1586977384110443,
      "learning_rate": 9.027795447972546e-07,
      "logits/chosen": -3.633265733718872,
      "logits/rejected": -3.5704939365386963,
      "logps/chosen": -0.14003516733646393,
      "logps/rejected": -61.44328689575195,
      "loss": 0.095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.362863063812256,
      "rewards/margins": 2.362863063812256,
      "rewards/rejected": 0.0,
      "step": 2144
    },
    {
      "epoch": 11.983240223463687,
      "grad_norm": 0.27893308729504274,
      "learning_rate": 9.004361202983716e-07,
      "logits/chosen": -3.5306766033172607,
      "logits/rejected": -3.5522029399871826,
      "logps/chosen": -0.7102161645889282,
      "logps/rejected": -68.01533508300781,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.899732828140259,
      "rewards/margins": 2.899732828140259,
      "rewards/rejected": 0.0,
      "step": 2145
    },
    {
      "epoch": 11.988826815642458,
      "grad_norm": 0.2350222086751996,
      "learning_rate": 8.980950730362803e-07,
      "logits/chosen": -3.275515556335449,
      "logits/rejected": -3.2706689834594727,
      "logps/chosen": -13.178627014160156,
      "logps/rejected": -62.02705001831055,
      "loss": 0.0943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.988784074783325,
      "rewards/margins": 2.988784074783325,
      "rewards/rejected": 0.0,
      "step": 2146
    },
    {
      "epoch": 11.994413407821229,
      "grad_norm": 0.7946492289580153,
      "learning_rate": 8.957564064901944e-07,
      "logits/chosen": -3.2292070388793945,
      "logits/rejected": -3.262571096420288,
      "logps/chosen": -11.629956245422363,
      "logps/rejected": -67.83067321777344,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7314813137054443,
      "rewards/margins": 2.7314813137054443,
      "rewards/rejected": 0.0,
      "step": 2147
    },
    {
      "epoch": 12.0,
      "grad_norm": 0.6325491277826553,
      "learning_rate": 8.934201241357884e-07,
      "logits/chosen": -3.398219108581543,
      "logits/rejected": -3.3386483192443848,
      "logps/chosen": -0.6196783781051636,
      "logps/rejected": -95.35536193847656,
      "loss": 0.1198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7379870414733887,
      "rewards/margins": 1.7379870414733887,
      "rewards/rejected": 0.0,
      "step": 2148
    },
    {
      "epoch": 12.005586592178771,
      "grad_norm": 0.10254772239677566,
      "learning_rate": 8.910862294451936e-07,
      "logits/chosen": -3.660902738571167,
      "logits/rejected": -3.6406760215759277,
      "logps/chosen": -0.4462162256240845,
      "logps/rejected": -82.29328918457031,
      "loss": 0.0961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.058657646179199,
      "rewards/margins": 3.058657646179199,
      "rewards/rejected": 0.0,
      "step": 2149
    },
    {
      "epoch": 12.011173184357542,
      "grad_norm": 0.14895410486043467,
      "learning_rate": 8.887547258869953e-07,
      "logits/chosen": -3.448310375213623,
      "logits/rejected": -3.533085823059082,
      "logps/chosen": -0.934620201587677,
      "logps/rejected": -74.51658630371094,
      "loss": 0.0823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.940392017364502,
      "rewards/margins": 2.940392017364502,
      "rewards/rejected": 0.0,
      "step": 2150
    },
    {
      "epoch": 12.016759776536313,
      "grad_norm": 0.1945127591820832,
      "learning_rate": 8.864256169262195e-07,
      "logits/chosen": -3.4144718647003174,
      "logits/rejected": -3.321838855743408,
      "logps/chosen": -1.1956619024276733,
      "logps/rejected": -82.52387237548828,
      "loss": 0.0838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9651145935058594,
      "rewards/margins": 3.9651145935058594,
      "rewards/rejected": 0.0,
      "step": 2151
    },
    {
      "epoch": 12.022346368715084,
      "grad_norm": 0.12549599012011753,
      "learning_rate": 8.840989060243399e-07,
      "logits/chosen": -3.2785422801971436,
      "logits/rejected": -3.2682042121887207,
      "logps/chosen": -8.808621406555176,
      "logps/rejected": -48.84770584106445,
      "loss": 0.1087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5042243003845215,
      "rewards/margins": 3.5042243003845215,
      "rewards/rejected": 0.0,
      "step": 2152
    },
    {
      "epoch": 12.027932960893855,
      "grad_norm": 0.18038067743058506,
      "learning_rate": 8.817745966392622e-07,
      "logits/chosen": -3.4707021713256836,
      "logits/rejected": -3.361968517303467,
      "logps/chosen": -0.17185574769973755,
      "logps/rejected": -63.8156623840332,
      "loss": 0.0808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.603159189224243,
      "rewards/margins": 2.603159189224243,
      "rewards/rejected": 0.0,
      "step": 2153
    },
    {
      "epoch": 12.033519553072626,
      "grad_norm": 0.09161019421888986,
      "learning_rate": 8.794526922253238e-07,
      "logits/chosen": -3.047985553741455,
      "logits/rejected": -3.1032683849334717,
      "logps/chosen": -10.246236801147461,
      "logps/rejected": -60.526485443115234,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.55924654006958,
      "rewards/margins": 4.55924654006958,
      "rewards/rejected": 0.0,
      "step": 2154
    },
    {
      "epoch": 12.039106145251397,
      "grad_norm": 0.09703589893213896,
      "learning_rate": 8.771331962332888e-07,
      "logits/chosen": -3.2945241928100586,
      "logits/rejected": -3.4224109649658203,
      "logps/chosen": -20.111207962036133,
      "logps/rejected": -99.22517395019531,
      "loss": 0.0767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.248067855834961,
      "rewards/margins": 5.248067855834961,
      "rewards/rejected": 0.0,
      "step": 2155
    },
    {
      "epoch": 12.044692737430168,
      "grad_norm": 0.1659597496162995,
      "learning_rate": 8.748161121103407e-07,
      "logits/chosen": -3.5819883346557617,
      "logits/rejected": -3.626478910446167,
      "logps/chosen": -0.2354295253753662,
      "logps/rejected": -58.466835021972656,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.379962921142578,
      "rewards/margins": 2.379962921142578,
      "rewards/rejected": 0.0,
      "step": 2156
    },
    {
      "epoch": 12.050279329608939,
      "grad_norm": 0.12219293252426848,
      "learning_rate": 8.725014433000795e-07,
      "logits/chosen": -3.49115252494812,
      "logits/rejected": -3.372987985610962,
      "logps/chosen": -6.652415752410889,
      "logps/rejected": -59.83267593383789,
      "loss": 0.0974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0202558040618896,
      "rewards/margins": 3.0202558040618896,
      "rewards/rejected": 0.0,
      "step": 2157
    },
    {
      "epoch": 12.05586592178771,
      "grad_norm": 0.08462498497986634,
      "learning_rate": 8.701891932425143e-07,
      "logits/chosen": -3.4296371936798096,
      "logits/rejected": -3.478586196899414,
      "logps/chosen": -0.231275737285614,
      "logps/rejected": -120.73809814453125,
      "loss": 0.0984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5790927410125732,
      "rewards/margins": 2.5790927410125732,
      "rewards/rejected": 0.0,
      "step": 2158
    },
    {
      "epoch": 12.061452513966481,
      "grad_norm": 0.22983540733538682,
      "learning_rate": 8.678793653740633e-07,
      "logits/chosen": -3.54162859916687,
      "logits/rejected": -3.590346097946167,
      "logps/chosen": -0.4066157341003418,
      "logps/rejected": -65.85282897949219,
      "loss": 0.0914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.558945655822754,
      "rewards/margins": 2.558945655822754,
      "rewards/rejected": 0.0,
      "step": 2159
    },
    {
      "epoch": 12.067039106145252,
      "grad_norm": 0.37155149263721154,
      "learning_rate": 8.655719631275383e-07,
      "logits/chosen": -3.424832582473755,
      "logits/rejected": -3.456247568130493,
      "logps/chosen": -0.5689570903778076,
      "logps/rejected": -68.01518249511719,
      "loss": 0.0896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1541800498962402,
      "rewards/margins": 3.1541800498962402,
      "rewards/rejected": 0.0,
      "step": 2160
    },
    {
      "epoch": 12.067039106145252,
      "eval_logits/chosen": -3.3665809631347656,
      "eval_logits/rejected": -3.48744535446167,
      "eval_logps/chosen": -42.931434631347656,
      "eval_logps/rejected": -68.28543853759766,
      "eval_loss": 2.104212999343872,
      "eval_rewards/accuracies": 0.22499999403953552,
      "eval_rewards/chosen": -1.2861391305923462,
      "eval_rewards/margins": -1.2861391305923462,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5444,
      "eval_samples_per_second": 12.136,
      "eval_steps_per_second": 0.391,
      "step": 2160
    },
    {
      "epoch": 12.072625698324023,
      "grad_norm": 0.1065669705375786,
      "learning_rate": 8.632669899321525e-07,
      "logits/chosen": -3.3381967544555664,
      "logits/rejected": -3.464473247528076,
      "logps/chosen": -2.3358356952667236,
      "logps/rejected": -76.52315521240234,
      "loss": 0.0954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.653250217437744,
      "rewards/margins": 2.653250217437744,
      "rewards/rejected": 0.0,
      "step": 2161
    },
    {
      "epoch": 12.078212290502794,
      "grad_norm": 0.12472341259278016,
      "learning_rate": 8.609644492135061e-07,
      "logits/chosen": -3.4795570373535156,
      "logits/rejected": -3.2816643714904785,
      "logps/chosen": -5.451906681060791,
      "logps/rejected": -54.80775451660156,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3784050941467285,
      "rewards/margins": 3.3784050941467285,
      "rewards/rejected": 0.0,
      "step": 2162
    },
    {
      "epoch": 12.083798882681565,
      "grad_norm": 0.08833750928158626,
      "learning_rate": 8.586643443935844e-07,
      "logits/chosen": -3.46003794670105,
      "logits/rejected": -3.505702495574951,
      "logps/chosen": -0.06967652589082718,
      "logps/rejected": -101.08442687988281,
      "loss": 0.1213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.768619179725647,
      "rewards/margins": 1.768619179725647,
      "rewards/rejected": 0.0,
      "step": 2163
    },
    {
      "epoch": 12.089385474860336,
      "grad_norm": 0.9467971162367058,
      "learning_rate": 8.563666788907527e-07,
      "logits/chosen": -3.0751988887786865,
      "logits/rejected": -2.9335875511169434,
      "logps/chosen": -6.495039463043213,
      "logps/rejected": -65.52890014648438,
      "loss": 0.0858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.667973041534424,
      "rewards/margins": 3.667973041534424,
      "rewards/rejected": 0.0,
      "step": 2164
    },
    {
      "epoch": 12.094972067039107,
      "grad_norm": 0.11535922420761385,
      "learning_rate": 8.540714561197511e-07,
      "logits/chosen": -3.570356607437134,
      "logits/rejected": -3.350933313369751,
      "logps/chosen": -3.1032345294952393,
      "logps/rejected": -52.639739990234375,
      "loss": 0.0786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.1048359870910645,
      "rewards/margins": 4.1048359870910645,
      "rewards/rejected": 0.0,
      "step": 2165
    },
    {
      "epoch": 12.100558659217878,
      "grad_norm": 0.11562329684921287,
      "learning_rate": 8.517786794916899e-07,
      "logits/chosen": -3.361679792404175,
      "logits/rejected": -3.4335505962371826,
      "logps/chosen": -0.8764649629592896,
      "logps/rejected": -97.46157836914062,
      "loss": 0.0747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.676395893096924,
      "rewards/margins": 2.676395893096924,
      "rewards/rejected": 0.0,
      "step": 2166
    },
    {
      "epoch": 12.106145251396647,
      "grad_norm": 0.11428824061073971,
      "learning_rate": 8.494883524140429e-07,
      "logits/chosen": -3.727858543395996,
      "logits/rejected": -3.82019305229187,
      "logps/chosen": -1.7895545959472656,
      "logps/rejected": -92.88700866699219,
      "loss": 0.0666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.711439371109009,
      "rewards/margins": 3.711439371109009,
      "rewards/rejected": 0.0,
      "step": 2167
    },
    {
      "epoch": 12.111731843575418,
      "grad_norm": 0.08158203869874489,
      "learning_rate": 8.472004782906443e-07,
      "logits/chosen": -3.644273519515991,
      "logits/rejected": -3.6097981929779053,
      "logps/chosen": -5.830967903137207,
      "logps/rejected": -41.88825225830078,
      "loss": 0.0891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.103666305541992,
      "rewards/margins": 4.103666305541992,
      "rewards/rejected": 0.0,
      "step": 2168
    },
    {
      "epoch": 12.11731843575419,
      "grad_norm": 0.09686682104992902,
      "learning_rate": 8.449150605216816e-07,
      "logits/chosen": -3.3987488746643066,
      "logits/rejected": -3.440174102783203,
      "logps/chosen": -1.5344719886779785,
      "logps/rejected": -44.51195526123047,
      "loss": 0.1161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3479418754577637,
      "rewards/margins": 3.3479418754577637,
      "rewards/rejected": 0.0,
      "step": 2169
    },
    {
      "epoch": 12.12290502793296,
      "grad_norm": 0.13995630228483852,
      "learning_rate": 8.426321025036951e-07,
      "logits/chosen": -3.6769537925720215,
      "logits/rejected": -3.6082985401153564,
      "logps/chosen": -1.79416024684906,
      "logps/rejected": -66.828125,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9939517974853516,
      "rewards/margins": 2.9939517974853516,
      "rewards/rejected": 0.0,
      "step": 2170
    },
    {
      "epoch": 12.128491620111731,
      "grad_norm": 0.12164946168983413,
      "learning_rate": 8.403516076295639e-07,
      "logits/chosen": -3.3520114421844482,
      "logits/rejected": -3.4161388874053955,
      "logps/chosen": -0.5499618053436279,
      "logps/rejected": -71.06038665771484,
      "loss": 0.068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5531601905822754,
      "rewards/margins": 3.5531601905822754,
      "rewards/rejected": 0.0,
      "step": 2171
    },
    {
      "epoch": 12.134078212290502,
      "grad_norm": 2.9633031085118615,
      "learning_rate": 8.380735792885116e-07,
      "logits/chosen": -3.5239481925964355,
      "logits/rejected": -3.5087242126464844,
      "logps/chosen": -3.018843412399292,
      "logps/rejected": -74.31365966796875,
      "loss": 0.083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7853174209594727,
      "rewards/margins": 3.7853174209594727,
      "rewards/rejected": 0.0,
      "step": 2172
    },
    {
      "epoch": 12.139664804469273,
      "grad_norm": 0.20005850622847662,
      "learning_rate": 8.357980208660934e-07,
      "logits/chosen": -3.59092116355896,
      "logits/rejected": -3.4832725524902344,
      "logps/chosen": -0.3926319479942322,
      "logps/rejected": -48.39067840576172,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.309767723083496,
      "rewards/margins": 3.309767723083496,
      "rewards/rejected": 0.0,
      "step": 2173
    },
    {
      "epoch": 12.145251396648044,
      "grad_norm": 0.1132934521171061,
      "learning_rate": 8.335249357441946e-07,
      "logits/chosen": -3.795271873474121,
      "logits/rejected": -3.856858491897583,
      "logps/chosen": -0.4241487383842468,
      "logps/rejected": -84.50342559814453,
      "loss": 0.1202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6713199615478516,
      "rewards/margins": 2.6713199615478516,
      "rewards/rejected": 0.0,
      "step": 2174
    },
    {
      "epoch": 12.150837988826815,
      "grad_norm": 0.1768681154342842,
      "learning_rate": 8.31254327301024e-07,
      "logits/chosen": -3.496389150619507,
      "logits/rejected": -3.4904351234436035,
      "logps/chosen": -6.638392925262451,
      "logps/rejected": -59.84599304199219,
      "loss": 0.0902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.620640516281128,
      "rewards/margins": 3.620640516281128,
      "rewards/rejected": 0.0,
      "step": 2175
    },
    {
      "epoch": 12.156424581005586,
      "grad_norm": 0.09696376246034302,
      "learning_rate": 8.289861989111106e-07,
      "logits/chosen": -3.6196939945220947,
      "logits/rejected": -3.7376344203948975,
      "logps/chosen": -0.09543084353208542,
      "logps/rejected": -75.278564453125,
      "loss": 0.0864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.011669397354126,
      "rewards/margins": 2.011669397354126,
      "rewards/rejected": 0.0,
      "step": 2176
    },
    {
      "epoch": 12.162011173184357,
      "grad_norm": 0.14590687356756327,
      "learning_rate": 8.267205539452966e-07,
      "logits/chosen": -3.2906410694122314,
      "logits/rejected": -3.294926643371582,
      "logps/chosen": -37.20609664916992,
      "logps/rejected": -68.93276977539062,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9052791595458984,
      "rewards/margins": 3.9052791595458984,
      "rewards/rejected": 0.0,
      "step": 2177
    },
    {
      "epoch": 12.167597765363128,
      "grad_norm": 0.1101686278706454,
      "learning_rate": 8.244573957707336e-07,
      "logits/chosen": -3.4795351028442383,
      "logits/rejected": -3.614997386932373,
      "logps/chosen": -0.35969236493110657,
      "logps/rejected": -83.75214385986328,
      "loss": 0.1066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3040404319763184,
      "rewards/margins": 2.3040404319763184,
      "rewards/rejected": 0.0,
      "step": 2178
    },
    {
      "epoch": 12.1731843575419,
      "grad_norm": 0.6738824956734727,
      "learning_rate": 8.221967277508797e-07,
      "logits/chosen": -3.4383652210235596,
      "logits/rejected": -3.403701066970825,
      "logps/chosen": -0.13539093732833862,
      "logps/rejected": -55.01152801513672,
      "loss": 0.1148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3137803077697754,
      "rewards/margins": 2.3137803077697754,
      "rewards/rejected": 0.0,
      "step": 2179
    },
    {
      "epoch": 12.17877094972067,
      "grad_norm": 0.3867134775541706,
      "learning_rate": 8.199385532454874e-07,
      "logits/chosen": -3.492438554763794,
      "logits/rejected": -3.5623831748962402,
      "logps/chosen": -3.0039432048797607,
      "logps/rejected": -70.82223510742188,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.795682430267334,
      "rewards/margins": 3.795682430267334,
      "rewards/rejected": 0.0,
      "step": 2180
    },
    {
      "epoch": 12.17877094972067,
      "eval_logits/chosen": -3.3561851978302,
      "eval_logits/rejected": -3.479769229888916,
      "eval_logps/chosen": -44.280982971191406,
      "eval_logps/rejected": -70.02418518066406,
      "eval_loss": 2.2080042362213135,
      "eval_rewards/accuracies": 0.15000000596046448,
      "eval_rewards/chosen": -1.4210938215255737,
      "eval_rewards/margins": -1.4210938215255737,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5376,
      "eval_samples_per_second": 12.139,
      "eval_steps_per_second": 0.392,
      "step": 2180
    },
    {
      "epoch": 12.184357541899441,
      "grad_norm": 1.0615278638318766,
      "learning_rate": 8.176828756106084e-07,
      "logits/chosen": -3.6680381298065186,
      "logits/rejected": -3.5350210666656494,
      "logps/chosen": -2.0726351737976074,
      "logps/rejected": -78.08897399902344,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8205463886260986,
      "rewards/margins": 2.8205463886260986,
      "rewards/rejected": 0.0,
      "step": 2181
    },
    {
      "epoch": 12.189944134078212,
      "grad_norm": 0.16792857741180547,
      "learning_rate": 8.154296981985801e-07,
      "logits/chosen": -3.550473928451538,
      "logits/rejected": -3.4473624229431152,
      "logps/chosen": -5.452842712402344,
      "logps/rejected": -68.52281188964844,
      "loss": 0.1123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.132704734802246,
      "rewards/margins": 4.132704734802246,
      "rewards/rejected": 0.0,
      "step": 2182
    },
    {
      "epoch": 12.195530726256983,
      "grad_norm": 0.43151489879562877,
      "learning_rate": 8.131790243580263e-07,
      "logits/chosen": -3.57478666305542,
      "logits/rejected": -3.61504864692688,
      "logps/chosen": -0.4931708574295044,
      "logps/rejected": -53.17253875732422,
      "loss": 0.0785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4862325191497803,
      "rewards/margins": 3.4862325191497803,
      "rewards/rejected": 0.0,
      "step": 2183
    },
    {
      "epoch": 12.201117318435754,
      "grad_norm": 0.2555173975336384,
      "learning_rate": 8.109308574338484e-07,
      "logits/chosen": -3.284435749053955,
      "logits/rejected": -3.367950439453125,
      "logps/chosen": -0.2981560230255127,
      "logps/rejected": -73.44334411621094,
      "loss": 0.1425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.757887363433838,
      "rewards/margins": 2.757887363433838,
      "rewards/rejected": 0.0,
      "step": 2184
    },
    {
      "epoch": 12.206703910614525,
      "grad_norm": 0.14111737091611384,
      "learning_rate": 8.086852007672233e-07,
      "logits/chosen": -3.5072433948516846,
      "logits/rejected": -3.5931742191314697,
      "logps/chosen": -0.2279001623392105,
      "logps/rejected": -101.49644470214844,
      "loss": 0.104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.381680965423584,
      "rewards/margins": 2.381680965423584,
      "rewards/rejected": 0.0,
      "step": 2185
    },
    {
      "epoch": 12.212290502793296,
      "grad_norm": 0.29308340883742384,
      "learning_rate": 8.064420576955967e-07,
      "logits/chosen": -3.369846820831299,
      "logits/rejected": -3.4479525089263916,
      "logps/chosen": -0.1024254709482193,
      "logps/rejected": -83.58304595947266,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.916998028755188,
      "rewards/margins": 1.916998028755188,
      "rewards/rejected": 0.0,
      "step": 2186
    },
    {
      "epoch": 12.217877094972067,
      "grad_norm": 0.102460156806573,
      "learning_rate": 8.042014315526775e-07,
      "logits/chosen": -3.509040117263794,
      "logits/rejected": -3.4784958362579346,
      "logps/chosen": -10.205280303955078,
      "logps/rejected": -97.50801849365234,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.090417861938477,
      "rewards/margins": 4.090417861938477,
      "rewards/rejected": 0.0,
      "step": 2187
    },
    {
      "epoch": 12.223463687150838,
      "grad_norm": 0.16641536036064655,
      "learning_rate": 8.019633256684378e-07,
      "logits/chosen": -3.4917831420898438,
      "logits/rejected": -3.697260618209839,
      "logps/chosen": -0.18131156265735626,
      "logps/rejected": -128.21035766601562,
      "loss": 0.0574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.54227614402771,
      "rewards/margins": 2.54227614402771,
      "rewards/rejected": 0.0,
      "step": 2188
    },
    {
      "epoch": 12.22905027932961,
      "grad_norm": 0.15800368631598566,
      "learning_rate": 7.997277433690984e-07,
      "logits/chosen": -3.572904348373413,
      "logits/rejected": -3.21777081489563,
      "logps/chosen": -1.9938091039657593,
      "logps/rejected": -107.69142150878906,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.649350643157959,
      "rewards/margins": 2.649350643157959,
      "rewards/rejected": 0.0,
      "step": 2189
    },
    {
      "epoch": 12.23463687150838,
      "grad_norm": 0.1310796055803447,
      "learning_rate": 7.974946879771351e-07,
      "logits/chosen": -3.292522668838501,
      "logits/rejected": -3.3569891452789307,
      "logps/chosen": -0.24615222215652466,
      "logps/rejected": -38.184837341308594,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.257990598678589,
      "rewards/margins": 2.257990598678589,
      "rewards/rejected": 0.0,
      "step": 2190
    },
    {
      "epoch": 12.240223463687151,
      "grad_norm": 2.1318834593354743,
      "learning_rate": 7.95264162811265e-07,
      "logits/chosen": -3.5825412273406982,
      "logits/rejected": -3.6854171752929688,
      "logps/chosen": -1.3761705160140991,
      "logps/rejected": -47.96111297607422,
      "loss": 0.1083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2716500759124756,
      "rewards/margins": 2.2716500759124756,
      "rewards/rejected": 0.0,
      "step": 2191
    },
    {
      "epoch": 12.245810055865922,
      "grad_norm": 0.2580833268889084,
      "learning_rate": 7.930361711864457e-07,
      "logits/chosen": -3.2511825561523438,
      "logits/rejected": -3.118647336959839,
      "logps/chosen": -15.545101165771484,
      "logps/rejected": -63.971160888671875,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1463072299957275,
      "rewards/margins": 3.1463072299957275,
      "rewards/rejected": 0.0,
      "step": 2192
    },
    {
      "epoch": 12.251396648044693,
      "grad_norm": 0.39774663948868005,
      "learning_rate": 7.908107164138699e-07,
      "logits/chosen": -3.637429714202881,
      "logits/rejected": -3.6206884384155273,
      "logps/chosen": -0.17972630262374878,
      "logps/rejected": -62.7713737487793,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.00229811668396,
      "rewards/margins": 2.00229811668396,
      "rewards/rejected": 0.0,
      "step": 2193
    },
    {
      "epoch": 12.256983240223464,
      "grad_norm": 0.28907044748614596,
      "learning_rate": 7.885878018009593e-07,
      "logits/chosen": -3.4087541103363037,
      "logits/rejected": -3.5876479148864746,
      "logps/chosen": -1.144510269165039,
      "logps/rejected": -65.46754455566406,
      "loss": 0.0867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6926960945129395,
      "rewards/margins": 3.6926960945129395,
      "rewards/rejected": 0.0,
      "step": 2194
    },
    {
      "epoch": 12.262569832402235,
      "grad_norm": 0.13499181770768093,
      "learning_rate": 7.86367430651361e-07,
      "logits/chosen": -3.569227457046509,
      "logits/rejected": -3.6841235160827637,
      "logps/chosen": -1.8511617183685303,
      "logps/rejected": -71.47462463378906,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6024856567382812,
      "rewards/margins": 3.6024856567382812,
      "rewards/rejected": 0.0,
      "step": 2195
    },
    {
      "epoch": 12.268156424581006,
      "grad_norm": 0.49555672497532194,
      "learning_rate": 7.841496062649423e-07,
      "logits/chosen": -3.4533755779266357,
      "logits/rejected": -3.460862636566162,
      "logps/chosen": -0.199654221534729,
      "logps/rejected": -92.62687683105469,
      "loss": 0.0982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2728629112243652,
      "rewards/margins": 2.2728629112243652,
      "rewards/rejected": 0.0,
      "step": 2196
    },
    {
      "epoch": 12.273743016759777,
      "grad_norm": 1.1607358316323173,
      "learning_rate": 7.819343319377848e-07,
      "logits/chosen": -3.3773584365844727,
      "logits/rejected": -3.4888947010040283,
      "logps/chosen": -1.6158230304718018,
      "logps/rejected": -88.06248474121094,
      "loss": 0.122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4649019241333008,
      "rewards/margins": 1.4649019241333008,
      "rewards/rejected": 0.0,
      "step": 2197
    },
    {
      "epoch": 12.279329608938548,
      "grad_norm": 0.354419389964084,
      "learning_rate": 7.797216109621799e-07,
      "logits/chosen": -3.369870662689209,
      "logits/rejected": -3.437084913253784,
      "logps/chosen": -1.6238006353378296,
      "logps/rejected": -78.35745239257812,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5160396099090576,
      "rewards/margins": 3.5160396099090576,
      "rewards/rejected": 0.0,
      "step": 2198
    },
    {
      "epoch": 12.28491620111732,
      "grad_norm": 0.08744076601619137,
      "learning_rate": 7.775114466266275e-07,
      "logits/chosen": -3.501275062561035,
      "logits/rejected": -3.482795238494873,
      "logps/chosen": -0.6856967806816101,
      "logps/rejected": -68.53883361816406,
      "loss": 0.0787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3792099952697754,
      "rewards/margins": 3.3792099952697754,
      "rewards/rejected": 0.0,
      "step": 2199
    },
    {
      "epoch": 12.29050279329609,
      "grad_norm": 0.07909513132031225,
      "learning_rate": 7.753038422158224e-07,
      "logits/chosen": -3.410501480102539,
      "logits/rejected": -3.236994981765747,
      "logps/chosen": -0.5070946216583252,
      "logps/rejected": -49.889068603515625,
      "loss": 0.1031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.391023635864258,
      "rewards/margins": 3.391023635864258,
      "rewards/rejected": 0.0,
      "step": 2200
    },
    {
      "epoch": 12.29050279329609,
      "eval_logits/chosen": -3.3605098724365234,
      "eval_logits/rejected": -3.485619306564331,
      "eval_logps/chosen": -44.16472625732422,
      "eval_logps/rejected": -70.3009262084961,
      "eval_loss": 2.202542304992676,
      "eval_rewards/accuracies": 0.20000000298023224,
      "eval_rewards/chosen": -1.4094680547714233,
      "eval_rewards/margins": -1.4094680547714233,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5401,
      "eval_samples_per_second": 12.138,
      "eval_steps_per_second": 0.392,
      "step": 2200
    },
    {
      "epoch": 12.296089385474861,
      "grad_norm": 0.10145950435099736,
      "learning_rate": 7.730988010106602e-07,
      "logits/chosen": -3.5856740474700928,
      "logits/rejected": -3.597609758377075,
      "logps/chosen": -18.42974281311035,
      "logps/rejected": -82.58443450927734,
      "loss": 0.1119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.266204833984375,
      "rewards/margins": 4.266204833984375,
      "rewards/rejected": 0.0,
      "step": 2201
    },
    {
      "epoch": 12.30167597765363,
      "grad_norm": 0.22222206662925362,
      "learning_rate": 7.708963262882238e-07,
      "logits/chosen": -3.545236349105835,
      "logits/rejected": -3.505925416946411,
      "logps/chosen": -0.5697951316833496,
      "logps/rejected": -97.42155456542969,
      "loss": 0.1007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.947145462036133,
      "rewards/margins": 2.947145462036133,
      "rewards/rejected": 0.0,
      "step": 2202
    },
    {
      "epoch": 12.307262569832401,
      "grad_norm": 0.1276092711272246,
      "learning_rate": 7.686964213217831e-07,
      "logits/chosen": -3.5831215381622314,
      "logits/rejected": -3.4893126487731934,
      "logps/chosen": -1.9516427516937256,
      "logps/rejected": -68.19964599609375,
      "loss": 0.1139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0268101692199707,
      "rewards/margins": 3.0268101692199707,
      "rewards/rejected": 0.0,
      "step": 2203
    },
    {
      "epoch": 12.312849162011172,
      "grad_norm": 0.26061650982642515,
      "learning_rate": 7.664990893807886e-07,
      "logits/chosen": -3.588297128677368,
      "logits/rejected": -3.4079651832580566,
      "logps/chosen": -0.548957109451294,
      "logps/rejected": -73.55818939208984,
      "loss": 0.1217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2185940742492676,
      "rewards/margins": 3.2185940742492676,
      "rewards/rejected": 0.0,
      "step": 2204
    },
    {
      "epoch": 12.318435754189943,
      "grad_norm": 0.12425643159348736,
      "learning_rate": 7.64304333730867e-07,
      "logits/chosen": -3.5779762268066406,
      "logits/rejected": -3.533071517944336,
      "logps/chosen": -6.4131903648376465,
      "logps/rejected": -72.72831726074219,
      "loss": 0.0861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.860288619995117,
      "rewards/margins": 4.860288619995117,
      "rewards/rejected": 0.0,
      "step": 2205
    },
    {
      "epoch": 12.324022346368714,
      "grad_norm": 0.36576310118394956,
      "learning_rate": 7.621121576338158e-07,
      "logits/chosen": -3.2886738777160645,
      "logits/rejected": -3.4614012241363525,
      "logps/chosen": -0.2177520990371704,
      "logps/rejected": -123.01470947265625,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.102242946624756,
      "rewards/margins": 2.102242946624756,
      "rewards/rejected": 0.0,
      "step": 2206
    },
    {
      "epoch": 12.329608938547485,
      "grad_norm": 0.3514203906836094,
      "learning_rate": 7.599225643475988e-07,
      "logits/chosen": -3.5565762519836426,
      "logits/rejected": -3.5639235973358154,
      "logps/chosen": -0.460485577583313,
      "logps/rejected": -54.81296157836914,
      "loss": 0.1003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.689765453338623,
      "rewards/margins": 2.689765453338623,
      "rewards/rejected": 0.0,
      "step": 2207
    },
    {
      "epoch": 12.335195530726256,
      "grad_norm": 1.1616508992810917,
      "learning_rate": 7.577355571263434e-07,
      "logits/chosen": -3.507600784301758,
      "logits/rejected": -3.5847432613372803,
      "logps/chosen": -0.9611777663230896,
      "logps/rejected": -108.77394104003906,
      "loss": 0.116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.032489776611328,
      "rewards/margins": 2.032489776611328,
      "rewards/rejected": 0.0,
      "step": 2208
    },
    {
      "epoch": 12.340782122905027,
      "grad_norm": 0.1013489973345589,
      "learning_rate": 7.555511392203288e-07,
      "logits/chosen": -3.590756416320801,
      "logits/rejected": -3.4581515789031982,
      "logps/chosen": -0.11960569769144058,
      "logps/rejected": -48.060546875,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5230588912963867,
      "rewards/margins": 2.5230588912963867,
      "rewards/rejected": 0.0,
      "step": 2209
    },
    {
      "epoch": 12.346368715083798,
      "grad_norm": 0.36165509422172304,
      "learning_rate": 7.533693138759914e-07,
      "logits/chosen": -3.7064852714538574,
      "logits/rejected": -3.7908544540405273,
      "logps/chosen": -5.437068462371826,
      "logps/rejected": -67.31111145019531,
      "loss": 0.0704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.33807897567749,
      "rewards/margins": 4.33807897567749,
      "rewards/rejected": 0.0,
      "step": 2210
    },
    {
      "epoch": 12.35195530726257,
      "grad_norm": 0.07730488426018318,
      "learning_rate": 7.511900843359116e-07,
      "logits/chosen": -3.837568521499634,
      "logits/rejected": -3.7271018028259277,
      "logps/chosen": -0.8071357011795044,
      "logps/rejected": -64.83325958251953,
      "loss": 0.0916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2321548461914062,
      "rewards/margins": 3.2321548461914062,
      "rewards/rejected": 0.0,
      "step": 2211
    },
    {
      "epoch": 12.35754189944134,
      "grad_norm": 0.21146281191220032,
      "learning_rate": 7.490134538388124e-07,
      "logits/chosen": -3.5426344871520996,
      "logits/rejected": -3.301419973373413,
      "logps/chosen": -0.14142431318759918,
      "logps/rejected": -71.39862060546875,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3464455604553223,
      "rewards/margins": 2.3464455604553223,
      "rewards/rejected": 0.0,
      "step": 2212
    },
    {
      "epoch": 12.363128491620111,
      "grad_norm": 0.14700295004885378,
      "learning_rate": 7.468394256195544e-07,
      "logits/chosen": -3.68599796295166,
      "logits/rejected": -3.5895421504974365,
      "logps/chosen": -0.07358697056770325,
      "logps/rejected": -62.77702713012695,
      "loss": 0.1242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7742491960525513,
      "rewards/margins": 1.7742491960525513,
      "rewards/rejected": 0.0,
      "step": 2213
    },
    {
      "epoch": 12.368715083798882,
      "grad_norm": 0.2879795055387402,
      "learning_rate": 7.446680029091311e-07,
      "logits/chosen": -3.4342074394226074,
      "logits/rejected": -3.4097957611083984,
      "logps/chosen": -0.5165625214576721,
      "logps/rejected": -89.05278015136719,
      "loss": 0.1061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.838270664215088,
      "rewards/margins": 2.838270664215088,
      "rewards/rejected": 0.0,
      "step": 2214
    },
    {
      "epoch": 12.374301675977653,
      "grad_norm": 0.1219707079496721,
      "learning_rate": 7.424991889346625e-07,
      "logits/chosen": -3.692018985748291,
      "logits/rejected": -3.703463315963745,
      "logps/chosen": -0.36028236150741577,
      "logps/rejected": -51.88376235961914,
      "loss": 0.0869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8737761974334717,
      "rewards/margins": 2.8737761974334717,
      "rewards/rejected": 0.0,
      "step": 2215
    },
    {
      "epoch": 12.379888268156424,
      "grad_norm": 0.08447187951585838,
      "learning_rate": 7.403329869193923e-07,
      "logits/chosen": -3.448641777038574,
      "logits/rejected": -3.343628168106079,
      "logps/chosen": -4.391959190368652,
      "logps/rejected": -96.07994079589844,
      "loss": 0.1057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6317970752716064,
      "rewards/margins": 3.6317970752716064,
      "rewards/rejected": 0.0,
      "step": 2216
    },
    {
      "epoch": 12.385474860335195,
      "grad_norm": 0.1168395357760994,
      "learning_rate": 7.38169400082685e-07,
      "logits/chosen": -3.552786111831665,
      "logits/rejected": -3.7208662033081055,
      "logps/chosen": -0.262599378824234,
      "logps/rejected": -92.16506958007812,
      "loss": 0.104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8984813690185547,
      "rewards/margins": 2.8984813690185547,
      "rewards/rejected": 0.0,
      "step": 2217
    },
    {
      "epoch": 12.391061452513966,
      "grad_norm": 1.9181942331663804,
      "learning_rate": 7.360084316400121e-07,
      "logits/chosen": -3.4557387828826904,
      "logits/rejected": -3.4349684715270996,
      "logps/chosen": -0.4065290689468384,
      "logps/rejected": -59.814998626708984,
      "loss": 0.102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7871999740600586,
      "rewards/margins": 2.7871999740600586,
      "rewards/rejected": 0.0,
      "step": 2218
    },
    {
      "epoch": 12.396648044692737,
      "grad_norm": 0.2565961927489584,
      "learning_rate": 7.338500848029603e-07,
      "logits/chosen": -3.278639554977417,
      "logits/rejected": -3.078571081161499,
      "logps/chosen": -37.93883514404297,
      "logps/rejected": -65.4959487915039,
      "loss": 0.0894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.811744689941406,
      "rewards/margins": 5.811744689941406,
      "rewards/rejected": 0.0,
      "step": 2219
    },
    {
      "epoch": 12.402234636871508,
      "grad_norm": 0.5905681096166113,
      "learning_rate": 7.316943627792166e-07,
      "logits/chosen": -3.247708797454834,
      "logits/rejected": -3.3719263076782227,
      "logps/chosen": -0.29293307662010193,
      "logps/rejected": -108.356201171875,
      "loss": 0.0805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4450294971466064,
      "rewards/margins": 2.4450294971466064,
      "rewards/rejected": 0.0,
      "step": 2220
    },
    {
      "epoch": 12.402234636871508,
      "eval_logits/chosen": -3.370436191558838,
      "eval_logits/rejected": -3.4964072704315186,
      "eval_logps/chosen": -44.6605224609375,
      "eval_logps/rejected": -70.91153717041016,
      "eval_loss": 2.228705406188965,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.4590485095977783,
      "eval_rewards/margins": -1.4590485095977783,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5407,
      "eval_samples_per_second": 12.137,
      "eval_steps_per_second": 0.392,
      "step": 2220
    },
    {
      "epoch": 12.40782122905028,
      "grad_norm": 0.3033524488483871,
      "learning_rate": 7.295412687725681e-07,
      "logits/chosen": -3.542189836502075,
      "logits/rejected": -3.6378915309906006,
      "logps/chosen": -4.4889655113220215,
      "logps/rejected": -38.02594757080078,
      "loss": 0.129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.772796154022217,
      "rewards/margins": 3.772796154022217,
      "rewards/rejected": 0.0,
      "step": 2221
    },
    {
      "epoch": 12.41340782122905,
      "grad_norm": 0.13094140952108407,
      "learning_rate": 7.273908059828955e-07,
      "logits/chosen": -3.450052261352539,
      "logits/rejected": -3.5853981971740723,
      "logps/chosen": -0.8291620016098022,
      "logps/rejected": -76.88418579101562,
      "loss": 0.0866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9808130264282227,
      "rewards/margins": 2.9808130264282227,
      "rewards/rejected": 0.0,
      "step": 2222
    },
    {
      "epoch": 12.418994413407821,
      "grad_norm": 0.5557672059517287,
      "learning_rate": 7.252429776061695e-07,
      "logits/chosen": -3.47406005859375,
      "logits/rejected": -3.6141366958618164,
      "logps/chosen": -1.0079765319824219,
      "logps/rejected": -75.19110870361328,
      "loss": 0.1122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5499491691589355,
      "rewards/margins": 2.5499491691589355,
      "rewards/rejected": 0.0,
      "step": 2223
    },
    {
      "epoch": 12.424581005586592,
      "grad_norm": 0.09906098337331838,
      "learning_rate": 7.230977868344457e-07,
      "logits/chosen": -3.067068338394165,
      "logits/rejected": -3.011582612991333,
      "logps/chosen": -5.141869068145752,
      "logps/rejected": -70.24311065673828,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3996453285217285,
      "rewards/margins": 3.3996453285217285,
      "rewards/rejected": 0.0,
      "step": 2224
    },
    {
      "epoch": 12.430167597765363,
      "grad_norm": 0.134164928556601,
      "learning_rate": 7.209552368558598e-07,
      "logits/chosen": -3.410630941390991,
      "logits/rejected": -3.287770986557007,
      "logps/chosen": -4.865798473358154,
      "logps/rejected": -69.9771957397461,
      "loss": 0.0903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.964966297149658,
      "rewards/margins": 3.964966297149658,
      "rewards/rejected": 0.0,
      "step": 2225
    },
    {
      "epoch": 12.435754189944134,
      "grad_norm": 1.1959426953168224,
      "learning_rate": 7.188153308546223e-07,
      "logits/chosen": -3.312589406967163,
      "logits/rejected": -3.4293532371520996,
      "logps/chosen": -0.21580761671066284,
      "logps/rejected": -109.14979553222656,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.852475166320801,
      "rewards/margins": 2.852475166320801,
      "rewards/rejected": 0.0,
      "step": 2226
    },
    {
      "epoch": 12.441340782122905,
      "grad_norm": 0.24547643986147472,
      "learning_rate": 7.166780720110142e-07,
      "logits/chosen": -3.5673019886016846,
      "logits/rejected": -3.8634068965911865,
      "logps/chosen": -1.0333565473556519,
      "logps/rejected": -81.93282318115234,
      "loss": 0.0711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.047893524169922,
      "rewards/margins": 3.047893524169922,
      "rewards/rejected": 0.0,
      "step": 2227
    },
    {
      "epoch": 12.446927374301676,
      "grad_norm": 0.1179427682698932,
      "learning_rate": 7.145434635013849e-07,
      "logits/chosen": -3.015780448913574,
      "logits/rejected": -3.0484142303466797,
      "logps/chosen": -26.108304977416992,
      "logps/rejected": -92.74834442138672,
      "loss": 0.0947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.631553888320923,
      "rewards/margins": 3.631553888320923,
      "rewards/rejected": 0.0,
      "step": 2228
    },
    {
      "epoch": 12.452513966480447,
      "grad_norm": 0.08824591757588997,
      "learning_rate": 7.124115084981395e-07,
      "logits/chosen": -3.620087146759033,
      "logits/rejected": -3.5941741466522217,
      "logps/chosen": -0.172592431306839,
      "logps/rejected": -58.853309631347656,
      "loss": 0.1561,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.780824661254883,
      "rewards/margins": 2.780824661254883,
      "rewards/rejected": 0.0,
      "step": 2229
    },
    {
      "epoch": 12.458100558659218,
      "grad_norm": 0.21610399198148433,
      "learning_rate": 7.102822101697455e-07,
      "logits/chosen": -3.5490987300872803,
      "logits/rejected": -3.5512425899505615,
      "logps/chosen": -0.3863663375377655,
      "logps/rejected": -50.45726776123047,
      "loss": 0.0887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1009395122528076,
      "rewards/margins": 3.1009395122528076,
      "rewards/rejected": 0.0,
      "step": 2230
    },
    {
      "epoch": 12.46368715083799,
      "grad_norm": 0.3573689386094063,
      "learning_rate": 7.081555716807184e-07,
      "logits/chosen": -3.646651029586792,
      "logits/rejected": -3.5201468467712402,
      "logps/chosen": -2.6739282608032227,
      "logps/rejected": -77.84584045410156,
      "loss": 0.0898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.999495029449463,
      "rewards/margins": 2.999495029449463,
      "rewards/rejected": 0.0,
      "step": 2231
    },
    {
      "epoch": 12.46927374301676,
      "grad_norm": 0.23591757451871498,
      "learning_rate": 7.060315961916217e-07,
      "logits/chosen": -3.687699794769287,
      "logits/rejected": -3.65145206451416,
      "logps/chosen": -0.3408184051513672,
      "logps/rejected": -56.04563903808594,
      "loss": 0.0893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.676250457763672,
      "rewards/margins": 2.676250457763672,
      "rewards/rejected": 0.0,
      "step": 2232
    },
    {
      "epoch": 12.474860335195531,
      "grad_norm": 0.4127265468484968,
      "learning_rate": 7.039102868590616e-07,
      "logits/chosen": -3.2535645961761475,
      "logits/rejected": -3.5377118587493896,
      "logps/chosen": -0.06184878945350647,
      "logps/rejected": -105.34030151367188,
      "loss": 0.1299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9724445343017578,
      "rewards/margins": 1.9724445343017578,
      "rewards/rejected": 0.0,
      "step": 2233
    },
    {
      "epoch": 12.480446927374302,
      "grad_norm": 0.3001829564445068,
      "learning_rate": 7.017916468356811e-07,
      "logits/chosen": -3.3122293949127197,
      "logits/rejected": -3.228391408920288,
      "logps/chosen": -1.5208539962768555,
      "logps/rejected": -77.4238052368164,
      "loss": 0.114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.915834665298462,
      "rewards/margins": 2.915834665298462,
      "rewards/rejected": 0.0,
      "step": 2234
    },
    {
      "epoch": 12.486033519553073,
      "grad_norm": 0.5571142694637294,
      "learning_rate": 6.996756792701571e-07,
      "logits/chosen": -3.5592308044433594,
      "logits/rejected": -3.5457026958465576,
      "logps/chosen": -0.1729048192501068,
      "logps/rejected": -50.82034683227539,
      "loss": 0.0962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6304407119750977,
      "rewards/margins": 2.6304407119750977,
      "rewards/rejected": 0.0,
      "step": 2235
    },
    {
      "epoch": 12.491620111731844,
      "grad_norm": 0.4969926494791256,
      "learning_rate": 6.975623873071929e-07,
      "logits/chosen": -3.255380630493164,
      "logits/rejected": -3.2257299423217773,
      "logps/chosen": -1.5819125175476074,
      "logps/rejected": -65.07452392578125,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0281922817230225,
      "rewards/margins": 3.0281922817230225,
      "rewards/rejected": 0.0,
      "step": 2236
    },
    {
      "epoch": 12.497206703910614,
      "grad_norm": 0.16470149766228076,
      "learning_rate": 6.954517740875194e-07,
      "logits/chosen": -3.5538127422332764,
      "logits/rejected": -3.607161283493042,
      "logps/chosen": -0.06990891695022583,
      "logps/rejected": -53.77130126953125,
      "loss": 0.0917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.119767189025879,
      "rewards/margins": 2.119767189025879,
      "rewards/rejected": 0.0,
      "step": 2237
    },
    {
      "epoch": 12.502793296089386,
      "grad_norm": 0.450573853779234,
      "learning_rate": 6.933438427478805e-07,
      "logits/chosen": -3.5186715126037598,
      "logits/rejected": -3.4668102264404297,
      "logps/chosen": -0.10543271899223328,
      "logps/rejected": -82.37443542480469,
      "loss": 0.0838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9921623468399048,
      "rewards/margins": 1.9921623468399048,
      "rewards/rejected": 0.0,
      "step": 2238
    },
    {
      "epoch": 12.508379888268156,
      "grad_norm": 0.36263385223415723,
      "learning_rate": 6.912385964210399e-07,
      "logits/chosen": -3.2566263675689697,
      "logits/rejected": -3.2345542907714844,
      "logps/chosen": -0.3076080083847046,
      "logps/rejected": -57.8537712097168,
      "loss": 0.0905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.902616024017334,
      "rewards/margins": 2.902616024017334,
      "rewards/rejected": 0.0,
      "step": 2239
    },
    {
      "epoch": 12.513966480446927,
      "grad_norm": 0.08633350206820649,
      "learning_rate": 6.891360382357679e-07,
      "logits/chosen": -3.5761513710021973,
      "logits/rejected": -3.525818109512329,
      "logps/chosen": -0.022936860099434853,
      "logps/rejected": -65.47378540039062,
      "loss": 0.1217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5970302820205688,
      "rewards/margins": 1.5970302820205688,
      "rewards/rejected": 0.0,
      "step": 2240
    },
    {
      "epoch": 12.513966480446927,
      "eval_logits/chosen": -3.3589859008789062,
      "eval_logits/rejected": -3.48603892326355,
      "eval_logps/chosen": -45.14112854003906,
      "eval_logps/rejected": -71.3597183227539,
      "eval_loss": 2.2616820335388184,
      "eval_rewards/accuracies": 0.17499999701976776,
      "eval_rewards/chosen": -1.507108449935913,
      "eval_rewards/margins": -1.507108449935913,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5375,
      "eval_samples_per_second": 12.139,
      "eval_steps_per_second": 0.392,
      "step": 2240
    },
    {
      "epoch": 12.519553072625698,
      "grad_norm": 0.21341746176112514,
      "learning_rate": 6.8703617131684e-07,
      "logits/chosen": -3.6513051986694336,
      "logits/rejected": -3.4867734909057617,
      "logps/chosen": -0.5486360788345337,
      "logps/rejected": -61.91944885253906,
      "loss": 0.108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0316991806030273,
      "rewards/margins": 3.0316991806030273,
      "rewards/rejected": 0.0,
      "step": 2241
    },
    {
      "epoch": 12.525139664804469,
      "grad_norm": 0.14571512039832873,
      "learning_rate": 6.84938998785033e-07,
      "logits/chosen": -3.601046562194824,
      "logits/rejected": -3.6993136405944824,
      "logps/chosen": -0.42476823925971985,
      "logps/rejected": -100.576171875,
      "loss": 0.0656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7136902809143066,
      "rewards/margins": 2.7136902809143066,
      "rewards/rejected": 0.0,
      "step": 2242
    },
    {
      "epoch": 12.53072625698324,
      "grad_norm": 0.6447230839865972,
      "learning_rate": 6.828445237571183e-07,
      "logits/chosen": -3.4767494201660156,
      "logits/rejected": -3.350771427154541,
      "logps/chosen": -0.2454582303762436,
      "logps/rejected": -79.89645385742188,
      "loss": 0.1006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7064146995544434,
      "rewards/margins": 2.7064146995544434,
      "rewards/rejected": 0.0,
      "step": 2243
    },
    {
      "epoch": 12.53631284916201,
      "grad_norm": 0.07694003043609456,
      "learning_rate": 6.80752749345859e-07,
      "logits/chosen": -3.33797025680542,
      "logits/rejected": -3.2996411323547363,
      "logps/chosen": -7.517304420471191,
      "logps/rejected": -105.21052551269531,
      "loss": 0.1227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.777785062789917,
      "rewards/margins": 3.777785062789917,
      "rewards/rejected": 0.0,
      "step": 2244
    },
    {
      "epoch": 12.541899441340782,
      "grad_norm": 0.3304602650644061,
      "learning_rate": 6.786636786600037e-07,
      "logits/chosen": -3.4993131160736084,
      "logits/rejected": -3.4683797359466553,
      "logps/chosen": -0.2916330099105835,
      "logps/rejected": -57.35548782348633,
      "loss": 0.1357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7258338928222656,
      "rewards/margins": 2.7258338928222656,
      "rewards/rejected": 0.0,
      "step": 2245
    },
    {
      "epoch": 12.547486033519553,
      "grad_norm": 0.08996573670428476,
      "learning_rate": 6.765773148042859e-07,
      "logits/chosen": -3.2920916080474854,
      "logits/rejected": -3.571930408477783,
      "logps/chosen": -0.2624906897544861,
      "logps/rejected": -52.964439392089844,
      "loss": 0.1034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.814146041870117,
      "rewards/margins": 2.814146041870117,
      "rewards/rejected": 0.0,
      "step": 2246
    },
    {
      "epoch": 12.553072625698324,
      "grad_norm": 0.11259552078390303,
      "learning_rate": 6.744936608794103e-07,
      "logits/chosen": -3.66802978515625,
      "logits/rejected": -3.6609458923339844,
      "logps/chosen": -0.5823183059692383,
      "logps/rejected": -99.8145523071289,
      "loss": 0.1061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.917518138885498,
      "rewards/margins": 2.917518138885498,
      "rewards/rejected": 0.0,
      "step": 2247
    },
    {
      "epoch": 12.558659217877095,
      "grad_norm": 0.22427424923714798,
      "learning_rate": 6.724127199820605e-07,
      "logits/chosen": -3.579449415206909,
      "logits/rejected": -3.5708861351013184,
      "logps/chosen": -7.163443088531494,
      "logps/rejected": -91.9281005859375,
      "loss": 0.093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.635258197784424,
      "rewards/margins": 4.635258197784424,
      "rewards/rejected": 0.0,
      "step": 2248
    },
    {
      "epoch": 12.564245810055866,
      "grad_norm": 2.043281734435734,
      "learning_rate": 6.70334495204884e-07,
      "logits/chosen": -3.65213680267334,
      "logits/rejected": -3.6763410568237305,
      "logps/chosen": -0.8135756254196167,
      "logps/rejected": -64.81053161621094,
      "loss": 0.0907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3758888244628906,
      "rewards/margins": 3.3758888244628906,
      "rewards/rejected": 0.0,
      "step": 2249
    },
    {
      "epoch": 12.569832402234637,
      "grad_norm": 0.9500407804276312,
      "learning_rate": 6.682589896364935e-07,
      "logits/chosen": -3.7711451053619385,
      "logits/rejected": -3.6182680130004883,
      "logps/chosen": -0.37147852778434753,
      "logps/rejected": -58.02679443359375,
      "loss": 0.0907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.968472957611084,
      "rewards/margins": 2.968472957611084,
      "rewards/rejected": 0.0,
      "step": 2250
    },
    {
      "epoch": 12.575418994413408,
      "grad_norm": 0.1124833103401606,
      "learning_rate": 6.661862063614597e-07,
      "logits/chosen": -3.387005090713501,
      "logits/rejected": -3.4388318061828613,
      "logps/chosen": -0.06950801610946655,
      "logps/rejected": -103.17023468017578,
      "loss": 0.0881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.711268424987793,
      "rewards/margins": 1.711268424987793,
      "rewards/rejected": 0.0,
      "step": 2251
    },
    {
      "epoch": 12.581005586592179,
      "grad_norm": 0.11865824893892338,
      "learning_rate": 6.641161484603073e-07,
      "logits/chosen": -3.0767784118652344,
      "logits/rejected": -2.8243749141693115,
      "logps/chosen": -12.574045181274414,
      "logps/rejected": -73.87748718261719,
      "loss": 0.1108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.878243923187256,
      "rewards/margins": 3.878243923187256,
      "rewards/rejected": 0.0,
      "step": 2252
    },
    {
      "epoch": 12.58659217877095,
      "grad_norm": 0.08965384325592246,
      "learning_rate": 6.620488190095115e-07,
      "logits/chosen": -3.877704620361328,
      "logits/rejected": -3.9646201133728027,
      "logps/chosen": -0.6290386915206909,
      "logps/rejected": -44.720916748046875,
      "loss": 0.0919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.887430429458618,
      "rewards/margins": 2.887430429458618,
      "rewards/rejected": 0.0,
      "step": 2253
    },
    {
      "epoch": 12.59217877094972,
      "grad_norm": 0.1829759342890795,
      "learning_rate": 6.599842210814916e-07,
      "logits/chosen": -3.5952391624450684,
      "logits/rejected": -3.6253550052642822,
      "logps/chosen": -4.890213966369629,
      "logps/rejected": -82.6927490234375,
      "loss": 0.1308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6505279541015625,
      "rewards/margins": 2.6505279541015625,
      "rewards/rejected": 0.0,
      "step": 2254
    },
    {
      "epoch": 12.597765363128492,
      "grad_norm": 0.0913656864068451,
      "learning_rate": 6.579223577446084e-07,
      "logits/chosen": -3.220032215118408,
      "logits/rejected": -3.4098033905029297,
      "logps/chosen": -3.000941038131714,
      "logps/rejected": -84.70130920410156,
      "loss": 0.087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.794586181640625,
      "rewards/margins": 3.794586181640625,
      "rewards/rejected": 0.0,
      "step": 2255
    },
    {
      "epoch": 12.603351955307263,
      "grad_norm": 1.0186798106249813,
      "learning_rate": 6.55863232063157e-07,
      "logits/chosen": -3.6517138481140137,
      "logits/rejected": -3.5198822021484375,
      "logps/chosen": -10.387565612792969,
      "logps/rejected": -55.11285400390625,
      "loss": 0.0979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9465489387512207,
      "rewards/margins": 3.9465489387512207,
      "rewards/rejected": 0.0,
      "step": 2256
    },
    {
      "epoch": 12.608938547486034,
      "grad_norm": 0.26016749129894035,
      "learning_rate": 6.538068470973666e-07,
      "logits/chosen": -3.3727545738220215,
      "logits/rejected": -3.560612201690674,
      "logps/chosen": -0.7755427956581116,
      "logps/rejected": -135.69158935546875,
      "loss": 0.1117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2633721828460693,
      "rewards/margins": 3.2633721828460693,
      "rewards/rejected": 0.0,
      "step": 2257
    },
    {
      "epoch": 12.614525139664805,
      "grad_norm": 0.19670905900436392,
      "learning_rate": 6.517532059033894e-07,
      "logits/chosen": -3.3698601722717285,
      "logits/rejected": -3.2186710834503174,
      "logps/chosen": -5.761754512786865,
      "logps/rejected": -46.94666290283203,
      "loss": 0.0962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.884753227233887,
      "rewards/margins": 4.884753227233887,
      "rewards/rejected": 0.0,
      "step": 2258
    },
    {
      "epoch": 12.620111731843576,
      "grad_norm": 1.211017461257243,
      "learning_rate": 6.497023115333034e-07,
      "logits/chosen": -3.498352527618408,
      "logits/rejected": -3.405221700668335,
      "logps/chosen": -0.5089658498764038,
      "logps/rejected": -76.45728302001953,
      "loss": 0.0811,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5289535522460938,
      "rewards/margins": 2.5289535522460938,
      "rewards/rejected": 0.0,
      "step": 2259
    },
    {
      "epoch": 12.625698324022347,
      "grad_norm": 0.43365306647870944,
      "learning_rate": 6.476541670351025e-07,
      "logits/chosen": -3.3338077068328857,
      "logits/rejected": -3.53731369972229,
      "logps/chosen": -0.07289311289787292,
      "logps/rejected": -131.89056396484375,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1146087646484375,
      "rewards/margins": 2.1146087646484375,
      "rewards/rejected": 0.0,
      "step": 2260
    },
    {
      "epoch": 12.625698324022347,
      "eval_logits/chosen": -3.354635238647461,
      "eval_logits/rejected": -3.4829201698303223,
      "eval_logps/chosen": -45.23137664794922,
      "eval_logps/rejected": -71.89067840576172,
      "eval_loss": 2.2707626819610596,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.5161330699920654,
      "eval_rewards/margins": -1.5161330699920654,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.4775,
      "eval_samples_per_second": 12.168,
      "eval_steps_per_second": 0.393,
      "step": 2260
    },
    {
      "epoch": 12.631284916201118,
      "grad_norm": 0.26663382203763225,
      "learning_rate": 6.456087754526941e-07,
      "logits/chosen": -3.223412036895752,
      "logits/rejected": -3.341928005218506,
      "logps/chosen": -13.103678703308105,
      "logps/rejected": -80.57732391357422,
      "loss": 0.0838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5308234691619873,
      "rewards/margins": 3.5308234691619873,
      "rewards/rejected": 0.0,
      "step": 2261
    },
    {
      "epoch": 12.636871508379889,
      "grad_norm": 0.259588185588646,
      "learning_rate": 6.435661398258943e-07,
      "logits/chosen": -3.6073262691497803,
      "logits/rejected": -3.7826313972473145,
      "logps/chosen": -0.3340875506401062,
      "logps/rejected": -103.56145477294922,
      "loss": 0.0845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.415807008743286,
      "rewards/margins": 2.415807008743286,
      "rewards/rejected": 0.0,
      "step": 2262
    },
    {
      "epoch": 12.64245810055866,
      "grad_norm": 0.3234960002417289,
      "learning_rate": 6.41526263190424e-07,
      "logits/chosen": -3.7524898052215576,
      "logits/rejected": -3.5782220363616943,
      "logps/chosen": -0.14454372227191925,
      "logps/rejected": -123.64569091796875,
      "loss": 0.1399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4265780448913574,
      "rewards/margins": 2.4265780448913574,
      "rewards/rejected": 0.0,
      "step": 2263
    },
    {
      "epoch": 12.64804469273743,
      "grad_norm": 0.13690546085427355,
      "learning_rate": 6.394891485779023e-07,
      "logits/chosen": -3.676905870437622,
      "logits/rejected": -3.5705578327178955,
      "logps/chosen": -2.6452982425689697,
      "logps/rejected": -99.85325622558594,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4046313762664795,
      "rewards/margins": 3.4046313762664795,
      "rewards/rejected": 0.0,
      "step": 2264
    },
    {
      "epoch": 12.653631284916202,
      "grad_norm": 0.09231661712723066,
      "learning_rate": 6.374547990158439e-07,
      "logits/chosen": -3.319514036178589,
      "logits/rejected": -3.1935408115386963,
      "logps/chosen": -2.170340061187744,
      "logps/rejected": -92.83390808105469,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.919471263885498,
      "rewards/margins": 2.919471263885498,
      "rewards/rejected": 0.0,
      "step": 2265
    },
    {
      "epoch": 12.659217877094973,
      "grad_norm": 0.1305607481727272,
      "learning_rate": 6.354232175276568e-07,
      "logits/chosen": -3.318600654602051,
      "logits/rejected": -3.347093105316162,
      "logps/chosen": -1.7944437265396118,
      "logps/rejected": -49.953086853027344,
      "loss": 0.092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.507636070251465,
      "rewards/margins": 3.507636070251465,
      "rewards/rejected": 0.0,
      "step": 2266
    },
    {
      "epoch": 12.664804469273744,
      "grad_norm": 0.08257322985718621,
      "learning_rate": 6.333944071326301e-07,
      "logits/chosen": -3.2576847076416016,
      "logits/rejected": -3.2436788082122803,
      "logps/chosen": -0.0729764923453331,
      "logps/rejected": -83.48963928222656,
      "loss": 0.0905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.043816566467285,
      "rewards/margins": 2.043816566467285,
      "rewards/rejected": 0.0,
      "step": 2267
    },
    {
      "epoch": 12.670391061452515,
      "grad_norm": 0.5595450946458308,
      "learning_rate": 6.313683708459393e-07,
      "logits/chosen": -3.70135760307312,
      "logits/rejected": -3.549161195755005,
      "logps/chosen": -0.30069923400878906,
      "logps/rejected": -80.69763946533203,
      "loss": 0.0932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.810758113861084,
      "rewards/margins": 2.810758113861084,
      "rewards/rejected": 0.0,
      "step": 2268
    },
    {
      "epoch": 12.675977653631286,
      "grad_norm": 0.08479624148928418,
      "learning_rate": 6.293451116786344e-07,
      "logits/chosen": -3.4262752532958984,
      "logits/rejected": -3.4353890419006348,
      "logps/chosen": -0.4452231228351593,
      "logps/rejected": -68.05882263183594,
      "loss": 0.0754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.915862798690796,
      "rewards/margins": 2.915862798690796,
      "rewards/rejected": 0.0,
      "step": 2269
    },
    {
      "epoch": 12.681564245810057,
      "grad_norm": 0.35002442892800506,
      "learning_rate": 6.273246326376392e-07,
      "logits/chosen": -3.4398837089538574,
      "logits/rejected": -3.3725829124450684,
      "logps/chosen": -0.09837726503610611,
      "logps/rejected": -77.57417297363281,
      "loss": 0.1241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.994058609008789,
      "rewards/margins": 1.994058609008789,
      "rewards/rejected": 0.0,
      "step": 2270
    },
    {
      "epoch": 12.687150837988828,
      "grad_norm": 0.3587371149491114,
      "learning_rate": 6.25306936725745e-07,
      "logits/chosen": -3.3311831951141357,
      "logits/rejected": -3.389683961868286,
      "logps/chosen": -4.5876569747924805,
      "logps/rejected": -75.933349609375,
      "loss": 0.0904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9711647033691406,
      "rewards/margins": 3.9711647033691406,
      "rewards/rejected": 0.0,
      "step": 2271
    },
    {
      "epoch": 12.692737430167599,
      "grad_norm": 0.08345393893737656,
      "learning_rate": 6.232920269416074e-07,
      "logits/chosen": -3.4720520973205566,
      "logits/rejected": -3.6347930431365967,
      "logps/chosen": -4.277005672454834,
      "logps/rejected": -67.94229125976562,
      "loss": 0.1257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.364170551300049,
      "rewards/margins": 3.364170551300049,
      "rewards/rejected": 0.0,
      "step": 2272
    },
    {
      "epoch": 12.69832402234637,
      "grad_norm": 0.19719391038913955,
      "learning_rate": 6.212799062797411e-07,
      "logits/chosen": -3.6341161727905273,
      "logits/rejected": -3.438502788543701,
      "logps/chosen": -0.5755277872085571,
      "logps/rejected": -59.11248016357422,
      "loss": 0.1135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7914340496063232,
      "rewards/margins": 2.7914340496063232,
      "rewards/rejected": 0.0,
      "step": 2273
    },
    {
      "epoch": 12.703910614525139,
      "grad_norm": 0.7973053754344259,
      "learning_rate": 6.192705777305152e-07,
      "logits/chosen": -3.4621007442474365,
      "logits/rejected": -3.5072593688964844,
      "logps/chosen": -2.432701587677002,
      "logps/rejected": -67.162353515625,
      "loss": 0.0732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9680051803588867,
      "rewards/margins": 2.9680051803588867,
      "rewards/rejected": 0.0,
      "step": 2274
    },
    {
      "epoch": 12.70949720670391,
      "grad_norm": 0.1733209698250686,
      "learning_rate": 6.172640442801519e-07,
      "logits/chosen": -3.58695387840271,
      "logits/rejected": -3.5564804077148438,
      "logps/chosen": -0.2993914783000946,
      "logps/rejected": -52.34805679321289,
      "loss": 0.1036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.221712589263916,
      "rewards/margins": 2.221712589263916,
      "rewards/rejected": 0.0,
      "step": 2275
    },
    {
      "epoch": 12.71508379888268,
      "grad_norm": 0.07532048985781706,
      "learning_rate": 6.15260308910714e-07,
      "logits/chosen": -3.577807903289795,
      "logits/rejected": -3.557210683822632,
      "logps/chosen": -0.15123456716537476,
      "logps/rejected": -50.36665344238281,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4327011108398438,
      "rewards/margins": 2.4327011108398438,
      "rewards/rejected": 0.0,
      "step": 2276
    },
    {
      "epoch": 12.720670391061452,
      "grad_norm": 0.2723678811246857,
      "learning_rate": 6.13259374600112e-07,
      "logits/chosen": -3.5621848106384277,
      "logits/rejected": -3.3281946182250977,
      "logps/chosen": -0.13655583560466766,
      "logps/rejected": -66.5269546508789,
      "loss": 0.1301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2382802963256836,
      "rewards/margins": 2.2382802963256836,
      "rewards/rejected": 0.0,
      "step": 2277
    },
    {
      "epoch": 12.726256983240223,
      "grad_norm": 0.13871672907303387,
      "learning_rate": 6.112612443220891e-07,
      "logits/chosen": -3.6357340812683105,
      "logits/rejected": -3.6592915058135986,
      "logps/chosen": -4.012609481811523,
      "logps/rejected": -45.59083557128906,
      "loss": 0.0866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9316704273223877,
      "rewards/margins": 3.9316704273223877,
      "rewards/rejected": 0.0,
      "step": 2278
    },
    {
      "epoch": 12.731843575418994,
      "grad_norm": 0.2117164155780098,
      "learning_rate": 6.092659210462232e-07,
      "logits/chosen": -3.3808088302612305,
      "logits/rejected": -3.385256767272949,
      "logps/chosen": -0.15518206357955933,
      "logps/rejected": -97.79454040527344,
      "loss": 0.1083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1025004386901855,
      "rewards/margins": 2.1025004386901855,
      "rewards/rejected": 0.0,
      "step": 2279
    },
    {
      "epoch": 12.737430167597765,
      "grad_norm": 0.10416979640926079,
      "learning_rate": 6.072734077379199e-07,
      "logits/chosen": -3.432586669921875,
      "logits/rejected": -3.4277217388153076,
      "logps/chosen": -1.177551031112671,
      "logps/rejected": -59.490386962890625,
      "loss": 0.0805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.914443254470825,
      "rewards/margins": 2.914443254470825,
      "rewards/rejected": 0.0,
      "step": 2280
    },
    {
      "epoch": 12.737430167597765,
      "eval_logits/chosen": -3.3496944904327393,
      "eval_logits/rejected": -3.4795279502868652,
      "eval_logps/chosen": -45.6035270690918,
      "eval_logps/rejected": -71.77366638183594,
      "eval_loss": 2.30177640914917,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.5533487796783447,
      "eval_rewards/margins": -1.5533487796783447,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6039,
      "eval_samples_per_second": 12.108,
      "eval_steps_per_second": 0.391,
      "step": 2280
    },
    {
      "epoch": 12.743016759776536,
      "grad_norm": 0.09893715129615782,
      "learning_rate": 6.052837073584089e-07,
      "logits/chosen": -3.5229735374450684,
      "logits/rejected": -3.560753107070923,
      "logps/chosen": -1.7236528396606445,
      "logps/rejected": -61.791603088378906,
      "loss": 0.0903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0803747177124023,
      "rewards/margins": 3.0803747177124023,
      "rewards/rejected": 0.0,
      "step": 2281
    },
    {
      "epoch": 12.748603351955307,
      "grad_norm": 0.10239738621163685,
      "learning_rate": 6.032968228647392e-07,
      "logits/chosen": -3.8157575130462646,
      "logits/rejected": -3.675696849822998,
      "logps/chosen": -2.667703151702881,
      "logps/rejected": -56.6728515625,
      "loss": 0.0781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3268051147460938,
      "rewards/margins": 3.3268051147460938,
      "rewards/rejected": 0.0,
      "step": 2282
    },
    {
      "epoch": 12.754189944134078,
      "grad_norm": 0.09223104810857743,
      "learning_rate": 6.013127572097754e-07,
      "logits/chosen": -3.7003021240234375,
      "logits/rejected": -3.7115604877471924,
      "logps/chosen": -0.8604039549827576,
      "logps/rejected": -80.38536071777344,
      "loss": 0.0862,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4727659225463867,
      "rewards/margins": 3.4727659225463867,
      "rewards/rejected": 0.0,
      "step": 2283
    },
    {
      "epoch": 12.759776536312849,
      "grad_norm": 0.18216931324537386,
      "learning_rate": 5.993315133421918e-07,
      "logits/chosen": -3.5860238075256348,
      "logits/rejected": -3.642956018447876,
      "logps/chosen": -0.492598295211792,
      "logps/rejected": -70.66374206542969,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1724660396575928,
      "rewards/margins": 3.1724660396575928,
      "rewards/rejected": 0.0,
      "step": 2284
    },
    {
      "epoch": 12.76536312849162,
      "grad_norm": 0.16225916089110387,
      "learning_rate": 5.973530942064692e-07,
      "logits/chosen": -3.6503846645355225,
      "logits/rejected": -3.7499403953552246,
      "logps/chosen": -0.08344462513923645,
      "logps/rejected": -98.05110168457031,
      "loss": 0.0949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.024770498275757,
      "rewards/margins": 2.024770498275757,
      "rewards/rejected": 0.0,
      "step": 2285
    },
    {
      "epoch": 12.77094972067039,
      "grad_norm": 0.12086775369123424,
      "learning_rate": 5.953775027428932e-07,
      "logits/chosen": -3.546950101852417,
      "logits/rejected": -3.5305943489074707,
      "logps/chosen": -6.928635597229004,
      "logps/rejected": -51.393096923828125,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.330551624298096,
      "rewards/margins": 4.330551624298096,
      "rewards/rejected": 0.0,
      "step": 2286
    },
    {
      "epoch": 12.776536312849162,
      "grad_norm": 0.09366695001979931,
      "learning_rate": 5.934047418875411e-07,
      "logits/chosen": -3.353851556777954,
      "logits/rejected": -3.443031072616577,
      "logps/chosen": -2.202850341796875,
      "logps/rejected": -76.08257293701172,
      "loss": 0.101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9218358993530273,
      "rewards/margins": 3.9218358993530273,
      "rewards/rejected": 0.0,
      "step": 2287
    },
    {
      "epoch": 12.782122905027933,
      "grad_norm": 0.11862954003341247,
      "learning_rate": 5.914348145722895e-07,
      "logits/chosen": -3.328933000564575,
      "logits/rejected": -3.2731192111968994,
      "logps/chosen": -8.417642593383789,
      "logps/rejected": -78.35466003417969,
      "loss": 0.0831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.356705665588379,
      "rewards/margins": 3.356705665588379,
      "rewards/rejected": 0.0,
      "step": 2288
    },
    {
      "epoch": 12.787709497206704,
      "grad_norm": 0.14490400896769573,
      "learning_rate": 5.894677237248e-07,
      "logits/chosen": -3.7178573608398438,
      "logits/rejected": -3.6568291187286377,
      "logps/chosen": -0.3006286919116974,
      "logps/rejected": -56.25794982910156,
      "loss": 0.1185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4759769439697266,
      "rewards/margins": 2.4759769439697266,
      "rewards/rejected": 0.0,
      "step": 2289
    },
    {
      "epoch": 12.793296089385475,
      "grad_norm": 0.09682377283155488,
      "learning_rate": 5.875034722685205e-07,
      "logits/chosen": -3.709879159927368,
      "logits/rejected": -3.6451339721679688,
      "logps/chosen": -4.253514289855957,
      "logps/rejected": -67.79463195800781,
      "loss": 0.1115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.308173179626465,
      "rewards/margins": 4.308173179626465,
      "rewards/rejected": 0.0,
      "step": 2290
    },
    {
      "epoch": 12.798882681564246,
      "grad_norm": 0.3433845899406549,
      "learning_rate": 5.855420631226783e-07,
      "logits/chosen": -3.757261037826538,
      "logits/rejected": -3.6908226013183594,
      "logps/chosen": -0.36719846725463867,
      "logps/rejected": -57.598045349121094,
      "loss": 0.0949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.807281255722046,
      "rewards/margins": 2.807281255722046,
      "rewards/rejected": 0.0,
      "step": 2291
    },
    {
      "epoch": 12.804469273743017,
      "grad_norm": 0.13255121933153183,
      "learning_rate": 5.835834992022765e-07,
      "logits/chosen": -3.5118043422698975,
      "logits/rejected": -3.552795648574829,
      "logps/chosen": -0.10286284983158112,
      "logps/rejected": -108.36598205566406,
      "loss": 0.1192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9102492332458496,
      "rewards/margins": 1.9102492332458496,
      "rewards/rejected": 0.0,
      "step": 2292
    },
    {
      "epoch": 12.810055865921788,
      "grad_norm": 0.1006322286190258,
      "learning_rate": 5.8162778341809e-07,
      "logits/chosen": -3.401216983795166,
      "logits/rejected": -3.6072990894317627,
      "logps/chosen": -0.09119925647974014,
      "logps/rejected": -139.12973022460938,
      "loss": 0.1014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.090881824493408,
      "rewards/margins": 2.090881824493408,
      "rewards/rejected": 0.0,
      "step": 2293
    },
    {
      "epoch": 12.815642458100559,
      "grad_norm": 0.0932232510689449,
      "learning_rate": 5.796749186766598e-07,
      "logits/chosen": -3.47281551361084,
      "logits/rejected": -3.561671018600464,
      "logps/chosen": -1.5021058320999146,
      "logps/rejected": -90.13780212402344,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.235504627227783,
      "rewards/margins": 4.235504627227783,
      "rewards/rejected": 0.0,
      "step": 2294
    },
    {
      "epoch": 12.82122905027933,
      "grad_norm": 0.07402175591780737,
      "learning_rate": 5.777249078802932e-07,
      "logits/chosen": -3.55774188041687,
      "logits/rejected": -3.441333532333374,
      "logps/chosen": -0.20372900366783142,
      "logps/rejected": -55.722862243652344,
      "loss": 0.1016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.784947156906128,
      "rewards/margins": 2.784947156906128,
      "rewards/rejected": 0.0,
      "step": 2295
    },
    {
      "epoch": 12.8268156424581,
      "grad_norm": 0.11588461410874612,
      "learning_rate": 5.757777539270506e-07,
      "logits/chosen": -3.5666894912719727,
      "logits/rejected": -3.6083571910858154,
      "logps/chosen": -1.6700191497802734,
      "logps/rejected": -63.93604278564453,
      "loss": 0.0855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.706993341445923,
      "rewards/margins": 2.706993341445923,
      "rewards/rejected": 0.0,
      "step": 2296
    },
    {
      "epoch": 12.832402234636872,
      "grad_norm": 0.16025587061633367,
      "learning_rate": 5.738334597107517e-07,
      "logits/chosen": -3.340813159942627,
      "logits/rejected": -3.156062364578247,
      "logps/chosen": -0.33533942699432373,
      "logps/rejected": -75.6090087890625,
      "loss": 0.0779,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.855161190032959,
      "rewards/margins": 2.855161190032959,
      "rewards/rejected": 0.0,
      "step": 2297
    },
    {
      "epoch": 12.837988826815643,
      "grad_norm": 0.20929697325487984,
      "learning_rate": 5.71892028120963e-07,
      "logits/chosen": -3.371018648147583,
      "logits/rejected": -3.465022325515747,
      "logps/chosen": -0.20324113965034485,
      "logps/rejected": -65.3719253540039,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.674511671066284,
      "rewards/margins": 2.674511671066284,
      "rewards/rejected": 0.0,
      "step": 2298
    },
    {
      "epoch": 12.843575418994414,
      "grad_norm": 0.13671680953669826,
      "learning_rate": 5.699534620429978e-07,
      "logits/chosen": -3.396329402923584,
      "logits/rejected": -3.3214175701141357,
      "logps/chosen": -0.6440017223358154,
      "logps/rejected": -59.99646759033203,
      "loss": 0.0779,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1435632705688477,
      "rewards/margins": 3.1435632705688477,
      "rewards/rejected": 0.0,
      "step": 2299
    },
    {
      "epoch": 12.849162011173185,
      "grad_norm": 0.17840043858380186,
      "learning_rate": 5.680177643579108e-07,
      "logits/chosen": -3.3425371646881104,
      "logits/rejected": -3.4421560764312744,
      "logps/chosen": -0.8651880025863647,
      "logps/rejected": -67.25711059570312,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.444180488586426,
      "rewards/margins": 3.444180488586426,
      "rewards/rejected": 0.0,
      "step": 2300
    },
    {
      "epoch": 12.849162011173185,
      "eval_logits/chosen": -3.3559410572052,
      "eval_logits/rejected": -3.483773708343506,
      "eval_logps/chosen": -45.552433013916016,
      "eval_logps/rejected": -71.8984146118164,
      "eval_loss": 2.319016695022583,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.5482391119003296,
      "eval_rewards/margins": -1.5482391119003296,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6176,
      "eval_samples_per_second": 12.101,
      "eval_steps_per_second": 0.39,
      "step": 2300
    },
    {
      "epoch": 12.854748603351956,
      "grad_norm": 0.1433197861383478,
      "learning_rate": 5.660849379424927e-07,
      "logits/chosen": -3.5154871940612793,
      "logits/rejected": -3.5883731842041016,
      "logps/chosen": -0.4321754574775696,
      "logps/rejected": -115.58190155029297,
      "loss": 0.1235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.824122428894043,
      "rewards/margins": 2.824122428894043,
      "rewards/rejected": 0.0,
      "step": 2301
    },
    {
      "epoch": 12.860335195530727,
      "grad_norm": 0.08978923753700167,
      "learning_rate": 5.641549856692685e-07,
      "logits/chosen": -3.5351314544677734,
      "logits/rejected": -3.7583625316619873,
      "logps/chosen": -0.04258780926465988,
      "logps/rejected": -130.0748291015625,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5902153253555298,
      "rewards/margins": 1.5902153253555298,
      "rewards/rejected": 0.0,
      "step": 2302
    },
    {
      "epoch": 12.865921787709498,
      "grad_norm": 0.4300029313126411,
      "learning_rate": 5.622279104064899e-07,
      "logits/chosen": -3.4135560989379883,
      "logits/rejected": -3.411367893218994,
      "logps/chosen": -0.07878328859806061,
      "logps/rejected": -62.26985549926758,
      "loss": 0.1322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7675995826721191,
      "rewards/margins": 1.7675995826721191,
      "rewards/rejected": 0.0,
      "step": 2303
    },
    {
      "epoch": 12.871508379888269,
      "grad_norm": 0.08579227083094132,
      "learning_rate": 5.603037150181362e-07,
      "logits/chosen": -3.603668689727783,
      "logits/rejected": -3.7250325679779053,
      "logps/chosen": -0.07879330962896347,
      "logps/rejected": -71.96653747558594,
      "loss": 0.0895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.263234853744507,
      "rewards/margins": 2.263234853744507,
      "rewards/rejected": 0.0,
      "step": 2304
    },
    {
      "epoch": 12.87709497206704,
      "grad_norm": 0.09948536000589427,
      "learning_rate": 5.583824023639017e-07,
      "logits/chosen": -3.4752449989318848,
      "logits/rejected": -3.523326873779297,
      "logps/chosen": -0.14934679865837097,
      "logps/rejected": -51.6447868347168,
      "loss": 0.074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5092530250549316,
      "rewards/margins": 2.5092530250549316,
      "rewards/rejected": 0.0,
      "step": 2305
    },
    {
      "epoch": 12.88268156424581,
      "grad_norm": 0.5687888508104226,
      "learning_rate": 5.564639752992019e-07,
      "logits/chosen": -3.5994062423706055,
      "logits/rejected": -3.5353009700775146,
      "logps/chosen": -1.0714080333709717,
      "logps/rejected": -57.62873840332031,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1900219917297363,
      "rewards/margins": 2.1900219917297363,
      "rewards/rejected": 0.0,
      "step": 2306
    },
    {
      "epoch": 12.888268156424582,
      "grad_norm": 0.10287546275440926,
      "learning_rate": 5.54548436675158e-07,
      "logits/chosen": -3.4563345909118652,
      "logits/rejected": -3.6399221420288086,
      "logps/chosen": -0.114817775785923,
      "logps/rejected": -103.6529769897461,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9067481756210327,
      "rewards/margins": 1.9067481756210327,
      "rewards/rejected": 0.0,
      "step": 2307
    },
    {
      "epoch": 12.893854748603353,
      "grad_norm": 0.14545941091982328,
      "learning_rate": 5.526357893386039e-07,
      "logits/chosen": -3.442643880844116,
      "logits/rejected": -3.5517473220825195,
      "logps/chosen": -0.20380650460720062,
      "logps/rejected": -74.34370422363281,
      "loss": 0.1243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.134744644165039,
      "rewards/margins": 2.134744644165039,
      "rewards/rejected": 0.0,
      "step": 2308
    },
    {
      "epoch": 12.899441340782122,
      "grad_norm": 0.15675564754631255,
      "learning_rate": 5.507260361320738e-07,
      "logits/chosen": -3.576706886291504,
      "logits/rejected": -3.314330577850342,
      "logps/chosen": -10.65918254852295,
      "logps/rejected": -38.50128173828125,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0466156005859375,
      "rewards/margins": 3.0466156005859375,
      "rewards/rejected": 0.0,
      "step": 2309
    },
    {
      "epoch": 12.905027932960895,
      "grad_norm": 0.11033339008379849,
      "learning_rate": 5.488191798938003e-07,
      "logits/chosen": -3.468125104904175,
      "logits/rejected": -3.5840485095977783,
      "logps/chosen": -0.6843056082725525,
      "logps/rejected": -69.72563171386719,
      "loss": 0.101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5341298580169678,
      "rewards/margins": 3.5341298580169678,
      "rewards/rejected": 0.0,
      "step": 2310
    },
    {
      "epoch": 12.910614525139664,
      "grad_norm": 0.19229427188135872,
      "learning_rate": 5.469152234577119e-07,
      "logits/chosen": -3.607957124710083,
      "logits/rejected": -3.5638513565063477,
      "logps/chosen": -0.7911775708198547,
      "logps/rejected": -77.24010467529297,
      "loss": 0.1151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6002297401428223,
      "rewards/margins": 2.6002297401428223,
      "rewards/rejected": 0.0,
      "step": 2311
    },
    {
      "epoch": 12.916201117318435,
      "grad_norm": 0.11502856991153093,
      "learning_rate": 5.45014169653427e-07,
      "logits/chosen": -3.6131365299224854,
      "logits/rejected": -3.6775052547454834,
      "logps/chosen": -0.17084035277366638,
      "logps/rejected": -75.61903381347656,
      "loss": 0.0899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3905701637268066,
      "rewards/margins": 2.3905701637268066,
      "rewards/rejected": 0.0,
      "step": 2312
    },
    {
      "epoch": 12.921787709497206,
      "grad_norm": 0.44846478352635416,
      "learning_rate": 5.431160213062494e-07,
      "logits/chosen": -3.6013808250427246,
      "logits/rejected": -3.606156826019287,
      "logps/chosen": -1.4459359645843506,
      "logps/rejected": -64.024169921875,
      "loss": 0.1054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2094199657440186,
      "rewards/margins": 3.2094199657440186,
      "rewards/rejected": 0.0,
      "step": 2313
    },
    {
      "epoch": 12.927374301675977,
      "grad_norm": 0.10564017972370411,
      "learning_rate": 5.412207812371658e-07,
      "logits/chosen": -3.4921560287475586,
      "logits/rejected": -3.5877530574798584,
      "logps/chosen": -1.4728573560714722,
      "logps/rejected": -68.94142150878906,
      "loss": 0.059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.266526222229004,
      "rewards/margins": 4.266526222229004,
      "rewards/rejected": 0.0,
      "step": 2314
    },
    {
      "epoch": 12.932960893854748,
      "grad_norm": 0.20579086784216336,
      "learning_rate": 5.393284522628422e-07,
      "logits/chosen": -3.35365629196167,
      "logits/rejected": -3.271205186843872,
      "logps/chosen": -29.111434936523438,
      "logps/rejected": -51.95122528076172,
      "loss": 0.0918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.799343109130859,
      "rewards/margins": 4.799343109130859,
      "rewards/rejected": 0.0,
      "step": 2315
    },
    {
      "epoch": 12.938547486033519,
      "grad_norm": 0.14399489902065973,
      "learning_rate": 5.374390371956137e-07,
      "logits/chosen": -3.2941269874572754,
      "logits/rejected": -3.419398307800293,
      "logps/chosen": -0.12628039717674255,
      "logps/rejected": -72.36720275878906,
      "loss": 0.1082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3898634910583496,
      "rewards/margins": 2.3898634910583496,
      "rewards/rejected": 0.0,
      "step": 2316
    },
    {
      "epoch": 12.94413407821229,
      "grad_norm": 0.08405257079450976,
      "learning_rate": 5.355525388434896e-07,
      "logits/chosen": -3.7584612369537354,
      "logits/rejected": -3.5421531200408936,
      "logps/chosen": -0.09059040993452072,
      "logps/rejected": -111.62838745117188,
      "loss": 0.0889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1599297523498535,
      "rewards/margins": 2.1599297523498535,
      "rewards/rejected": 0.0,
      "step": 2317
    },
    {
      "epoch": 12.949720670391061,
      "grad_norm": 0.444326039150901,
      "learning_rate": 5.336689600101417e-07,
      "logits/chosen": -3.6093132495880127,
      "logits/rejected": -3.6669435501098633,
      "logps/chosen": -0.14301827549934387,
      "logps/rejected": -80.34127044677734,
      "loss": 0.0721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.243166208267212,
      "rewards/margins": 2.243166208267212,
      "rewards/rejected": 0.0,
      "step": 2318
    },
    {
      "epoch": 12.955307262569832,
      "grad_norm": 1.1124369535984773,
      "learning_rate": 5.317883034949034e-07,
      "logits/chosen": -3.3219892978668213,
      "logits/rejected": -3.2389910221099854,
      "logps/chosen": -0.8911861181259155,
      "logps/rejected": -68.47648620605469,
      "loss": 0.0896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6829323768615723,
      "rewards/margins": 2.6829323768615723,
      "rewards/rejected": 0.0,
      "step": 2319
    },
    {
      "epoch": 12.960893854748603,
      "grad_norm": 0.1418125486369656,
      "learning_rate": 5.299105720927655e-07,
      "logits/chosen": -3.381338357925415,
      "logits/rejected": -3.4358766078948975,
      "logps/chosen": -0.2006550282239914,
      "logps/rejected": -95.4887466430664,
      "loss": 0.0945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.387957811355591,
      "rewards/margins": 2.387957811355591,
      "rewards/rejected": 0.0,
      "step": 2320
    },
    {
      "epoch": 12.960893854748603,
      "eval_logits/chosen": -3.35034441947937,
      "eval_logits/rejected": -3.47920298576355,
      "eval_logps/chosen": -46.04875183105469,
      "eval_logps/rejected": -72.26363372802734,
      "eval_loss": 2.3450510501861572,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.59787118434906,
      "eval_rewards/margins": -1.59787118434906,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5452,
      "eval_samples_per_second": 12.135,
      "eval_steps_per_second": 0.391,
      "step": 2320
    },
    {
      "epoch": 12.966480446927374,
      "grad_norm": 0.1316320215563325,
      "learning_rate": 5.280357685943713e-07,
      "logits/chosen": -3.3141796588897705,
      "logits/rejected": -3.4846088886260986,
      "logps/chosen": -0.07934596389532089,
      "logps/rejected": -65.36231994628906,
      "loss": 0.0821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1445956230163574,
      "rewards/margins": 2.1445956230163574,
      "rewards/rejected": 0.0,
      "step": 2321
    },
    {
      "epoch": 12.972067039106145,
      "grad_norm": 0.1515315657158575,
      "learning_rate": 5.261638957860126e-07,
      "logits/chosen": -3.5436294078826904,
      "logits/rejected": -3.49223256111145,
      "logps/chosen": -0.16079261898994446,
      "logps/rejected": -95.5890121459961,
      "loss": 0.1062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3046774864196777,
      "rewards/margins": 2.3046774864196777,
      "rewards/rejected": 0.0,
      "step": 2322
    },
    {
      "epoch": 12.977653631284916,
      "grad_norm": 0.09301781852194996,
      "learning_rate": 5.24294956449625e-07,
      "logits/chosen": -3.4357640743255615,
      "logits/rejected": -3.4852664470672607,
      "logps/chosen": -11.127570152282715,
      "logps/rejected": -64.85619354248047,
      "loss": 0.107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.158937931060791,
      "rewards/margins": 5.158937931060791,
      "rewards/rejected": 0.0,
      "step": 2323
    },
    {
      "epoch": 12.983240223463687,
      "grad_norm": 0.398070858923522,
      "learning_rate": 5.224289533627875e-07,
      "logits/chosen": -3.359501361846924,
      "logits/rejected": -3.3766496181488037,
      "logps/chosen": -1.575634479522705,
      "logps/rejected": -74.5666732788086,
      "loss": 0.0876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7265524864196777,
      "rewards/margins": 3.7265524864196777,
      "rewards/rejected": 0.0,
      "step": 2324
    },
    {
      "epoch": 12.988826815642458,
      "grad_norm": 0.3352244597178905,
      "learning_rate": 5.205658892987097e-07,
      "logits/chosen": -3.684081792831421,
      "logits/rejected": -3.6866772174835205,
      "logps/chosen": -0.4886262118816376,
      "logps/rejected": -66.82278442382812,
      "loss": 0.0865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1717617511749268,
      "rewards/margins": 3.1717617511749268,
      "rewards/rejected": 0.0,
      "step": 2325
    },
    {
      "epoch": 12.994413407821229,
      "grad_norm": 0.07248852484851338,
      "learning_rate": 5.187057670262393e-07,
      "logits/chosen": -3.363440752029419,
      "logits/rejected": -3.532114028930664,
      "logps/chosen": -0.5741982460021973,
      "logps/rejected": -56.25327682495117,
      "loss": 0.0693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3493638038635254,
      "rewards/margins": 3.3493638038635254,
      "rewards/rejected": 0.0,
      "step": 2326
    },
    {
      "epoch": 13.0,
      "grad_norm": 0.06976610020455822,
      "learning_rate": 5.168485893098482e-07,
      "logits/chosen": -3.6658077239990234,
      "logits/rejected": -3.647418260574341,
      "logps/chosen": -0.05462373048067093,
      "logps/rejected": -111.98223876953125,
      "loss": 0.1065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6676750183105469,
      "rewards/margins": 1.6676750183105469,
      "rewards/rejected": 0.0,
      "step": 2327
    },
    {
      "epoch": 13.005586592178771,
      "grad_norm": 0.6418333659990051,
      "learning_rate": 5.149943589096335e-07,
      "logits/chosen": -3.7365448474884033,
      "logits/rejected": -3.814751625061035,
      "logps/chosen": -0.6457440257072449,
      "logps/rejected": -168.28858947753906,
      "loss": 0.1392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.137490749359131,
      "rewards/margins": 2.137490749359131,
      "rewards/rejected": 0.0,
      "step": 2328
    },
    {
      "epoch": 13.011173184357542,
      "grad_norm": 0.19025364239306858,
      "learning_rate": 5.131430785813116e-07,
      "logits/chosen": -3.2519400119781494,
      "logits/rejected": -3.376729965209961,
      "logps/chosen": -0.37146902084350586,
      "logps/rejected": -76.24604797363281,
      "loss": 0.0781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.845217704772949,
      "rewards/margins": 2.845217704772949,
      "rewards/rejected": 0.0,
      "step": 2329
    },
    {
      "epoch": 13.016759776536313,
      "grad_norm": 0.07454833232147721,
      "learning_rate": 5.112947510762145e-07,
      "logits/chosen": -3.6577749252319336,
      "logits/rejected": -3.4596827030181885,
      "logps/chosen": -0.4111977517604828,
      "logps/rejected": -65.58619689941406,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.228107452392578,
      "rewards/margins": 3.228107452392578,
      "rewards/rejected": 0.0,
      "step": 2330
    },
    {
      "epoch": 13.022346368715084,
      "grad_norm": 0.17617080505660834,
      "learning_rate": 5.094493791412866e-07,
      "logits/chosen": -3.761929988861084,
      "logits/rejected": -3.701007843017578,
      "logps/chosen": -0.1018652617931366,
      "logps/rejected": -66.26068115234375,
      "loss": 0.1096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.136481761932373,
      "rewards/margins": 2.136481761932373,
      "rewards/rejected": 0.0,
      "step": 2331
    },
    {
      "epoch": 13.027932960893855,
      "grad_norm": 0.06052746299125067,
      "learning_rate": 5.076069655190782e-07,
      "logits/chosen": -3.1227221488952637,
      "logits/rejected": -3.206955909729004,
      "logps/chosen": -11.33049201965332,
      "logps/rejected": -72.9752426147461,
      "loss": 0.0909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.183177947998047,
      "rewards/margins": 4.183177947998047,
      "rewards/rejected": 0.0,
      "step": 2332
    },
    {
      "epoch": 13.033519553072626,
      "grad_norm": 0.08056759796474999,
      "learning_rate": 5.057675129477463e-07,
      "logits/chosen": -3.5049266815185547,
      "logits/rejected": -3.6136879920959473,
      "logps/chosen": -2.038541316986084,
      "logps/rejected": -74.24696350097656,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8326029777526855,
      "rewards/margins": 3.8326029777526855,
      "rewards/rejected": 0.0,
      "step": 2333
    },
    {
      "epoch": 13.039106145251397,
      "grad_norm": 0.20785711234600926,
      "learning_rate": 5.039310241610421e-07,
      "logits/chosen": -3.3584213256835938,
      "logits/rejected": -3.4080893993377686,
      "logps/chosen": -0.08545253425836563,
      "logps/rejected": -68.42716979980469,
      "loss": 0.0974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9042706489562988,
      "rewards/margins": 1.9042706489562988,
      "rewards/rejected": 0.0,
      "step": 2334
    },
    {
      "epoch": 13.044692737430168,
      "grad_norm": 0.18605999850951851,
      "learning_rate": 5.020975018883176e-07,
      "logits/chosen": -3.402069330215454,
      "logits/rejected": -3.302852153778076,
      "logps/chosen": -17.27945899963379,
      "logps/rejected": -82.4462890625,
      "loss": 0.0944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1662278175354004,
      "rewards/margins": 3.1662278175354004,
      "rewards/rejected": 0.0,
      "step": 2335
    },
    {
      "epoch": 13.050279329608939,
      "grad_norm": 0.0641457095325415,
      "learning_rate": 5.002669488545111e-07,
      "logits/chosen": -3.5279643535614014,
      "logits/rejected": -3.4702913761138916,
      "logps/chosen": -0.1655784249305725,
      "logps/rejected": -97.58221435546875,
      "loss": 0.1098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4412941932678223,
      "rewards/margins": 2.4412941932678223,
      "rewards/rejected": 0.0,
      "step": 2336
    },
    {
      "epoch": 13.05586592178771,
      "grad_norm": 0.17515961903312807,
      "learning_rate": 4.984393677801522e-07,
      "logits/chosen": -3.343071222305298,
      "logits/rejected": -3.293121337890625,
      "logps/chosen": -0.4246513843536377,
      "logps/rejected": -54.556549072265625,
      "loss": 0.1216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.303307056427002,
      "rewards/margins": 3.303307056427002,
      "rewards/rejected": 0.0,
      "step": 2337
    },
    {
      "epoch": 13.061452513966481,
      "grad_norm": 0.10278910602521377,
      "learning_rate": 4.966147613813513e-07,
      "logits/chosen": -3.4435691833496094,
      "logits/rejected": -3.471031427383423,
      "logps/chosen": -3.20812726020813,
      "logps/rejected": -80.81615447998047,
      "loss": 0.0967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.936457872390747,
      "rewards/margins": 3.936457872390747,
      "rewards/rejected": 0.0,
      "step": 2338
    },
    {
      "epoch": 13.067039106145252,
      "grad_norm": 0.0858247617266109,
      "learning_rate": 4.947931323697983e-07,
      "logits/chosen": -3.5163543224334717,
      "logits/rejected": -3.618976354598999,
      "logps/chosen": -4.342674255371094,
      "logps/rejected": -125.63754272460938,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9120399951934814,
      "rewards/margins": 3.9120399951934814,
      "rewards/rejected": 0.0,
      "step": 2339
    },
    {
      "epoch": 13.072625698324023,
      "grad_norm": 0.645843427137349,
      "learning_rate": 4.929744834527583e-07,
      "logits/chosen": -3.0633034706115723,
      "logits/rejected": -3.128042697906494,
      "logps/chosen": -0.8560799956321716,
      "logps/rejected": -61.278343200683594,
      "loss": 0.1069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9127979278564453,
      "rewards/margins": 2.9127979278564453,
      "rewards/rejected": 0.0,
      "step": 2340
    },
    {
      "epoch": 13.072625698324023,
      "eval_logits/chosen": -3.353998899459839,
      "eval_logits/rejected": -3.4816031455993652,
      "eval_logps/chosen": -46.13653564453125,
      "eval_logps/rejected": -72.74333953857422,
      "eval_loss": 2.3499388694763184,
      "eval_rewards/accuracies": 0.15000000596046448,
      "eval_rewards/chosen": -1.606649398803711,
      "eval_rewards/margins": -1.606649398803711,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5025,
      "eval_samples_per_second": 12.156,
      "eval_steps_per_second": 0.392,
      "step": 2340
    },
    {
      "epoch": 13.078212290502794,
      "grad_norm": 0.09583642286816736,
      "learning_rate": 4.911588173330678e-07,
      "logits/chosen": -3.31001615524292,
      "logits/rejected": -3.5502357482910156,
      "logps/chosen": -2.656379461288452,
      "logps/rejected": -75.39154052734375,
      "loss": 0.0847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.203608989715576,
      "rewards/margins": 3.203608989715576,
      "rewards/rejected": 0.0,
      "step": 2341
    },
    {
      "epoch": 13.083798882681565,
      "grad_norm": 0.09951558860632484,
      "learning_rate": 4.893461367091298e-07,
      "logits/chosen": -3.4383323192596436,
      "logits/rejected": -3.424530506134033,
      "logps/chosen": -0.8386834859848022,
      "logps/rejected": -46.915931701660156,
      "loss": 0.0933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.373680591583252,
      "rewards/margins": 3.373680591583252,
      "rewards/rejected": 0.0,
      "step": 2342
    },
    {
      "epoch": 13.089385474860336,
      "grad_norm": 0.21150849064820765,
      "learning_rate": 4.875364442749098e-07,
      "logits/chosen": -3.5851426124572754,
      "logits/rejected": -3.676271677017212,
      "logps/chosen": -13.341169357299805,
      "logps/rejected": -75.01953125,
      "loss": 0.0735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.5096025466918945,
      "rewards/margins": 4.5096025466918945,
      "rewards/rejected": 0.0,
      "step": 2343
    },
    {
      "epoch": 13.094972067039107,
      "grad_norm": 0.13671793679807542,
      "learning_rate": 4.857297427199348e-07,
      "logits/chosen": -3.7888834476470947,
      "logits/rejected": -3.57891583442688,
      "logps/chosen": -0.18057756125926971,
      "logps/rejected": -78.03916931152344,
      "loss": 0.1305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4886438846588135,
      "rewards/margins": 2.4886438846588135,
      "rewards/rejected": 0.0,
      "step": 2344
    },
    {
      "epoch": 13.100558659217878,
      "grad_norm": 0.07397950631921027,
      "learning_rate": 4.839260347292832e-07,
      "logits/chosen": -3.5582990646362305,
      "logits/rejected": -3.4508273601531982,
      "logps/chosen": -10.881428718566895,
      "logps/rejected": -76.01599884033203,
      "loss": 0.0906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.662540435791016,
      "rewards/margins": 4.662540435791016,
      "rewards/rejected": 0.0,
      "step": 2345
    },
    {
      "epoch": 13.106145251396647,
      "grad_norm": 0.07628360643508866,
      "learning_rate": 4.821253229835873e-07,
      "logits/chosen": -3.4505109786987305,
      "logits/rejected": -3.4952337741851807,
      "logps/chosen": -0.9379197955131531,
      "logps/rejected": -116.2783203125,
      "loss": 0.1119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9011127948760986,
      "rewards/margins": 2.9011127948760986,
      "rewards/rejected": 0.0,
      "step": 2346
    },
    {
      "epoch": 13.111731843575418,
      "grad_norm": 0.07792910271625256,
      "learning_rate": 4.803276101590256e-07,
      "logits/chosen": -3.384596824645996,
      "logits/rejected": -3.215394973754883,
      "logps/chosen": -8.830188751220703,
      "logps/rejected": -58.37457275390625,
      "loss": 0.0999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.050571441650391,
      "rewards/margins": 4.050571441650391,
      "rewards/rejected": 0.0,
      "step": 2347
    },
    {
      "epoch": 13.11731843575419,
      "grad_norm": 0.08259364150537783,
      "learning_rate": 4.785328989273191e-07,
      "logits/chosen": -3.443798065185547,
      "logits/rejected": -3.591818332672119,
      "logps/chosen": -0.2024080455303192,
      "logps/rejected": -123.34164428710938,
      "loss": 0.1236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.101670265197754,
      "rewards/margins": 2.101670265197754,
      "rewards/rejected": 0.0,
      "step": 2348
    },
    {
      "epoch": 13.12290502793296,
      "grad_norm": 0.09150898694887458,
      "learning_rate": 4.767411919557277e-07,
      "logits/chosen": -3.6083364486694336,
      "logits/rejected": -3.4983301162719727,
      "logps/chosen": -12.200567245483398,
      "logps/rejected": -68.13478088378906,
      "loss": 0.086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.417600631713867,
      "rewards/margins": 4.417600631713867,
      "rewards/rejected": 0.0,
      "step": 2349
    },
    {
      "epoch": 13.128491620111731,
      "grad_norm": 0.1970617904579865,
      "learning_rate": 4.749524919070492e-07,
      "logits/chosen": -3.293276786804199,
      "logits/rejected": -3.3597466945648193,
      "logps/chosen": -0.9417994022369385,
      "logps/rejected": -48.398101806640625,
      "loss": 0.0807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3255090713500977,
      "rewards/margins": 3.3255090713500977,
      "rewards/rejected": 0.0,
      "step": 2350
    },
    {
      "epoch": 13.134078212290502,
      "grad_norm": 0.06748742955928196,
      "learning_rate": 4.731668014396079e-07,
      "logits/chosen": -3.468803882598877,
      "logits/rejected": -3.6224238872528076,
      "logps/chosen": -0.4035766124725342,
      "logps/rejected": -74.30254364013672,
      "loss": 0.1201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.267674207687378,
      "rewards/margins": 3.267674207687378,
      "rewards/rejected": 0.0,
      "step": 2351
    },
    {
      "epoch": 13.139664804469273,
      "grad_norm": 0.09503479852483274,
      "learning_rate": 4.713841232072586e-07,
      "logits/chosen": -3.61975359916687,
      "logits/rejected": -3.6807308197021484,
      "logps/chosen": -0.6110123991966248,
      "logps/rejected": -59.08464813232422,
      "loss": 0.0936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3487439155578613,
      "rewards/margins": 2.3487439155578613,
      "rewards/rejected": 0.0,
      "step": 2352
    },
    {
      "epoch": 13.145251396648044,
      "grad_norm": 0.09021401340663597,
      "learning_rate": 4.696044598593802e-07,
      "logits/chosen": -3.4063143730163574,
      "logits/rejected": -3.4189727306365967,
      "logps/chosen": -0.1092749685049057,
      "logps/rejected": -85.14401245117188,
      "loss": 0.102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3180320262908936,
      "rewards/margins": 2.3180320262908936,
      "rewards/rejected": 0.0,
      "step": 2353
    },
    {
      "epoch": 13.150837988826815,
      "grad_norm": 0.07159812984906265,
      "learning_rate": 4.678278140408668e-07,
      "logits/chosen": -3.3585703372955322,
      "logits/rejected": -3.265885591506958,
      "logps/chosen": -0.13775989413261414,
      "logps/rejected": -99.69073486328125,
      "loss": 0.1428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.224139928817749,
      "rewards/margins": 2.224139928817749,
      "rewards/rejected": 0.0,
      "step": 2354
    },
    {
      "epoch": 13.156424581005586,
      "grad_norm": 0.09122867619294994,
      "learning_rate": 4.660541883921324e-07,
      "logits/chosen": -3.3294968605041504,
      "logits/rejected": -3.2347652912139893,
      "logps/chosen": -1.459070086479187,
      "logps/rejected": -62.5692253112793,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6303224563598633,
      "rewards/margins": 3.6303224563598633,
      "rewards/rejected": 0.0,
      "step": 2355
    },
    {
      "epoch": 13.162011173184357,
      "grad_norm": 0.1485890352374918,
      "learning_rate": 4.6428358554909986e-07,
      "logits/chosen": -3.303612470626831,
      "logits/rejected": -3.437373399734497,
      "logps/chosen": -0.3052166700363159,
      "logps/rejected": -73.16818237304688,
      "loss": 0.1176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.993596076965332,
      "rewards/margins": 2.993596076965332,
      "rewards/rejected": 0.0,
      "step": 2356
    },
    {
      "epoch": 13.167597765363128,
      "grad_norm": 0.1189528170930055,
      "learning_rate": 4.625160081432006e-07,
      "logits/chosen": -3.391552448272705,
      "logits/rejected": -3.500352144241333,
      "logps/chosen": -3.224017858505249,
      "logps/rejected": -46.76387405395508,
      "loss": 0.1027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.531614303588867,
      "rewards/margins": 2.531614303588867,
      "rewards/rejected": 0.0,
      "step": 2357
    },
    {
      "epoch": 13.1731843575419,
      "grad_norm": 0.066724603388373,
      "learning_rate": 4.607514588013687e-07,
      "logits/chosen": -3.437931537628174,
      "logits/rejected": -3.46559739112854,
      "logps/chosen": -0.1633237898349762,
      "logps/rejected": -101.50218963623047,
      "loss": 0.0708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.740285634994507,
      "rewards/margins": 2.740285634994507,
      "rewards/rejected": 0.0,
      "step": 2358
    },
    {
      "epoch": 13.17877094972067,
      "grad_norm": 0.1665269822035803,
      "learning_rate": 4.589899401460407e-07,
      "logits/chosen": -3.504716396331787,
      "logits/rejected": -3.4521288871765137,
      "logps/chosen": -2.6304192543029785,
      "logps/rejected": -67.97142028808594,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9993362426757812,
      "rewards/margins": 3.9993362426757812,
      "rewards/rejected": 0.0,
      "step": 2359
    },
    {
      "epoch": 13.184357541899441,
      "grad_norm": 0.29923807458189006,
      "learning_rate": 4.572314547951448e-07,
      "logits/chosen": -3.5113542079925537,
      "logits/rejected": -3.464677095413208,
      "logps/chosen": -14.092560768127441,
      "logps/rejected": -64.75141906738281,
      "loss": 0.088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5790412425994873,
      "rewards/margins": 3.5790412425994873,
      "rewards/rejected": 0.0,
      "step": 2360
    },
    {
      "epoch": 13.184357541899441,
      "eval_logits/chosen": -3.3520870208740234,
      "eval_logits/rejected": -3.4795660972595215,
      "eval_logps/chosen": -46.59404373168945,
      "eval_logps/rejected": -73.07565307617188,
      "eval_loss": 2.3788130283355713,
      "eval_rewards/accuracies": 0.15000000596046448,
      "eval_rewards/chosen": -1.652400255203247,
      "eval_rewards/margins": -1.652400255203247,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5588,
      "eval_samples_per_second": 12.129,
      "eval_steps_per_second": 0.391,
      "step": 2360
    },
    {
      "epoch": 13.189944134078212,
      "grad_norm": 0.09717991718874217,
      "learning_rate": 4.5547600536210353e-07,
      "logits/chosen": -3.5462725162506104,
      "logits/rejected": -3.6992228031158447,
      "logps/chosen": -1.0089384317398071,
      "logps/rejected": -64.65962219238281,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.34206485748291,
      "rewards/margins": 3.34206485748291,
      "rewards/rejected": 0.0,
      "step": 2361
    },
    {
      "epoch": 13.195530726256983,
      "grad_norm": 0.08305811714856412,
      "learning_rate": 4.5372359445582965e-07,
      "logits/chosen": -3.3632335662841797,
      "logits/rejected": -3.485802412033081,
      "logps/chosen": -0.14904357492923737,
      "logps/rejected": -81.00834655761719,
      "loss": 0.1016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7276611328125,
      "rewards/margins": 2.7276611328125,
      "rewards/rejected": 0.0,
      "step": 2362
    },
    {
      "epoch": 13.201117318435754,
      "grad_norm": 0.08029768095336837,
      "learning_rate": 4.5197422468071486e-07,
      "logits/chosen": -3.569096803665161,
      "logits/rejected": -3.5541741847991943,
      "logps/chosen": -9.431748390197754,
      "logps/rejected": -86.56019592285156,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8818717002868652,
      "rewards/margins": 3.8818717002868652,
      "rewards/rejected": 0.0,
      "step": 2363
    },
    {
      "epoch": 13.206703910614525,
      "grad_norm": 0.12700924335196287,
      "learning_rate": 4.502278986366371e-07,
      "logits/chosen": -3.213494062423706,
      "logits/rejected": -3.358944892883301,
      "logps/chosen": -0.25112926959991455,
      "logps/rejected": -71.07521057128906,
      "loss": 0.1011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8735084533691406,
      "rewards/margins": 2.8735084533691406,
      "rewards/rejected": 0.0,
      "step": 2364
    },
    {
      "epoch": 13.212290502793296,
      "grad_norm": 0.0845572687740815,
      "learning_rate": 4.4848461891894485e-07,
      "logits/chosen": -3.337512969970703,
      "logits/rejected": -3.2945666313171387,
      "logps/chosen": -16.513648986816406,
      "logps/rejected": -60.405677795410156,
      "loss": 0.0725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.279664993286133,
      "rewards/margins": 5.279664993286133,
      "rewards/rejected": 0.0,
      "step": 2365
    },
    {
      "epoch": 13.217877094972067,
      "grad_norm": 0.09071424757712401,
      "learning_rate": 4.467443881184647e-07,
      "logits/chosen": -3.610560417175293,
      "logits/rejected": -3.73707914352417,
      "logps/chosen": -0.1489546000957489,
      "logps/rejected": -77.1484146118164,
      "loss": 0.1037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5615689754486084,
      "rewards/margins": 2.5615689754486084,
      "rewards/rejected": 0.0,
      "step": 2366
    },
    {
      "epoch": 13.223463687150838,
      "grad_norm": 0.09547360498547428,
      "learning_rate": 4.4500720882148826e-07,
      "logits/chosen": -3.274169921875,
      "logits/rejected": -3.3194327354431152,
      "logps/chosen": -2.478492021560669,
      "logps/rejected": -77.68093872070312,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.935755729675293,
      "rewards/margins": 2.935755729675293,
      "rewards/rejected": 0.0,
      "step": 2367
    },
    {
      "epoch": 13.22905027932961,
      "grad_norm": 0.3286303864307798,
      "learning_rate": 4.4327308360977403e-07,
      "logits/chosen": -3.3583054542541504,
      "logits/rejected": -3.3257603645324707,
      "logps/chosen": -4.1445465087890625,
      "logps/rejected": -81.30121612548828,
      "loss": 0.0965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6544013023376465,
      "rewards/margins": 3.6544013023376465,
      "rewards/rejected": 0.0,
      "step": 2368
    },
    {
      "epoch": 13.23463687150838,
      "grad_norm": 0.10093075018208633,
      "learning_rate": 4.4154201506053985e-07,
      "logits/chosen": -3.324838638305664,
      "logits/rejected": -3.620448112487793,
      "logps/chosen": -0.1968832165002823,
      "logps/rejected": -103.41017150878906,
      "loss": 0.1452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7165145874023438,
      "rewards/margins": 2.7165145874023438,
      "rewards/rejected": 0.0,
      "step": 2369
    },
    {
      "epoch": 13.240223463687151,
      "grad_norm": 0.10296323833157112,
      "learning_rate": 4.3981400574646366e-07,
      "logits/chosen": -3.3665947914123535,
      "logits/rejected": -3.3447265625,
      "logps/chosen": -2.013845920562744,
      "logps/rejected": -59.445533752441406,
      "loss": 0.0999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5815491676330566,
      "rewards/margins": 3.5815491676330566,
      "rewards/rejected": 0.0,
      "step": 2370
    },
    {
      "epoch": 13.245810055865922,
      "grad_norm": 0.08035083247125835,
      "learning_rate": 4.380890582356737e-07,
      "logits/chosen": -3.4131131172180176,
      "logits/rejected": -3.4496049880981445,
      "logps/chosen": -4.0461320877075195,
      "logps/rejected": -72.68350219726562,
      "loss": 0.0757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3010993003845215,
      "rewards/margins": 3.3010993003845215,
      "rewards/rejected": 0.0,
      "step": 2371
    },
    {
      "epoch": 13.251396648044693,
      "grad_norm": 0.12673745203266848,
      "learning_rate": 4.3636717509174866e-07,
      "logits/chosen": -3.3105075359344482,
      "logits/rejected": -3.4122469425201416,
      "logps/chosen": -22.07501220703125,
      "logps/rejected": -73.4105224609375,
      "loss": 0.0961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.576569080352783,
      "rewards/margins": 3.576569080352783,
      "rewards/rejected": 0.0,
      "step": 2372
    },
    {
      "epoch": 13.256983240223464,
      "grad_norm": 0.1185591642335577,
      "learning_rate": 4.3464835887371545e-07,
      "logits/chosen": -3.4597537517547607,
      "logits/rejected": -3.5353665351867676,
      "logps/chosen": -2.761237621307373,
      "logps/rejected": -64.93196868896484,
      "loss": 0.0825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.940885543823242,
      "rewards/margins": 4.940885543823242,
      "rewards/rejected": 0.0,
      "step": 2373
    },
    {
      "epoch": 13.262569832402235,
      "grad_norm": 0.15904330976272366,
      "learning_rate": 4.329326121360389e-07,
      "logits/chosen": -2.940678834915161,
      "logits/rejected": -3.1144423484802246,
      "logps/chosen": -1.2044460773468018,
      "logps/rejected": -58.461692810058594,
      "loss": 0.126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.186143398284912,
      "rewards/margins": 3.186143398284912,
      "rewards/rejected": 0.0,
      "step": 2374
    },
    {
      "epoch": 13.268156424581006,
      "grad_norm": 0.15558826974440426,
      "learning_rate": 4.3121993742862604e-07,
      "logits/chosen": -3.655787467956543,
      "logits/rejected": -3.549283981323242,
      "logps/chosen": -1.5161187648773193,
      "logps/rejected": -66.79326629638672,
      "loss": 0.0876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1286368370056152,
      "rewards/margins": 3.1286368370056152,
      "rewards/rejected": 0.0,
      "step": 2375
    },
    {
      "epoch": 13.273743016759777,
      "grad_norm": 0.0716960439175312,
      "learning_rate": 4.295103372968154e-07,
      "logits/chosen": -3.591437816619873,
      "logits/rejected": -3.5254125595092773,
      "logps/chosen": -1.2727251052856445,
      "logps/rejected": -130.39434814453125,
      "loss": 0.0888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.308932304382324,
      "rewards/margins": 4.308932304382324,
      "rewards/rejected": 0.0,
      "step": 2376
    },
    {
      "epoch": 13.279329608938548,
      "grad_norm": 0.21320902517549747,
      "learning_rate": 4.2780381428137786e-07,
      "logits/chosen": -2.802349805831909,
      "logits/rejected": -2.850440263748169,
      "logps/chosen": -5.306392669677734,
      "logps/rejected": -90.96321868896484,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6557435989379883,
      "rewards/margins": 2.6557435989379883,
      "rewards/rejected": 0.0,
      "step": 2377
    },
    {
      "epoch": 13.28491620111732,
      "grad_norm": 0.07589878988784862,
      "learning_rate": 4.2610037091850935e-07,
      "logits/chosen": -3.642465591430664,
      "logits/rejected": -3.661285638809204,
      "logps/chosen": -0.08915361762046814,
      "logps/rejected": -68.56631469726562,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.132692337036133,
      "rewards/margins": 2.132692337036133,
      "rewards/rejected": 0.0,
      "step": 2378
    },
    {
      "epoch": 13.29050279329609,
      "grad_norm": 0.10389953469567653,
      "learning_rate": 4.2440000973983235e-07,
      "logits/chosen": -3.4384982585906982,
      "logits/rejected": -3.6116085052490234,
      "logps/chosen": -0.4708046615123749,
      "logps/rejected": -59.43729019165039,
      "loss": 0.0544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.647056818008423,
      "rewards/margins": 3.647056818008423,
      "rewards/rejected": 0.0,
      "step": 2379
    },
    {
      "epoch": 13.296089385474861,
      "grad_norm": 0.06607354944080203,
      "learning_rate": 4.2270273327238405e-07,
      "logits/chosen": -3.5759708881378174,
      "logits/rejected": -3.5401830673217773,
      "logps/chosen": -0.6937935948371887,
      "logps/rejected": -81.88150024414062,
      "loss": 0.1107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.330674648284912,
      "rewards/margins": 2.330674648284912,
      "rewards/rejected": 0.0,
      "step": 2380
    },
    {
      "epoch": 13.296089385474861,
      "eval_logits/chosen": -3.354031801223755,
      "eval_logits/rejected": -3.4805188179016113,
      "eval_logps/chosen": -46.80321502685547,
      "eval_logps/rejected": -73.61221313476562,
      "eval_loss": 2.39788818359375,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.6733176708221436,
      "eval_rewards/margins": -1.6733176708221436,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5023,
      "eval_samples_per_second": 12.156,
      "eval_steps_per_second": 0.392,
      "step": 2380
    },
    {
      "epoch": 13.30167597765363,
      "grad_norm": 0.10908249095506699,
      "learning_rate": 4.210085440386208e-07,
      "logits/chosen": -3.484001398086548,
      "logits/rejected": -3.751460552215576,
      "logps/chosen": -0.08413586765527725,
      "logps/rejected": -88.45378875732422,
      "loss": 0.1024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.156501293182373,
      "rewards/margins": 2.156501293182373,
      "rewards/rejected": 0.0,
      "step": 2381
    },
    {
      "epoch": 13.307262569832401,
      "grad_norm": 0.07856992330285188,
      "learning_rate": 4.193174445564099e-07,
      "logits/chosen": -3.5617361068725586,
      "logits/rejected": -3.5280981063842773,
      "logps/chosen": -0.07057482749223709,
      "logps/rejected": -44.3885498046875,
      "loss": 0.0919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7885136604309082,
      "rewards/margins": 1.7885136604309082,
      "rewards/rejected": 0.0,
      "step": 2382
    },
    {
      "epoch": 13.312849162011172,
      "grad_norm": 0.10215457809185459,
      "learning_rate": 4.176294373390244e-07,
      "logits/chosen": -3.463021993637085,
      "logits/rejected": -3.3689513206481934,
      "logps/chosen": -2.392714023590088,
      "logps/rejected": -68.97950744628906,
      "loss": 0.0824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.329698085784912,
      "rewards/margins": 3.329698085784912,
      "rewards/rejected": 0.0,
      "step": 2383
    },
    {
      "epoch": 13.318435754189943,
      "grad_norm": 0.11357017415939338,
      "learning_rate": 4.1594452489514575e-07,
      "logits/chosen": -3.350276231765747,
      "logits/rejected": -3.3890724182128906,
      "logps/chosen": -1.7104978561401367,
      "logps/rejected": -55.44786071777344,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.586601734161377,
      "rewards/margins": 3.586601734161377,
      "rewards/rejected": 0.0,
      "step": 2384
    },
    {
      "epoch": 13.324022346368714,
      "grad_norm": 0.08138875026592655,
      "learning_rate": 4.142627097288529e-07,
      "logits/chosen": -3.5360164642333984,
      "logits/rejected": -3.5340769290924072,
      "logps/chosen": -0.17133747041225433,
      "logps/rejected": -65.82359313964844,
      "loss": 0.105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.445951461791992,
      "rewards/margins": 2.445951461791992,
      "rewards/rejected": 0.0,
      "step": 2385
    },
    {
      "epoch": 13.329608938547485,
      "grad_norm": 0.09464445082048523,
      "learning_rate": 4.125839943396229e-07,
      "logits/chosen": -3.641810417175293,
      "logits/rejected": -3.716230869293213,
      "logps/chosen": -2.634326696395874,
      "logps/rejected": -101.59146118164062,
      "loss": 0.0853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.733835458755493,
      "rewards/margins": 3.733835458755493,
      "rewards/rejected": 0.0,
      "step": 2386
    },
    {
      "epoch": 13.335195530726256,
      "grad_norm": 0.10055362694492198,
      "learning_rate": 4.1090838122232526e-07,
      "logits/chosen": -3.3924787044525146,
      "logits/rejected": -3.238204002380371,
      "logps/chosen": -0.10390670597553253,
      "logps/rejected": -56.593833923339844,
      "loss": 0.1039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3669164180755615,
      "rewards/margins": 2.3669164180755615,
      "rewards/rejected": 0.0,
      "step": 2387
    },
    {
      "epoch": 13.340782122905027,
      "grad_norm": 0.09151151812325893,
      "learning_rate": 4.0923587286722106e-07,
      "logits/chosen": -3.6608753204345703,
      "logits/rejected": -3.3988101482391357,
      "logps/chosen": -0.07522915303707123,
      "logps/rejected": -65.96997833251953,
      "loss": 0.1084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2355637550354004,
      "rewards/margins": 2.2355637550354004,
      "rewards/rejected": 0.0,
      "step": 2388
    },
    {
      "epoch": 13.346368715083798,
      "grad_norm": 0.05897486573890948,
      "learning_rate": 4.075664717599534e-07,
      "logits/chosen": -3.5354223251342773,
      "logits/rejected": -3.5021004676818848,
      "logps/chosen": -0.11513124406337738,
      "logps/rejected": -83.27766418457031,
      "loss": 0.1007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.259751796722412,
      "rewards/margins": 2.259751796722412,
      "rewards/rejected": 0.0,
      "step": 2389
    },
    {
      "epoch": 13.35195530726257,
      "grad_norm": 0.08341094668280054,
      "learning_rate": 4.0590018038155125e-07,
      "logits/chosen": -3.6924984455108643,
      "logits/rejected": -3.7378110885620117,
      "logps/chosen": -1.0775041580200195,
      "logps/rejected": -100.16629028320312,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.740509271621704,
      "rewards/margins": 2.740509271621704,
      "rewards/rejected": 0.0,
      "step": 2390
    },
    {
      "epoch": 13.35754189944134,
      "grad_norm": 0.1442893107021938,
      "learning_rate": 4.042370012084204e-07,
      "logits/chosen": -3.413963556289673,
      "logits/rejected": -3.478569269180298,
      "logps/chosen": -0.11893672496080399,
      "logps/rejected": -72.97813415527344,
      "loss": 0.0744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6002769470214844,
      "rewards/margins": 2.6002769470214844,
      "rewards/rejected": 0.0,
      "step": 2391
    },
    {
      "epoch": 13.363128491620111,
      "grad_norm": 0.07687124757142288,
      "learning_rate": 4.025769367123397e-07,
      "logits/chosen": -3.360154151916504,
      "logits/rejected": -3.1293749809265137,
      "logps/chosen": -0.42295849323272705,
      "logps/rejected": -64.27865600585938,
      "loss": 0.0901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3634114265441895,
      "rewards/margins": 3.3634114265441895,
      "rewards/rejected": 0.0,
      "step": 2392
    },
    {
      "epoch": 13.368715083798882,
      "grad_norm": 0.07403303990131362,
      "learning_rate": 4.009199893604629e-07,
      "logits/chosen": -3.4864580631256104,
      "logits/rejected": -3.590585470199585,
      "logps/chosen": -0.15596607327461243,
      "logps/rejected": -58.389705657958984,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.316098213195801,
      "rewards/margins": 2.316098213195801,
      "rewards/rejected": 0.0,
      "step": 2393
    },
    {
      "epoch": 13.374301675977653,
      "grad_norm": 0.48774654436756126,
      "learning_rate": 3.992661616153062e-07,
      "logits/chosen": -3.7485849857330322,
      "logits/rejected": -3.749330759048462,
      "logps/chosen": -2.743675947189331,
      "logps/rejected": -61.78852844238281,
      "loss": 0.1159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7152695655822754,
      "rewards/margins": 3.7152695655822754,
      "rewards/rejected": 0.0,
      "step": 2394
    },
    {
      "epoch": 13.379888268156424,
      "grad_norm": 0.06486223224235604,
      "learning_rate": 3.976154559347542e-07,
      "logits/chosen": -3.5824191570281982,
      "logits/rejected": -3.4099743366241455,
      "logps/chosen": -1.0806165933609009,
      "logps/rejected": -80.55619812011719,
      "loss": 0.1328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5059432983398438,
      "rewards/margins": 3.5059432983398438,
      "rewards/rejected": 0.0,
      "step": 2395
    },
    {
      "epoch": 13.385474860335195,
      "grad_norm": 0.0815801078845251,
      "learning_rate": 3.9596787477204886e-07,
      "logits/chosen": -3.524552822113037,
      "logits/rejected": -3.458749294281006,
      "logps/chosen": -2.2036752700805664,
      "logps/rejected": -66.55563354492188,
      "loss": 0.1119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.31294584274292,
      "rewards/margins": 4.31294584274292,
      "rewards/rejected": 0.0,
      "step": 2396
    },
    {
      "epoch": 13.391061452513966,
      "grad_norm": 0.10812612677192339,
      "learning_rate": 3.9432342057578925e-07,
      "logits/chosen": -3.396843910217285,
      "logits/rejected": -3.163593053817749,
      "logps/chosen": -4.0607476234436035,
      "logps/rejected": -79.9134292602539,
      "loss": 0.1074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.721312999725342,
      "rewards/margins": 3.721312999725342,
      "rewards/rejected": 0.0,
      "step": 2397
    },
    {
      "epoch": 13.396648044692737,
      "grad_norm": 0.09064771519364266,
      "learning_rate": 3.926820957899266e-07,
      "logits/chosen": -2.820913076400757,
      "logits/rejected": -2.756129741668701,
      "logps/chosen": -1.2037519216537476,
      "logps/rejected": -48.849342346191406,
      "loss": 0.0949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.278182029724121,
      "rewards/margins": 3.278182029724121,
      "rewards/rejected": 0.0,
      "step": 2398
    },
    {
      "epoch": 13.402234636871508,
      "grad_norm": 0.0728018374076396,
      "learning_rate": 3.910439028537638e-07,
      "logits/chosen": -3.556046485900879,
      "logits/rejected": -3.469937562942505,
      "logps/chosen": -3.2048377990722656,
      "logps/rejected": -80.20484924316406,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2772216796875,
      "rewards/margins": 3.2772216796875,
      "rewards/rejected": 0.0,
      "step": 2399
    },
    {
      "epoch": 13.40782122905028,
      "grad_norm": 0.07568427472869052,
      "learning_rate": 3.894088442019445e-07,
      "logits/chosen": -3.7153420448303223,
      "logits/rejected": -3.7107832431793213,
      "logps/chosen": -0.037309933453798294,
      "logps/rejected": -82.29194641113281,
      "loss": 0.1081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6380164623260498,
      "rewards/margins": 1.6380164623260498,
      "rewards/rejected": 0.0,
      "step": 2400
    },
    {
      "epoch": 13.40782122905028,
      "eval_logits/chosen": -3.348306179046631,
      "eval_logits/rejected": -3.4757156372070312,
      "eval_logps/chosen": -46.80491256713867,
      "eval_logps/rejected": -73.73841857910156,
      "eval_loss": 2.4104015827178955,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.6734873056411743,
      "eval_rewards/margins": -1.6734873056411743,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5679,
      "eval_samples_per_second": 12.125,
      "eval_steps_per_second": 0.391,
      "step": 2400
    },
    {
      "epoch": 13.41340782122905,
      "grad_norm": 0.14738864581160505,
      "learning_rate": 3.8777692226445893e-07,
      "logits/chosen": -3.502089500427246,
      "logits/rejected": -3.4978837966918945,
      "logps/chosen": -3.2908568382263184,
      "logps/rejected": -73.71754455566406,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.366363763809204,
      "rewards/margins": 3.366363763809204,
      "rewards/rejected": 0.0,
      "step": 2401
    },
    {
      "epoch": 13.418994413407821,
      "grad_norm": 0.11635723293652013,
      "learning_rate": 3.8614813946663386e-07,
      "logits/chosen": -3.4786300659179688,
      "logits/rejected": -3.468635082244873,
      "logps/chosen": -2.5704123973846436,
      "logps/rejected": -58.047523498535156,
      "loss": 0.0971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7932496070861816,
      "rewards/margins": 3.7932496070861816,
      "rewards/rejected": 0.0,
      "step": 2402
    },
    {
      "epoch": 13.424581005586592,
      "grad_norm": 0.07828182331710691,
      "learning_rate": 3.84522498229129e-07,
      "logits/chosen": -3.6448073387145996,
      "logits/rejected": -3.6129167079925537,
      "logps/chosen": -0.06785455346107483,
      "logps/rejected": -59.43236541748047,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.077080011367798,
      "rewards/margins": 2.077080011367798,
      "rewards/rejected": 0.0,
      "step": 2403
    },
    {
      "epoch": 13.430167597765363,
      "grad_norm": 0.21210454959297903,
      "learning_rate": 3.829000009679382e-07,
      "logits/chosen": -3.5420358180999756,
      "logits/rejected": -3.337660074234009,
      "logps/chosen": -0.11697079241275787,
      "logps/rejected": -126.64588928222656,
      "loss": 0.1301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8320258855819702,
      "rewards/margins": 1.8320258855819702,
      "rewards/rejected": 0.0,
      "step": 2404
    },
    {
      "epoch": 13.435754189944134,
      "grad_norm": 0.08179383835272667,
      "learning_rate": 3.8128065009438055e-07,
      "logits/chosen": -3.59652042388916,
      "logits/rejected": -3.5469086170196533,
      "logps/chosen": -1.155144214630127,
      "logps/rejected": -56.633174896240234,
      "loss": 0.1259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8097023963928223,
      "rewards/margins": 3.8097023963928223,
      "rewards/rejected": 0.0,
      "step": 2405
    },
    {
      "epoch": 13.441340782122905,
      "grad_norm": 0.11492008621605941,
      "learning_rate": 3.796644480151004e-07,
      "logits/chosen": -3.4782094955444336,
      "logits/rejected": -3.3721461296081543,
      "logps/chosen": -0.054418548941612244,
      "logps/rejected": -72.76170349121094,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2070512771606445,
      "rewards/margins": 2.2070512771606445,
      "rewards/rejected": 0.0,
      "step": 2406
    },
    {
      "epoch": 13.446927374301676,
      "grad_norm": 0.1223466759046393,
      "learning_rate": 3.7805139713206097e-07,
      "logits/chosen": -3.2708752155303955,
      "logits/rejected": -3.3411552906036377,
      "logps/chosen": -3.164421796798706,
      "logps/rejected": -122.8577880859375,
      "loss": 0.0691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.105147361755371,
      "rewards/margins": 4.105147361755371,
      "rewards/rejected": 0.0,
      "step": 2407
    },
    {
      "epoch": 13.452513966480447,
      "grad_norm": 0.24268080560663355,
      "learning_rate": 3.7644149984254504e-07,
      "logits/chosen": -3.6297974586486816,
      "logits/rejected": -3.600715160369873,
      "logps/chosen": -8.93201732635498,
      "logps/rejected": -84.84858703613281,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.932894706726074,
      "rewards/margins": 4.932894706726074,
      "rewards/rejected": 0.0,
      "step": 2408
    },
    {
      "epoch": 13.458100558659218,
      "grad_norm": 0.09444280738346643,
      "learning_rate": 3.74834758539144e-07,
      "logits/chosen": -3.758460760116577,
      "logits/rejected": -3.6840896606445312,
      "logps/chosen": -0.4876845180988312,
      "logps/rejected": -86.92129516601562,
      "loss": 0.1057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.085153102874756,
      "rewards/margins": 3.085153102874756,
      "rewards/rejected": 0.0,
      "step": 2409
    },
    {
      "epoch": 13.46368715083799,
      "grad_norm": 0.06929263345250801,
      "learning_rate": 3.7323117560976397e-07,
      "logits/chosen": -3.3203847408294678,
      "logits/rejected": -3.3409314155578613,
      "logps/chosen": -0.07676055282354355,
      "logps/rejected": -64.52201080322266,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.511639356613159,
      "rewards/margins": 2.511639356613159,
      "rewards/rejected": 0.0,
      "step": 2410
    },
    {
      "epoch": 13.46927374301676,
      "grad_norm": 0.07396742396842713,
      "learning_rate": 3.71630753437614e-07,
      "logits/chosen": -3.431149482727051,
      "logits/rejected": -3.415910005569458,
      "logps/chosen": -3.4936373233795166,
      "logps/rejected": -71.47201538085938,
      "loss": 0.092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8979527950286865,
      "rewards/margins": 3.8979527950286865,
      "rewards/rejected": 0.0,
      "step": 2411
    },
    {
      "epoch": 13.474860335195531,
      "grad_norm": 0.07707483140535139,
      "learning_rate": 3.700334944012055e-07,
      "logits/chosen": -3.3456616401672363,
      "logits/rejected": -3.3433666229248047,
      "logps/chosen": -4.409855842590332,
      "logps/rejected": -76.59762573242188,
      "loss": 0.0781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0333781242370605,
      "rewards/margins": 3.0333781242370605,
      "rewards/rejected": 0.0,
      "step": 2412
    },
    {
      "epoch": 13.480446927374302,
      "grad_norm": 0.17304872670350502,
      "learning_rate": 3.684394008743508e-07,
      "logits/chosen": -3.3046412467956543,
      "logits/rejected": -3.2734296321868896,
      "logps/chosen": -0.5493013262748718,
      "logps/rejected": -92.08245086669922,
      "loss": 0.0889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.939560651779175,
      "rewards/margins": 2.939560651779175,
      "rewards/rejected": 0.0,
      "step": 2413
    },
    {
      "epoch": 13.486033519553073,
      "grad_norm": 0.07836790286988436,
      "learning_rate": 3.668484752261567e-07,
      "logits/chosen": -3.3691844940185547,
      "logits/rejected": -3.2704591751098633,
      "logps/chosen": -0.18993207812309265,
      "logps/rejected": -69.08438110351562,
      "loss": 0.0812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.523850440979004,
      "rewards/margins": 2.523850440979004,
      "rewards/rejected": 0.0,
      "step": 2414
    },
    {
      "epoch": 13.491620111731844,
      "grad_norm": 0.06724756177538199,
      "learning_rate": 3.6526071982102204e-07,
      "logits/chosen": -3.7247323989868164,
      "logits/rejected": -3.656930923461914,
      "logps/chosen": -22.9239559173584,
      "logps/rejected": -57.19197082519531,
      "loss": 0.0883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.296345233917236,
      "rewards/margins": 5.296345233917236,
      "rewards/rejected": 0.0,
      "step": 2415
    },
    {
      "epoch": 13.497206703910614,
      "grad_norm": 0.060900865977249204,
      "learning_rate": 3.6367613701863314e-07,
      "logits/chosen": -3.5141096115112305,
      "logits/rejected": -3.6663858890533447,
      "logps/chosen": -0.13058701157569885,
      "logps/rejected": -102.78010559082031,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1434192657470703,
      "rewards/margins": 2.1434192657470703,
      "rewards/rejected": 0.0,
      "step": 2416
    },
    {
      "epoch": 13.502793296089386,
      "grad_norm": 0.0813410416668334,
      "learning_rate": 3.620947291739643e-07,
      "logits/chosen": -3.425926446914673,
      "logits/rejected": -3.595884323120117,
      "logps/chosen": -0.26099666953086853,
      "logps/rejected": -82.52583312988281,
      "loss": 0.1221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8349461555480957,
      "rewards/margins": 2.8349461555480957,
      "rewards/rejected": 0.0,
      "step": 2417
    },
    {
      "epoch": 13.508379888268156,
      "grad_norm": 0.21080753292581472,
      "learning_rate": 3.6051649863726725e-07,
      "logits/chosen": -3.307145833969116,
      "logits/rejected": -3.580261707305908,
      "logps/chosen": -0.2805855870246887,
      "logps/rejected": -106.5877456665039,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.286137580871582,
      "rewards/margins": 2.286137580871582,
      "rewards/rejected": 0.0,
      "step": 2418
    },
    {
      "epoch": 13.513966480446927,
      "grad_norm": 0.06401143190343084,
      "learning_rate": 3.5894144775407454e-07,
      "logits/chosen": -3.4060654640197754,
      "logits/rejected": -3.495098829269409,
      "logps/chosen": -0.1740775853395462,
      "logps/rejected": -88.56179809570312,
      "loss": 0.1185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.770761251449585,
      "rewards/margins": 2.770761251449585,
      "rewards/rejected": 0.0,
      "step": 2419
    },
    {
      "epoch": 13.519553072625698,
      "grad_norm": 0.20421350285668263,
      "learning_rate": 3.5736957886519236e-07,
      "logits/chosen": -3.476874589920044,
      "logits/rejected": -3.411682367324829,
      "logps/chosen": -0.11385145783424377,
      "logps/rejected": -78.89096069335938,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.829246997833252,
      "rewards/margins": 2.829246997833252,
      "rewards/rejected": 0.0,
      "step": 2420
    },
    {
      "epoch": 13.519553072625698,
      "eval_logits/chosen": -3.3448517322540283,
      "eval_logits/rejected": -3.4729888439178467,
      "eval_logps/chosen": -47.071556091308594,
      "eval_logps/rejected": -73.60029602050781,
      "eval_loss": 2.423509120941162,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.700150728225708,
      "eval_rewards/margins": -1.700150728225708,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 27.8763,
      "eval_samples_per_second": 11.121,
      "eval_steps_per_second": 0.359,
      "step": 2420
    },
    {
      "epoch": 13.525139664804469,
      "grad_norm": 0.07082864084628239,
      "learning_rate": 3.5580089430669735e-07,
      "logits/chosen": -3.485508680343628,
      "logits/rejected": -3.5469813346862793,
      "logps/chosen": -0.18249845504760742,
      "logps/rejected": -75.79119110107422,
      "loss": 0.1008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.249706745147705,
      "rewards/margins": 2.249706745147705,
      "rewards/rejected": 0.0,
      "step": 2421
    },
    {
      "epoch": 13.53072625698324,
      "grad_norm": 0.32225874236436897,
      "learning_rate": 3.5423539640993515e-07,
      "logits/chosen": -3.535936117172241,
      "logits/rejected": -3.5459561347961426,
      "logps/chosen": -0.12420013546943665,
      "logps/rejected": -77.0419921875,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.47371244430542,
      "rewards/margins": 2.47371244430542,
      "rewards/rejected": 0.0,
      "step": 2422
    },
    {
      "epoch": 13.53631284916201,
      "grad_norm": 0.2958126591661973,
      "learning_rate": 3.5267308750151245e-07,
      "logits/chosen": -3.4110255241394043,
      "logits/rejected": -3.492873430252075,
      "logps/chosen": -0.062307197600603104,
      "logps/rejected": -73.58877563476562,
      "loss": 0.1011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7871346473693848,
      "rewards/margins": 1.7871346473693848,
      "rewards/rejected": 0.0,
      "step": 2423
    },
    {
      "epoch": 13.541899441340782,
      "grad_norm": 0.07396934580466905,
      "learning_rate": 3.511139699033003e-07,
      "logits/chosen": -3.530174970626831,
      "logits/rejected": -3.6910154819488525,
      "logps/chosen": -0.6298149824142456,
      "logps/rejected": -82.11556243896484,
      "loss": 0.0685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.439830780029297,
      "rewards/margins": 3.439830780029297,
      "rewards/rejected": 0.0,
      "step": 2424
    },
    {
      "epoch": 13.547486033519553,
      "grad_norm": 0.314573451893643,
      "learning_rate": 3.4955804593242463e-07,
      "logits/chosen": -3.3857715129852295,
      "logits/rejected": -3.49529767036438,
      "logps/chosen": -0.1998807191848755,
      "logps/rejected": -82.67332458496094,
      "loss": 0.1039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.56441068649292,
      "rewards/margins": 2.56441068649292,
      "rewards/rejected": 0.0,
      "step": 2425
    },
    {
      "epoch": 13.553072625698324,
      "grad_norm": 0.07405300558564933,
      "learning_rate": 3.4800531790126546e-07,
      "logits/chosen": -3.6854283809661865,
      "logits/rejected": -3.5427370071411133,
      "logps/chosen": -1.0475825071334839,
      "logps/rejected": -111.07952880859375,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.358628273010254,
      "rewards/margins": 3.358628273010254,
      "rewards/rejected": 0.0,
      "step": 2426
    },
    {
      "epoch": 13.558659217877095,
      "grad_norm": 0.0850827061258327,
      "learning_rate": 3.464557881174524e-07,
      "logits/chosen": -3.588014602661133,
      "logits/rejected": -3.4559223651885986,
      "logps/chosen": -0.10150431841611862,
      "logps/rejected": -65.59356689453125,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.213263511657715,
      "rewards/margins": 2.213263511657715,
      "rewards/rejected": 0.0,
      "step": 2427
    },
    {
      "epoch": 13.564245810055866,
      "grad_norm": 0.07177613622952454,
      "learning_rate": 3.449094588838647e-07,
      "logits/chosen": -3.548358917236328,
      "logits/rejected": -3.4721429347991943,
      "logps/chosen": -1.9232832193374634,
      "logps/rejected": -63.36542892456055,
      "loss": 0.0744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2691593170166016,
      "rewards/margins": 3.2691593170166016,
      "rewards/rejected": 0.0,
      "step": 2428
    },
    {
      "epoch": 13.569832402234637,
      "grad_norm": 0.09368320112253432,
      "learning_rate": 3.4336633249862084e-07,
      "logits/chosen": -3.576014995574951,
      "logits/rejected": -3.674995183944702,
      "logps/chosen": -0.21109536290168762,
      "logps/rejected": -123.41665649414062,
      "loss": 0.1434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7819783687591553,
      "rewards/margins": 2.7819783687591553,
      "rewards/rejected": 0.0,
      "step": 2429
    },
    {
      "epoch": 13.575418994413408,
      "grad_norm": 0.2159256795083037,
      "learning_rate": 3.418264112550826e-07,
      "logits/chosen": -3.479005813598633,
      "logits/rejected": -3.401298761367798,
      "logps/chosen": -0.26282066106796265,
      "logps/rejected": -71.68814086914062,
      "loss": 0.0943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4452500343322754,
      "rewards/margins": 2.4452500343322754,
      "rewards/rejected": 0.0,
      "step": 2430
    },
    {
      "epoch": 13.581005586592179,
      "grad_norm": 0.09010696274629004,
      "learning_rate": 3.402896974418471e-07,
      "logits/chosen": -3.6892404556274414,
      "logits/rejected": -3.762402057647705,
      "logps/chosen": -1.0251109600067139,
      "logps/rejected": -97.33385467529297,
      "loss": 0.0699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7137656211853027,
      "rewards/margins": 3.7137656211853027,
      "rewards/rejected": 0.0,
      "step": 2431
    },
    {
      "epoch": 13.58659217877095,
      "grad_norm": 0.1031355162499906,
      "learning_rate": 3.387561933427444e-07,
      "logits/chosen": -3.6867258548736572,
      "logits/rejected": -3.7253613471984863,
      "logps/chosen": -0.6960359215736389,
      "logps/rejected": -89.77147674560547,
      "loss": 0.0846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.138943672180176,
      "rewards/margins": 3.138943672180176,
      "rewards/rejected": 0.0,
      "step": 2432
    },
    {
      "epoch": 13.59217877094972,
      "grad_norm": 0.3934637456026942,
      "learning_rate": 3.372259012368348e-07,
      "logits/chosen": -3.591291904449463,
      "logits/rejected": -3.4391002655029297,
      "logps/chosen": -0.40246590971946716,
      "logps/rejected": -81.24476623535156,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0640506744384766,
      "rewards/margins": 2.0640506744384766,
      "rewards/rejected": 0.0,
      "step": 2433
    },
    {
      "epoch": 13.597765363128492,
      "grad_norm": 0.06563108572214671,
      "learning_rate": 3.3569882339840516e-07,
      "logits/chosen": -3.5015101432800293,
      "logits/rejected": -3.5751280784606934,
      "logps/chosen": -0.11808285117149353,
      "logps/rejected": -91.91254425048828,
      "loss": 0.0912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2330522537231445,
      "rewards/margins": 2.2330522537231445,
      "rewards/rejected": 0.0,
      "step": 2434
    },
    {
      "epoch": 13.603351955307263,
      "grad_norm": 0.14324991474593404,
      "learning_rate": 3.3417496209696477e-07,
      "logits/chosen": -3.5621461868286133,
      "logits/rejected": -3.3222765922546387,
      "logps/chosen": -0.31496670842170715,
      "logps/rejected": -86.99099731445312,
      "loss": 0.1038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.103086471557617,
      "rewards/margins": 3.103086471557617,
      "rewards/rejected": 0.0,
      "step": 2435
    },
    {
      "epoch": 13.608938547486034,
      "grad_norm": 0.20736939652277114,
      "learning_rate": 3.326543195972426e-07,
      "logits/chosen": -3.0016777515411377,
      "logits/rejected": -2.986591339111328,
      "logps/chosen": -37.192996978759766,
      "logps/rejected": -107.050537109375,
      "loss": 0.0956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2771408557891846,
      "rewards/margins": 3.2771408557891846,
      "rewards/rejected": 0.0,
      "step": 2436
    },
    {
      "epoch": 13.614525139664805,
      "grad_norm": 0.07458093382349272,
      "learning_rate": 3.311368981591856e-07,
      "logits/chosen": -3.4710559844970703,
      "logits/rejected": -3.5487639904022217,
      "logps/chosen": -0.160185769200325,
      "logps/rejected": -99.90351867675781,
      "loss": 0.0943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4614205360412598,
      "rewards/margins": 2.4614205360412598,
      "rewards/rejected": 0.0,
      "step": 2437
    },
    {
      "epoch": 13.620111731843576,
      "grad_norm": 0.05871970512369245,
      "learning_rate": 3.2962270003795037e-07,
      "logits/chosen": -3.6246044635772705,
      "logits/rejected": -3.61747407913208,
      "logps/chosen": -0.93093341588974,
      "logps/rejected": -27.946964263916016,
      "loss": 0.1242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0623703002929688,
      "rewards/margins": 3.0623703002929688,
      "rewards/rejected": 0.0,
      "step": 2438
    },
    {
      "epoch": 13.625698324022347,
      "grad_norm": 0.30862783521483567,
      "learning_rate": 3.281117274839066e-07,
      "logits/chosen": -3.450073719024658,
      "logits/rejected": -3.539985418319702,
      "logps/chosen": -0.09324783831834793,
      "logps/rejected": -81.36856079101562,
      "loss": 0.1151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.467654228210449,
      "rewards/margins": 2.467654228210449,
      "rewards/rejected": 0.0,
      "step": 2439
    },
    {
      "epoch": 13.631284916201118,
      "grad_norm": 0.16099600366866668,
      "learning_rate": 3.2660398274262763e-07,
      "logits/chosen": -3.916867256164551,
      "logits/rejected": -3.7951126098632812,
      "logps/chosen": -0.2749519348144531,
      "logps/rejected": -56.33202362060547,
      "loss": 0.0969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.603975772857666,
      "rewards/margins": 3.603975772857666,
      "rewards/rejected": 0.0,
      "step": 2440
    },
    {
      "epoch": 13.631284916201118,
      "eval_logits/chosen": -3.34370493888855,
      "eval_logits/rejected": -3.470942258834839,
      "eval_logps/chosen": -46.841182708740234,
      "eval_logps/rejected": -74.23516845703125,
      "eval_loss": 2.4306135177612305,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.6771137714385986,
      "eval_rewards/margins": -1.6771137714385986,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6912,
      "eval_samples_per_second": 12.066,
      "eval_steps_per_second": 0.389,
      "step": 2440
    },
    {
      "epoch": 13.636871508379889,
      "grad_norm": 0.09907241702746304,
      "learning_rate": 3.2509946805489107e-07,
      "logits/chosen": -3.6485111713409424,
      "logits/rejected": -3.6105005741119385,
      "logps/chosen": -0.29173585772514343,
      "logps/rejected": -56.18252182006836,
      "loss": 0.0904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.537933111190796,
      "rewards/margins": 2.537933111190796,
      "rewards/rejected": 0.0,
      "step": 2441
    },
    {
      "epoch": 13.64245810055866,
      "grad_norm": 1.9206540167781851,
      "learning_rate": 3.235981856566736e-07,
      "logits/chosen": -3.3616139888763428,
      "logits/rejected": -3.1806344985961914,
      "logps/chosen": -0.41706615686416626,
      "logps/rejected": -96.85348510742188,
      "loss": 0.1221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.225367784500122,
      "rewards/margins": 2.225367784500122,
      "rewards/rejected": 0.0,
      "step": 2442
    },
    {
      "epoch": 13.64804469273743,
      "grad_norm": 0.09409555324852656,
      "learning_rate": 3.2210013777914826e-07,
      "logits/chosen": -3.4942398071289062,
      "logits/rejected": -3.477076292037964,
      "logps/chosen": -0.33811110258102417,
      "logps/rejected": -46.66889190673828,
      "loss": 0.0908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.478121280670166,
      "rewards/margins": 2.478121280670166,
      "rewards/rejected": 0.0,
      "step": 2443
    },
    {
      "epoch": 13.653631284916202,
      "grad_norm": 0.13312049429054154,
      "learning_rate": 3.206053266486808e-07,
      "logits/chosen": -3.435533285140991,
      "logits/rejected": -3.5012431144714355,
      "logps/chosen": -0.6334303617477417,
      "logps/rejected": -67.7122573852539,
      "loss": 0.078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3824126720428467,
      "rewards/margins": 3.3824126720428467,
      "rewards/rejected": 0.0,
      "step": 2444
    },
    {
      "epoch": 13.659217877094973,
      "grad_norm": 0.07987159658320954,
      "learning_rate": 3.191137544868267e-07,
      "logits/chosen": -3.5642192363739014,
      "logits/rejected": -3.429804801940918,
      "logps/chosen": -0.23171459138393402,
      "logps/rejected": -64.57881164550781,
      "loss": 0.0908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7254679203033447,
      "rewards/margins": 2.7254679203033447,
      "rewards/rejected": 0.0,
      "step": 2445
    },
    {
      "epoch": 13.664804469273744,
      "grad_norm": 0.10492077504096194,
      "learning_rate": 3.1762542351032876e-07,
      "logits/chosen": -3.474973201751709,
      "logits/rejected": -3.4431824684143066,
      "logps/chosen": -0.15608976781368256,
      "logps/rejected": -84.06350708007812,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.361654281616211,
      "rewards/margins": 2.361654281616211,
      "rewards/rejected": 0.0,
      "step": 2446
    },
    {
      "epoch": 13.670391061452515,
      "grad_norm": 0.2032523412486985,
      "learning_rate": 3.161403359311102e-07,
      "logits/chosen": -3.5028929710388184,
      "logits/rejected": -3.5372958183288574,
      "logps/chosen": -5.227700233459473,
      "logps/rejected": -103.8941421508789,
      "loss": 0.0907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9048268795013428,
      "rewards/margins": 3.9048268795013428,
      "rewards/rejected": 0.0,
      "step": 2447
    },
    {
      "epoch": 13.675977653631286,
      "grad_norm": 0.16783595925709335,
      "learning_rate": 3.146584939562769e-07,
      "logits/chosen": -3.4976625442504883,
      "logits/rejected": -3.4480159282684326,
      "logps/chosen": -0.05306287109851837,
      "logps/rejected": -65.9729995727539,
      "loss": 0.0738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1029577255249023,
      "rewards/margins": 2.1029577255249023,
      "rewards/rejected": 0.0,
      "step": 2448
    },
    {
      "epoch": 13.681564245810057,
      "grad_norm": 0.07608390835187703,
      "learning_rate": 3.1317989978810995e-07,
      "logits/chosen": -3.4238765239715576,
      "logits/rejected": -3.4021248817443848,
      "logps/chosen": -0.4015945792198181,
      "logps/rejected": -46.298091888427734,
      "loss": 0.0628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.988058090209961,
      "rewards/margins": 2.988058090209961,
      "rewards/rejected": 0.0,
      "step": 2449
    },
    {
      "epoch": 13.687150837988828,
      "grad_norm": 0.5145066111488015,
      "learning_rate": 3.1170455562406304e-07,
      "logits/chosen": -3.31148362159729,
      "logits/rejected": -3.3502583503723145,
      "logps/chosen": -0.0630597174167633,
      "logps/rejected": -130.39483642578125,
      "loss": 0.1324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2631068229675293,
      "rewards/margins": 2.2631068229675293,
      "rewards/rejected": 0.0,
      "step": 2450
    },
    {
      "epoch": 13.692737430167599,
      "grad_norm": 0.0876305180722228,
      "learning_rate": 3.1023246365676053e-07,
      "logits/chosen": -3.34834885597229,
      "logits/rejected": -3.3078389167785645,
      "logps/chosen": -1.783242106437683,
      "logps/rejected": -58.87797546386719,
      "loss": 0.0849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8752596378326416,
      "rewards/margins": 3.8752596378326416,
      "rewards/rejected": 0.0,
      "step": 2451
    },
    {
      "epoch": 13.69832402234637,
      "grad_norm": 0.06846204133384884,
      "learning_rate": 3.0876362607399345e-07,
      "logits/chosen": -3.458085298538208,
      "logits/rejected": -3.3878490924835205,
      "logps/chosen": -0.13402509689331055,
      "logps/rejected": -117.20646667480469,
      "loss": 0.0796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.59611177444458,
      "rewards/margins": 2.59611177444458,
      "rewards/rejected": 0.0,
      "step": 2452
    },
    {
      "epoch": 13.703910614525139,
      "grad_norm": 0.08466640198359883,
      "learning_rate": 3.072980450587157e-07,
      "logits/chosen": -3.2924885749816895,
      "logits/rejected": -3.438551425933838,
      "logps/chosen": -0.35600656270980835,
      "logps/rejected": -107.81973266601562,
      "loss": 0.0878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7128448486328125,
      "rewards/margins": 2.7128448486328125,
      "rewards/rejected": 0.0,
      "step": 2453
    },
    {
      "epoch": 13.70949720670391,
      "grad_norm": 0.06597387559549729,
      "learning_rate": 3.058357227890421e-07,
      "logits/chosen": -3.7192819118499756,
      "logits/rejected": -3.6782500743865967,
      "logps/chosen": -0.047431278973817825,
      "logps/rejected": -46.5926513671875,
      "loss": 0.0937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6579246520996094,
      "rewards/margins": 1.6579246520996094,
      "rewards/rejected": 0.0,
      "step": 2454
    },
    {
      "epoch": 13.71508379888268,
      "grad_norm": 0.1270952670183531,
      "learning_rate": 3.0437666143824345e-07,
      "logits/chosen": -3.5042660236358643,
      "logits/rejected": -3.425229072570801,
      "logps/chosen": -1.266618251800537,
      "logps/rejected": -70.47573852539062,
      "loss": 0.0976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0743818283081055,
      "rewards/margins": 3.0743818283081055,
      "rewards/rejected": 0.0,
      "step": 2455
    },
    {
      "epoch": 13.720670391061452,
      "grad_norm": 0.07622842173894674,
      "learning_rate": 3.029208631747446e-07,
      "logits/chosen": -3.293858051300049,
      "logits/rejected": -3.446103096008301,
      "logps/chosen": -0.2705242931842804,
      "logps/rejected": -67.51332092285156,
      "loss": 0.0821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.804863452911377,
      "rewards/margins": 2.804863452911377,
      "rewards/rejected": 0.0,
      "step": 2456
    },
    {
      "epoch": 13.726256983240223,
      "grad_norm": 0.10190619948120286,
      "learning_rate": 3.014683301621227e-07,
      "logits/chosen": -3.488222122192383,
      "logits/rejected": -3.3116676807403564,
      "logps/chosen": -3.5773229598999023,
      "logps/rejected": -95.57415008544922,
      "loss": 0.0906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9734511375427246,
      "rewards/margins": 2.9734511375427246,
      "rewards/rejected": 0.0,
      "step": 2457
    },
    {
      "epoch": 13.731843575418994,
      "grad_norm": 0.23318116917880538,
      "learning_rate": 3.000190645590981e-07,
      "logits/chosen": -3.5139145851135254,
      "logits/rejected": -3.3161673545837402,
      "logps/chosen": -0.0592997781932354,
      "logps/rejected": -123.78335571289062,
      "loss": 0.1003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7666339874267578,
      "rewards/margins": 1.7666339874267578,
      "rewards/rejected": 0.0,
      "step": 2458
    },
    {
      "epoch": 13.737430167597765,
      "grad_norm": 0.059924531199242174,
      "learning_rate": 2.98573068519539e-07,
      "logits/chosen": -3.5426254272460938,
      "logits/rejected": -3.6924123764038086,
      "logps/chosen": -1.3409572839736938,
      "logps/rejected": -57.77281188964844,
      "loss": 0.0725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4590113162994385,
      "rewards/margins": 3.4590113162994385,
      "rewards/rejected": 0.0,
      "step": 2459
    },
    {
      "epoch": 13.743016759776536,
      "grad_norm": 0.06640875027981515,
      "learning_rate": 2.971303441924531e-07,
      "logits/chosen": -3.3699896335601807,
      "logits/rejected": -3.564453125,
      "logps/chosen": -5.141402244567871,
      "logps/rejected": -83.04364013671875,
      "loss": 0.0722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.195014953613281,
      "rewards/margins": 4.195014953613281,
      "rewards/rejected": 0.0,
      "step": 2460
    },
    {
      "epoch": 13.743016759776536,
      "eval_logits/chosen": -3.3431663513183594,
      "eval_logits/rejected": -3.470017671585083,
      "eval_logps/chosen": -46.86882781982422,
      "eval_logps/rejected": -74.01853942871094,
      "eval_loss": 2.4314708709716797,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.67987859249115,
      "eval_rewards/margins": -1.67987859249115,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5282,
      "eval_samples_per_second": 12.143,
      "eval_steps_per_second": 0.392,
      "step": 2460
    },
    {
      "epoch": 13.748603351955307,
      "grad_norm": 0.09911024869144616,
      "learning_rate": 2.956908937219852e-07,
      "logits/chosen": -3.6708669662475586,
      "logits/rejected": -3.721815586090088,
      "logps/chosen": -0.10116290301084518,
      "logps/rejected": -79.97049713134766,
      "loss": 0.0951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.175449848175049,
      "rewards/margins": 2.175449848175049,
      "rewards/rejected": 0.0,
      "step": 2461
    },
    {
      "epoch": 13.754189944134078,
      "grad_norm": 0.07010858870151275,
      "learning_rate": 2.9425471924741514e-07,
      "logits/chosen": -3.588087797164917,
      "logits/rejected": -3.3423104286193848,
      "logps/chosen": -0.33588942885398865,
      "logps/rejected": -60.60133361816406,
      "loss": 0.0935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.968235969543457,
      "rewards/margins": 2.968235969543457,
      "rewards/rejected": 0.0,
      "step": 2462
    },
    {
      "epoch": 13.759776536312849,
      "grad_norm": 0.22188695887182946,
      "learning_rate": 2.928218229031543e-07,
      "logits/chosen": -3.1085641384124756,
      "logits/rejected": -3.2751505374908447,
      "logps/chosen": -27.54231834411621,
      "logps/rejected": -94.22998046875,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2613844871520996,
      "rewards/margins": 3.2613844871520996,
      "rewards/rejected": 0.0,
      "step": 2463
    },
    {
      "epoch": 13.76536312849162,
      "grad_norm": 0.11562890294209809,
      "learning_rate": 2.913922068187414e-07,
      "logits/chosen": -3.4804775714874268,
      "logits/rejected": -3.3976409435272217,
      "logps/chosen": -0.41449832916259766,
      "logps/rejected": -106.88577270507812,
      "loss": 0.095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2877750396728516,
      "rewards/margins": 2.2877750396728516,
      "rewards/rejected": 0.0,
      "step": 2464
    },
    {
      "epoch": 13.77094972067039,
      "grad_norm": 0.2828938647834178,
      "learning_rate": 2.899658731188401e-07,
      "logits/chosen": -3.667658805847168,
      "logits/rejected": -3.4458975791931152,
      "logps/chosen": -3.5535480976104736,
      "logps/rejected": -45.37517547607422,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.170999526977539,
      "rewards/margins": 4.170999526977539,
      "rewards/rejected": 0.0,
      "step": 2465
    },
    {
      "epoch": 13.776536312849162,
      "grad_norm": 0.06902475779494822,
      "learning_rate": 2.8854282392323795e-07,
      "logits/chosen": -3.8324949741363525,
      "logits/rejected": -3.6666088104248047,
      "logps/chosen": -0.6882755160331726,
      "logps/rejected": -94.47148895263672,
      "loss": 0.0775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.889638900756836,
      "rewards/margins": 2.889638900756836,
      "rewards/rejected": 0.0,
      "step": 2466
    },
    {
      "epoch": 13.782122905027933,
      "grad_norm": 0.08040126016142347,
      "learning_rate": 2.871230613468373e-07,
      "logits/chosen": -3.648038148880005,
      "logits/rejected": -3.4645159244537354,
      "logps/chosen": -9.20377254486084,
      "logps/rejected": -57.941986083984375,
      "loss": 0.0577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.365961074829102,
      "rewards/margins": 4.365961074829102,
      "rewards/rejected": 0.0,
      "step": 2467
    },
    {
      "epoch": 13.787709497206704,
      "grad_norm": 0.1588509928347571,
      "learning_rate": 2.8570658749966007e-07,
      "logits/chosen": -3.2533135414123535,
      "logits/rejected": -3.134036064147949,
      "logps/chosen": -2.5509605407714844,
      "logps/rejected": -57.289432525634766,
      "loss": 0.0628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.437003135681152,
      "rewards/margins": 4.437003135681152,
      "rewards/rejected": 0.0,
      "step": 2468
    },
    {
      "epoch": 13.793296089385475,
      "grad_norm": 0.06683631978069966,
      "learning_rate": 2.842934044868378e-07,
      "logits/chosen": -3.315890312194824,
      "logits/rejected": -3.253732442855835,
      "logps/chosen": -0.5833942890167236,
      "logps/rejected": -69.43388366699219,
      "loss": 0.0627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.443880558013916,
      "rewards/margins": 3.443880558013916,
      "rewards/rejected": 0.0,
      "step": 2469
    },
    {
      "epoch": 13.798882681564246,
      "grad_norm": 0.08464634686494026,
      "learning_rate": 2.828835144086123e-07,
      "logits/chosen": -3.2561187744140625,
      "logits/rejected": -3.3787333965301514,
      "logps/chosen": -0.17147111892700195,
      "logps/rejected": -74.27529907226562,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2719664573669434,
      "rewards/margins": 2.2719664573669434,
      "rewards/rejected": 0.0,
      "step": 2470
    },
    {
      "epoch": 13.804469273743017,
      "grad_norm": 0.20423036238591724,
      "learning_rate": 2.814769193603309e-07,
      "logits/chosen": -3.391170024871826,
      "logits/rejected": -3.538818597793579,
      "logps/chosen": -0.23642005026340485,
      "logps/rejected": -100.37882995605469,
      "loss": 0.0905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.242096424102783,
      "rewards/margins": 3.242096424102783,
      "rewards/rejected": 0.0,
      "step": 2471
    },
    {
      "epoch": 13.810055865921788,
      "grad_norm": 0.09697066298286379,
      "learning_rate": 2.800736214324451e-07,
      "logits/chosen": -3.423704147338867,
      "logits/rejected": -3.4110913276672363,
      "logps/chosen": -2.38804292678833,
      "logps/rejected": -61.195919036865234,
      "loss": 0.0937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.396393299102783,
      "rewards/margins": 3.396393299102783,
      "rewards/rejected": 0.0,
      "step": 2472
    },
    {
      "epoch": 13.815642458100559,
      "grad_norm": 0.0880379224108075,
      "learning_rate": 2.7867362271050495e-07,
      "logits/chosen": -3.3954391479492188,
      "logits/rejected": -3.3528265953063965,
      "logps/chosen": -0.1360182762145996,
      "logps/rejected": -72.215576171875,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5446200370788574,
      "rewards/margins": 2.5446200370788574,
      "rewards/rejected": 0.0,
      "step": 2473
    },
    {
      "epoch": 13.82122905027933,
      "grad_norm": 0.1061135115969224,
      "learning_rate": 2.7727692527515756e-07,
      "logits/chosen": -3.273857355117798,
      "logits/rejected": -3.693626642227173,
      "logps/chosen": -1.7035045623779297,
      "logps/rejected": -73.22212219238281,
      "loss": 0.1279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.102421283721924,
      "rewards/margins": 3.102421283721924,
      "rewards/rejected": 0.0,
      "step": 2474
    },
    {
      "epoch": 13.8268156424581,
      "grad_norm": 0.1119052986831734,
      "learning_rate": 2.758835312021455e-07,
      "logits/chosen": -3.2618813514709473,
      "logits/rejected": -3.226534843444824,
      "logps/chosen": -0.5969275832176208,
      "logps/rejected": -75.78498077392578,
      "loss": 0.0977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.316628932952881,
      "rewards/margins": 2.316628932952881,
      "rewards/rejected": 0.0,
      "step": 2475
    },
    {
      "epoch": 13.832402234636872,
      "grad_norm": 0.077142268049799,
      "learning_rate": 2.7449344256229833e-07,
      "logits/chosen": -3.533404588699341,
      "logits/rejected": -3.496398448944092,
      "logps/chosen": -0.07222733646631241,
      "logps/rejected": -67.92062377929688,
      "loss": 0.122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.152993679046631,
      "rewards/margins": 2.152993679046631,
      "rewards/rejected": 0.0,
      "step": 2476
    },
    {
      "epoch": 13.837988826815643,
      "grad_norm": 0.15695166953809173,
      "learning_rate": 2.731066614215375e-07,
      "logits/chosen": -3.590592622756958,
      "logits/rejected": -3.3784372806549072,
      "logps/chosen": -0.17519450187683105,
      "logps/rejected": -91.80224609375,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0065598487854004,
      "rewards/margins": 2.0065598487854004,
      "rewards/rejected": 0.0,
      "step": 2477
    },
    {
      "epoch": 13.843575418994414,
      "grad_norm": 0.06121567004566908,
      "learning_rate": 2.7172318984086536e-07,
      "logits/chosen": -3.497330665588379,
      "logits/rejected": -3.38264799118042,
      "logps/chosen": -1.2570643424987793,
      "logps/rejected": -96.36807250976562,
      "loss": 0.1002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.849271774291992,
      "rewards/margins": 3.849271774291992,
      "rewards/rejected": 0.0,
      "step": 2478
    },
    {
      "epoch": 13.849162011173185,
      "grad_norm": 0.08072708740396184,
      "learning_rate": 2.703430298763676e-07,
      "logits/chosen": -3.5005717277526855,
      "logits/rejected": -3.454916477203369,
      "logps/chosen": -0.11737184226512909,
      "logps/rejected": -105.57450103759766,
      "loss": 0.0853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3745217323303223,
      "rewards/margins": 2.3745217323303223,
      "rewards/rejected": 0.0,
      "step": 2479
    },
    {
      "epoch": 13.854748603351956,
      "grad_norm": 0.07481123983775373,
      "learning_rate": 2.689661835792079e-07,
      "logits/chosen": -3.5959267616271973,
      "logits/rejected": -3.655562400817871,
      "logps/chosen": -18.476953506469727,
      "logps/rejected": -86.87364959716797,
      "loss": 0.0736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.492825508117676,
      "rewards/margins": 5.492825508117676,
      "rewards/rejected": 0.0,
      "step": 2480
    },
    {
      "epoch": 13.854748603351956,
      "eval_logits/chosen": -3.3472354412078857,
      "eval_logits/rejected": -3.473146438598633,
      "eval_logps/chosen": -47.103267669677734,
      "eval_logps/rejected": -74.13362121582031,
      "eval_loss": 2.4496212005615234,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7033226490020752,
      "eval_rewards/margins": -1.7033226490020752,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6165,
      "eval_samples_per_second": 12.102,
      "eval_steps_per_second": 0.39,
      "step": 2480
    },
    {
      "epoch": 13.860335195530727,
      "grad_norm": 0.08135791867978787,
      "learning_rate": 2.6759265299562467e-07,
      "logits/chosen": -3.332637071609497,
      "logits/rejected": -3.4613583087921143,
      "logps/chosen": -1.5516191720962524,
      "logps/rejected": -120.31358337402344,
      "loss": 0.1288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.075535774230957,
      "rewards/margins": 3.075535774230957,
      "rewards/rejected": 0.0,
      "step": 2481
    },
    {
      "epoch": 13.865921787709498,
      "grad_norm": 0.07798911447960502,
      "learning_rate": 2.662224401669289e-07,
      "logits/chosen": -3.8855860233306885,
      "logits/rejected": -3.7364699840545654,
      "logps/chosen": -0.2946701645851135,
      "logps/rejected": -85.08551025390625,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.823657989501953,
      "rewards/margins": 2.823657989501953,
      "rewards/rejected": 0.0,
      "step": 2482
    },
    {
      "epoch": 13.871508379888269,
      "grad_norm": 0.0924198839500047,
      "learning_rate": 2.648555471295014e-07,
      "logits/chosen": -3.5021135807037354,
      "logits/rejected": -3.635871171951294,
      "logps/chosen": -1.5724374055862427,
      "logps/rejected": -58.580650329589844,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7273828983306885,
      "rewards/margins": 3.7273828983306885,
      "rewards/rejected": 0.0,
      "step": 2483
    },
    {
      "epoch": 13.87709497206704,
      "grad_norm": 0.16623899623717336,
      "learning_rate": 2.63491975914788e-07,
      "logits/chosen": -3.5819506645202637,
      "logits/rejected": -3.49332332611084,
      "logps/chosen": -12.128485679626465,
      "logps/rejected": -51.12892532348633,
      "loss": 0.0746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.803791522979736,
      "rewards/margins": 4.803791522979736,
      "rewards/rejected": 0.0,
      "step": 2484
    },
    {
      "epoch": 13.88268156424581,
      "grad_norm": 0.13786083854518152,
      "learning_rate": 2.621317285492983e-07,
      "logits/chosen": -3.56851863861084,
      "logits/rejected": -3.530449867248535,
      "logps/chosen": -6.696420192718506,
      "logps/rejected": -63.08879089355469,
      "loss": 0.108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9861745834350586,
      "rewards/margins": 2.9861745834350586,
      "rewards/rejected": 0.0,
      "step": 2485
    },
    {
      "epoch": 13.888268156424582,
      "grad_norm": 0.16835268538739195,
      "learning_rate": 2.607748070546037e-07,
      "logits/chosen": -3.7477030754089355,
      "logits/rejected": -3.7204768657684326,
      "logps/chosen": -0.3630250096321106,
      "logps/rejected": -68.06639099121094,
      "loss": 0.0923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5493857860565186,
      "rewards/margins": 3.5493857860565186,
      "rewards/rejected": 0.0,
      "step": 2486
    },
    {
      "epoch": 13.893854748603353,
      "grad_norm": 0.08450968729866455,
      "learning_rate": 2.5942121344732867e-07,
      "logits/chosen": -3.4604673385620117,
      "logits/rejected": -3.427232265472412,
      "logps/chosen": -3.2420918941497803,
      "logps/rejected": -56.75363540649414,
      "loss": 0.098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.824456214904785,
      "rewards/margins": 3.824456214904785,
      "rewards/rejected": 0.0,
      "step": 2487
    },
    {
      "epoch": 13.899441340782122,
      "grad_norm": 0.07440629416157299,
      "learning_rate": 2.5807094973915645e-07,
      "logits/chosen": -3.3546013832092285,
      "logits/rejected": -3.438730001449585,
      "logps/chosen": -2.0886452198028564,
      "logps/rejected": -88.70906066894531,
      "loss": 0.0933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1983344554901123,
      "rewards/margins": 3.1983344554901123,
      "rewards/rejected": 0.0,
      "step": 2488
    },
    {
      "epoch": 13.905027932960895,
      "grad_norm": 0.07483050046518659,
      "learning_rate": 2.5672401793681854e-07,
      "logits/chosen": -3.6138057708740234,
      "logits/rejected": -3.5102365016937256,
      "logps/chosen": -12.786505699157715,
      "logps/rejected": -102.26689147949219,
      "loss": 0.0845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.093844890594482,
      "rewards/margins": 4.093844890594482,
      "rewards/rejected": 0.0,
      "step": 2489
    },
    {
      "epoch": 13.910614525139664,
      "grad_norm": 0.08114240360088558,
      "learning_rate": 2.553804200420956e-07,
      "logits/chosen": -3.4356765747070312,
      "logits/rejected": -3.223724365234375,
      "logps/chosen": -2.1925296783447266,
      "logps/rejected": -66.80078125,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.3402299880981445,
      "rewards/margins": 4.3402299880981445,
      "rewards/rejected": 0.0,
      "step": 2490
    },
    {
      "epoch": 13.916201117318435,
      "grad_norm": 0.06297801755694768,
      "learning_rate": 2.5404015805181344e-07,
      "logits/chosen": -3.566406011581421,
      "logits/rejected": -3.5989394187927246,
      "logps/chosen": -0.042248670011758804,
      "logps/rejected": -54.98863983154297,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7091035842895508,
      "rewards/margins": 1.7091035842895508,
      "rewards/rejected": 0.0,
      "step": 2491
    },
    {
      "epoch": 13.921787709497206,
      "grad_norm": 0.09325409579384265,
      "learning_rate": 2.5270323395783974e-07,
      "logits/chosen": -3.451336622238159,
      "logits/rejected": -3.479708671569824,
      "logps/chosen": -0.06145664304494858,
      "logps/rejected": -65.9593734741211,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8049941062927246,
      "rewards/margins": 1.8049941062927246,
      "rewards/rejected": 0.0,
      "step": 2492
    },
    {
      "epoch": 13.927374301675977,
      "grad_norm": 0.11176623336992994,
      "learning_rate": 2.5136964974708216e-07,
      "logits/chosen": -3.3889973163604736,
      "logits/rejected": -3.712376356124878,
      "logps/chosen": -0.16728883981704712,
      "logps/rejected": -83.91727447509766,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6167244911193848,
      "rewards/margins": 2.6167244911193848,
      "rewards/rejected": 0.0,
      "step": 2493
    },
    {
      "epoch": 13.932960893854748,
      "grad_norm": 0.2997199324360231,
      "learning_rate": 2.5003940740148393e-07,
      "logits/chosen": -3.562473773956299,
      "logits/rejected": -3.594219923019409,
      "logps/chosen": -0.030688609927892685,
      "logps/rejected": -136.00814819335938,
      "loss": 0.1317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8337887525558472,
      "rewards/margins": 1.8337887525558472,
      "rewards/rejected": 0.0,
      "step": 2494
    },
    {
      "epoch": 13.938547486033519,
      "grad_norm": 0.0954579763764201,
      "learning_rate": 2.48712508898023e-07,
      "logits/chosen": -3.4618515968322754,
      "logits/rejected": -3.511124610900879,
      "logps/chosen": -4.067011833190918,
      "logps/rejected": -90.26811981201172,
      "loss": 0.0924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9880166053771973,
      "rewards/margins": 3.9880166053771973,
      "rewards/rejected": 0.0,
      "step": 2495
    },
    {
      "epoch": 13.94413407821229,
      "grad_norm": 0.07793839298687713,
      "learning_rate": 2.473889562087056e-07,
      "logits/chosen": -3.2634592056274414,
      "logits/rejected": -3.455894947052002,
      "logps/chosen": -27.139284133911133,
      "logps/rejected": -74.80935668945312,
      "loss": 0.1073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0391135215759277,
      "rewards/margins": 3.0391135215759277,
      "rewards/rejected": 0.0,
      "step": 2496
    },
    {
      "epoch": 13.949720670391061,
      "grad_norm": 0.07971981128132777,
      "learning_rate": 2.4606875130056747e-07,
      "logits/chosen": -3.519185781478882,
      "logits/rejected": -3.3224329948425293,
      "logps/chosen": -0.32605379819869995,
      "logps/rejected": -67.78988647460938,
      "loss": 0.0787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2115440368652344,
      "rewards/margins": 2.2115440368652344,
      "rewards/rejected": 0.0,
      "step": 2497
    },
    {
      "epoch": 13.955307262569832,
      "grad_norm": 0.24229470404476416,
      "learning_rate": 2.447518961356685e-07,
      "logits/chosen": -3.2996163368225098,
      "logits/rejected": -3.2385737895965576,
      "logps/chosen": -0.147914856672287,
      "logps/rejected": -110.00891876220703,
      "loss": 0.1231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4794585704803467,
      "rewards/margins": 2.4794585704803467,
      "rewards/rejected": 0.0,
      "step": 2498
    },
    {
      "epoch": 13.960893854748603,
      "grad_norm": 0.09922993465787687,
      "learning_rate": 2.434383926710893e-07,
      "logits/chosen": -3.797638416290283,
      "logits/rejected": -3.691267728805542,
      "logps/chosen": -0.1353895664215088,
      "logps/rejected": -58.67414093017578,
      "loss": 0.0855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.143000841140747,
      "rewards/margins": 2.143000841140747,
      "rewards/rejected": 0.0,
      "step": 2499
    },
    {
      "epoch": 13.966480446927374,
      "grad_norm": 0.16947535869091093,
      "learning_rate": 2.4212824285893025e-07,
      "logits/chosen": -3.776907444000244,
      "logits/rejected": -3.571009635925293,
      "logps/chosen": -2.2301583290100098,
      "logps/rejected": -62.216827392578125,
      "loss": 0.0854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.2155256271362305,
      "rewards/margins": 4.2155256271362305,
      "rewards/rejected": 0.0,
      "step": 2500
    },
    {
      "epoch": 13.966480446927374,
      "eval_logits/chosen": -3.336491107940674,
      "eval_logits/rejected": -3.465226650238037,
      "eval_logps/chosen": -47.031185150146484,
      "eval_logps/rejected": -74.54609680175781,
      "eval_loss": 2.4522461891174316,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.6961143016815186,
      "eval_rewards/margins": -1.6961143016815186,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6643,
      "eval_samples_per_second": 12.079,
      "eval_steps_per_second": 0.39,
      "step": 2500
    },
    {
      "epoch": 13.972067039106145,
      "grad_norm": 0.23737673857502714,
      "learning_rate": 2.408214486463073e-07,
      "logits/chosen": -3.370893716812134,
      "logits/rejected": -3.2753801345825195,
      "logps/chosen": -2.7280914783477783,
      "logps/rejected": -44.45240783691406,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.023446559906006,
      "rewards/margins": 4.023446559906006,
      "rewards/rejected": 0.0,
      "step": 2501
    },
    {
      "epoch": 13.977653631284916,
      "grad_norm": 0.07661718730879181,
      "learning_rate": 2.395180119753493e-07,
      "logits/chosen": -3.341371774673462,
      "logits/rejected": -3.181255578994751,
      "logps/chosen": -0.19031748175621033,
      "logps/rejected": -39.12879180908203,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2615551948547363,
      "rewards/margins": 2.2615551948547363,
      "rewards/rejected": 0.0,
      "step": 2502
    },
    {
      "epoch": 13.983240223463687,
      "grad_norm": 0.5683047056509717,
      "learning_rate": 2.3821793478319506e-07,
      "logits/chosen": -3.465157985687256,
      "logits/rejected": -3.3312458992004395,
      "logps/chosen": -0.3046277165412903,
      "logps/rejected": -48.78263854980469,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3188037872314453,
      "rewards/margins": 2.3188037872314453,
      "rewards/rejected": 0.0,
      "step": 2503
    },
    {
      "epoch": 13.988826815642458,
      "grad_norm": 0.10182264453136626,
      "learning_rate": 2.3692121900199178e-07,
      "logits/chosen": -3.592302083969116,
      "logits/rejected": -3.4533843994140625,
      "logps/chosen": -1.1839135885238647,
      "logps/rejected": -66.97758483886719,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4577438831329346,
      "rewards/margins": 3.4577438831329346,
      "rewards/rejected": 0.0,
      "step": 2504
    },
    {
      "epoch": 13.994413407821229,
      "grad_norm": 0.08071943637539937,
      "learning_rate": 2.3562786655888876e-07,
      "logits/chosen": -3.7546908855438232,
      "logits/rejected": -3.6228585243225098,
      "logps/chosen": -0.11584276705980301,
      "logps/rejected": -82.10264587402344,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3257553577423096,
      "rewards/margins": 2.3257553577423096,
      "rewards/rejected": 0.0,
      "step": 2505
    },
    {
      "epoch": 14.0,
      "grad_norm": 0.10374005619134981,
      "learning_rate": 2.343378793760387e-07,
      "logits/chosen": -3.2698750495910645,
      "logits/rejected": -3.5047221183776855,
      "logps/chosen": -0.1311020702123642,
      "logps/rejected": -77.5145492553711,
      "loss": 0.1454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9676532745361328,
      "rewards/margins": 1.9676532745361328,
      "rewards/rejected": 0.0,
      "step": 2506
    },
    {
      "epoch": 14.005586592178771,
      "grad_norm": 0.14013986552658206,
      "learning_rate": 2.3305125937059248e-07,
      "logits/chosen": -2.7158968448638916,
      "logits/rejected": -2.9273548126220703,
      "logps/chosen": -17.694000244140625,
      "logps/rejected": -72.4950942993164,
      "loss": 0.1045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.201321601867676,
      "rewards/margins": 4.201321601867676,
      "rewards/rejected": 0.0,
      "step": 2507
    },
    {
      "epoch": 14.011173184357542,
      "grad_norm": 0.09971498993903723,
      "learning_rate": 2.3176800845469638e-07,
      "logits/chosen": -3.341444253921509,
      "logits/rejected": -3.421206474304199,
      "logps/chosen": -0.11163078993558884,
      "logps/rejected": -86.33575439453125,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9950203895568848,
      "rewards/margins": 1.9950203895568848,
      "rewards/rejected": 0.0,
      "step": 2508
    },
    {
      "epoch": 14.016759776536313,
      "grad_norm": 0.0661936696942743,
      "learning_rate": 2.3048812853548958e-07,
      "logits/chosen": -3.2697293758392334,
      "logits/rejected": -3.3647100925445557,
      "logps/chosen": -6.447643756866455,
      "logps/rejected": -73.08621215820312,
      "loss": 0.1169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.76605486869812,
      "rewards/margins": 3.76605486869812,
      "rewards/rejected": 0.0,
      "step": 2509
    },
    {
      "epoch": 14.022346368715084,
      "grad_norm": 0.08082574951685483,
      "learning_rate": 2.2921162151510162e-07,
      "logits/chosen": -3.597073554992676,
      "logits/rejected": -3.543595552444458,
      "logps/chosen": -2.6354141235351562,
      "logps/rejected": -73.28901672363281,
      "loss": 0.0708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.720069169998169,
      "rewards/margins": 3.720069169998169,
      "rewards/rejected": 0.0,
      "step": 2510
    },
    {
      "epoch": 14.027932960893855,
      "grad_norm": 0.14704040332114662,
      "learning_rate": 2.279384892906497e-07,
      "logits/chosen": -3.3883321285247803,
      "logits/rejected": -3.30584716796875,
      "logps/chosen": -1.5573253631591797,
      "logps/rejected": -73.78379821777344,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.0905914306640625,
      "rewards/margins": 4.0905914306640625,
      "rewards/rejected": 0.0,
      "step": 2511
    },
    {
      "epoch": 14.033519553072626,
      "grad_norm": 0.14568319522711412,
      "learning_rate": 2.266687337542345e-07,
      "logits/chosen": -3.628732919692993,
      "logits/rejected": -3.5474350452423096,
      "logps/chosen": -1.0698703527450562,
      "logps/rejected": -64.0962905883789,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.24981951713562,
      "rewards/margins": 3.24981951713562,
      "rewards/rejected": 0.0,
      "step": 2512
    },
    {
      "epoch": 14.039106145251397,
      "grad_norm": 0.07153064900557074,
      "learning_rate": 2.2540235679293893e-07,
      "logits/chosen": -2.93424391746521,
      "logits/rejected": -2.971891403198242,
      "logps/chosen": -0.9876858592033386,
      "logps/rejected": -56.9947395324707,
      "loss": 0.0984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.114886999130249,
      "rewards/margins": 3.114886999130249,
      "rewards/rejected": 0.0,
      "step": 2513
    },
    {
      "epoch": 14.044692737430168,
      "grad_norm": 0.10239431821418471,
      "learning_rate": 2.2413936028882476e-07,
      "logits/chosen": -3.5952985286712646,
      "logits/rejected": -3.3726935386657715,
      "logps/chosen": -0.10697811841964722,
      "logps/rejected": -47.908836364746094,
      "loss": 0.1163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8073182106018066,
      "rewards/margins": 1.8073182106018066,
      "rewards/rejected": 0.0,
      "step": 2514
    },
    {
      "epoch": 14.050279329608939,
      "grad_norm": 0.06853810462738727,
      "learning_rate": 2.2287974611893055e-07,
      "logits/chosen": -3.4513936042785645,
      "logits/rejected": -3.2141473293304443,
      "logps/chosen": -0.6686931848526001,
      "logps/rejected": -72.4605712890625,
      "loss": 0.0958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9004945755004883,
      "rewards/margins": 2.9004945755004883,
      "rewards/rejected": 0.0,
      "step": 2515
    },
    {
      "epoch": 14.05586592178771,
      "grad_norm": 0.22828548783033334,
      "learning_rate": 2.2162351615526544e-07,
      "logits/chosen": -3.760463237762451,
      "logits/rejected": -3.5301713943481445,
      "logps/chosen": -0.5674949884414673,
      "logps/rejected": -110.13203430175781,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1015992164611816,
      "rewards/margins": 3.1015992164611816,
      "rewards/rejected": 0.0,
      "step": 2516
    },
    {
      "epoch": 14.061452513966481,
      "grad_norm": 0.06800467588158036,
      "learning_rate": 2.2037067226481274e-07,
      "logits/chosen": -3.4358224868774414,
      "logits/rejected": -3.3365976810455322,
      "logps/chosen": -0.2602235674858093,
      "logps/rejected": -87.76690673828125,
      "loss": 0.1033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8202412128448486,
      "rewards/margins": 2.8202412128448486,
      "rewards/rejected": 0.0,
      "step": 2517
    },
    {
      "epoch": 14.067039106145252,
      "grad_norm": 0.09165313457613469,
      "learning_rate": 2.1912121630952078e-07,
      "logits/chosen": -3.552253484725952,
      "logits/rejected": -3.421189546585083,
      "logps/chosen": -0.2683897912502289,
      "logps/rejected": -72.16471862792969,
      "loss": 0.1224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2618961334228516,
      "rewards/margins": 2.2618961334228516,
      "rewards/rejected": 0.0,
      "step": 2518
    },
    {
      "epoch": 14.072625698324023,
      "grad_norm": 0.13925067158142224,
      "learning_rate": 2.178751501463036e-07,
      "logits/chosen": -3.5429043769836426,
      "logits/rejected": -3.349855422973633,
      "logps/chosen": -7.927128314971924,
      "logps/rejected": -68.4919662475586,
      "loss": 0.0794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.632591724395752,
      "rewards/margins": 3.632591724395752,
      "rewards/rejected": 0.0,
      "step": 2519
    },
    {
      "epoch": 14.078212290502794,
      "grad_norm": 0.06726802930646461,
      "learning_rate": 2.1663247562703743e-07,
      "logits/chosen": -3.451415777206421,
      "logits/rejected": -3.3215339183807373,
      "logps/chosen": -0.2214897722005844,
      "logps/rejected": -86.51838684082031,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.895322322845459,
      "rewards/margins": 2.895322322845459,
      "rewards/rejected": 0.0,
      "step": 2520
    },
    {
      "epoch": 14.078212290502794,
      "eval_logits/chosen": -3.3341622352600098,
      "eval_logits/rejected": -3.4623122215270996,
      "eval_logps/chosen": -47.24674606323242,
      "eval_logps/rejected": -74.30899047851562,
      "eval_loss": 2.457296133041382,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7176704406738281,
      "eval_rewards/margins": -1.7176704406738281,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5582,
      "eval_samples_per_second": 12.129,
      "eval_steps_per_second": 0.391,
      "step": 2520
    },
    {
      "epoch": 14.083798882681565,
      "grad_norm": 0.13737195275248965,
      "learning_rate": 2.1539319459855823e-07,
      "logits/chosen": -3.2203261852264404,
      "logits/rejected": -3.2778687477111816,
      "logps/chosen": -4.572885990142822,
      "logps/rejected": -50.97980499267578,
      "loss": 0.1013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.739049434661865,
      "rewards/margins": 4.739049434661865,
      "rewards/rejected": 0.0,
      "step": 2521
    },
    {
      "epoch": 14.089385474860336,
      "grad_norm": 0.06132734559392991,
      "learning_rate": 2.1415730890265774e-07,
      "logits/chosen": -3.6293156147003174,
      "logits/rejected": -3.5913796424865723,
      "logps/chosen": -0.06428254395723343,
      "logps/rejected": -92.34864807128906,
      "loss": 0.0726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1865394115448,
      "rewards/margins": 2.1865394115448,
      "rewards/rejected": 0.0,
      "step": 2522
    },
    {
      "epoch": 14.094972067039107,
      "grad_norm": 0.07756271217561486,
      "learning_rate": 2.1292482037608225e-07,
      "logits/chosen": -3.323148250579834,
      "logits/rejected": -3.372734546661377,
      "logps/chosen": -22.646482467651367,
      "logps/rejected": -62.29520034790039,
      "loss": 0.0771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.03524112701416,
      "rewards/margins": 4.03524112701416,
      "rewards/rejected": 0.0,
      "step": 2523
    },
    {
      "epoch": 14.100558659217878,
      "grad_norm": 0.08284230366705944,
      "learning_rate": 2.1169573085053047e-07,
      "logits/chosen": -3.5249741077423096,
      "logits/rejected": -3.6445086002349854,
      "logps/chosen": -0.15742528438568115,
      "logps/rejected": -55.12018585205078,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7372398376464844,
      "rewards/margins": 2.7372398376464844,
      "rewards/rejected": 0.0,
      "step": 2524
    },
    {
      "epoch": 14.106145251396647,
      "grad_norm": 0.07429317782934595,
      "learning_rate": 2.1047004215264621e-07,
      "logits/chosen": -3.3724966049194336,
      "logits/rejected": -3.3290064334869385,
      "logps/chosen": -0.1407449096441269,
      "logps/rejected": -55.875701904296875,
      "loss": 0.0807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6455893516540527,
      "rewards/margins": 2.6455893516540527,
      "rewards/rejected": 0.0,
      "step": 2525
    },
    {
      "epoch": 14.111731843575418,
      "grad_norm": 0.15631999903647442,
      "learning_rate": 2.0924775610402294e-07,
      "logits/chosen": -3.457911252975464,
      "logits/rejected": -3.4672064781188965,
      "logps/chosen": -0.03070785105228424,
      "logps/rejected": -74.26724243164062,
      "loss": 0.1056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1218106746673584,
      "rewards/margins": 1.1218106746673584,
      "rewards/rejected": 0.0,
      "step": 2526
    },
    {
      "epoch": 14.11731843575419,
      "grad_norm": 0.13657787478277916,
      "learning_rate": 2.0802887452119473e-07,
      "logits/chosen": -3.402095079421997,
      "logits/rejected": -3.4178078174591064,
      "logps/chosen": -1.4716047048568726,
      "logps/rejected": -46.01858901977539,
      "loss": 0.1081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.277496814727783,
      "rewards/margins": 3.277496814727783,
      "rewards/rejected": 0.0,
      "step": 2527
    },
    {
      "epoch": 14.12290502793296,
      "grad_norm": 0.06569397145361298,
      "learning_rate": 2.068133992156368e-07,
      "logits/chosen": -3.282944917678833,
      "logits/rejected": -3.340379476547241,
      "logps/chosen": -23.16541290283203,
      "logps/rejected": -85.74346923828125,
      "loss": 0.1008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9732766151428223,
      "rewards/margins": 3.9732766151428223,
      "rewards/rejected": 0.0,
      "step": 2528
    },
    {
      "epoch": 14.128491620111731,
      "grad_norm": 0.14213887252865237,
      "learning_rate": 2.0560133199376163e-07,
      "logits/chosen": -3.5409719944000244,
      "logits/rejected": -3.5160844326019287,
      "logps/chosen": -3.9998013973236084,
      "logps/rejected": -70.05116271972656,
      "loss": 0.1111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.687107086181641,
      "rewards/margins": 4.687107086181641,
      "rewards/rejected": 0.0,
      "step": 2529
    },
    {
      "epoch": 14.134078212290502,
      "grad_norm": 0.12271915927088044,
      "learning_rate": 2.043926746569172e-07,
      "logits/chosen": -3.564129590988159,
      "logits/rejected": -3.5463054180145264,
      "logps/chosen": -0.5550088882446289,
      "logps/rejected": -45.70170211791992,
      "loss": 0.111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.368903160095215,
      "rewards/margins": 2.368903160095215,
      "rewards/rejected": 0.0,
      "step": 2530
    },
    {
      "epoch": 14.139664804469273,
      "grad_norm": 0.07231349112311466,
      "learning_rate": 2.0318742900138384e-07,
      "logits/chosen": -3.5774142742156982,
      "logits/rejected": -3.5145418643951416,
      "logps/chosen": -0.19059038162231445,
      "logps/rejected": -86.80207824707031,
      "loss": 0.0754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2444393634796143,
      "rewards/margins": 2.2444393634796143,
      "rewards/rejected": 0.0,
      "step": 2531
    },
    {
      "epoch": 14.145251396648044,
      "grad_norm": 0.12505182349889774,
      "learning_rate": 2.019855968183701e-07,
      "logits/chosen": -3.609261989593506,
      "logits/rejected": -3.4609596729278564,
      "logps/chosen": -3.0571508407592773,
      "logps/rejected": -71.85231018066406,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.767023086547852,
      "rewards/margins": 4.767023086547852,
      "rewards/rejected": 0.0,
      "step": 2532
    },
    {
      "epoch": 14.150837988826815,
      "grad_norm": 0.08279720217036263,
      "learning_rate": 2.0078717989401447e-07,
      "logits/chosen": -3.5681591033935547,
      "logits/rejected": -3.7504966259002686,
      "logps/chosen": -0.16645142436027527,
      "logps/rejected": -109.19563293457031,
      "loss": 0.123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.110144853591919,
      "rewards/margins": 2.110144853591919,
      "rewards/rejected": 0.0,
      "step": 2533
    },
    {
      "epoch": 14.156424581005586,
      "grad_norm": 0.08557000038815782,
      "learning_rate": 1.9959218000937615e-07,
      "logits/chosen": -3.409817934036255,
      "logits/rejected": -3.538771629333496,
      "logps/chosen": -0.3764108419418335,
      "logps/rejected": -89.08796691894531,
      "loss": 0.1213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.670111656188965,
      "rewards/margins": 2.670111656188965,
      "rewards/rejected": 0.0,
      "step": 2534
    },
    {
      "epoch": 14.162011173184357,
      "grad_norm": 0.078029180028824,
      "learning_rate": 1.9840059894043916e-07,
      "logits/chosen": -3.545056104660034,
      "logits/rejected": -3.7028989791870117,
      "logps/chosen": -0.1015029326081276,
      "logps/rejected": -109.74855041503906,
      "loss": 0.1079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1597750186920166,
      "rewards/margins": 2.1597750186920166,
      "rewards/rejected": 0.0,
      "step": 2535
    },
    {
      "epoch": 14.167597765363128,
      "grad_norm": 0.08492463163481093,
      "learning_rate": 1.9721243845810396e-07,
      "logits/chosen": -3.5347533226013184,
      "logits/rejected": -3.429949998855591,
      "logps/chosen": -18.332744598388672,
      "logps/rejected": -76.03514862060547,
      "loss": 0.0647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7897939682006836,
      "rewards/margins": 3.7897939682006836,
      "rewards/rejected": 0.0,
      "step": 2536
    },
    {
      "epoch": 14.1731843575419,
      "grad_norm": 0.15084817829304878,
      "learning_rate": 1.9602770032818962e-07,
      "logits/chosen": -3.0627658367156982,
      "logits/rejected": -2.984898567199707,
      "logps/chosen": -0.4305160939693451,
      "logps/rejected": -59.69812774658203,
      "loss": 0.1015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.968848466873169,
      "rewards/margins": 2.968848466873169,
      "rewards/rejected": 0.0,
      "step": 2537
    },
    {
      "epoch": 14.17877094972067,
      "grad_norm": 0.08511010682903346,
      "learning_rate": 1.9484638631142749e-07,
      "logits/chosen": -3.2118635177612305,
      "logits/rejected": -3.3119735717773438,
      "logps/chosen": -3.6060609817504883,
      "logps/rejected": -78.73741149902344,
      "loss": 0.0926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0264501571655273,
      "rewards/margins": 3.0264501571655273,
      "rewards/rejected": 0.0,
      "step": 2538
    },
    {
      "epoch": 14.184357541899441,
      "grad_norm": 0.09903347034506677,
      "learning_rate": 1.9366849816346094e-07,
      "logits/chosen": -3.637615919113159,
      "logits/rejected": -3.7389092445373535,
      "logps/chosen": -0.12064376473426819,
      "logps/rejected": -108.15946960449219,
      "loss": 0.0811,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3407177925109863,
      "rewards/margins": 2.3407177925109863,
      "rewards/rejected": 0.0,
      "step": 2539
    },
    {
      "epoch": 14.189944134078212,
      "grad_norm": 0.06300957742028607,
      "learning_rate": 1.924940376348411e-07,
      "logits/chosen": -3.3689234256744385,
      "logits/rejected": -3.5600688457489014,
      "logps/chosen": -0.05647522956132889,
      "logps/rejected": -83.05337524414062,
      "loss": 0.1226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.209959030151367,
      "rewards/margins": 2.209959030151367,
      "rewards/rejected": 0.0,
      "step": 2540
    },
    {
      "epoch": 14.189944134078212,
      "eval_logits/chosen": -3.331552505493164,
      "eval_logits/rejected": -3.460592269897461,
      "eval_logps/chosen": -47.31194305419922,
      "eval_logps/rejected": -74.72709655761719,
      "eval_loss": 2.4649882316589355,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7241897583007812,
      "eval_rewards/margins": -1.7241897583007812,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6122,
      "eval_samples_per_second": 12.104,
      "eval_steps_per_second": 0.39,
      "step": 2540
    },
    {
      "epoch": 14.195530726256983,
      "grad_norm": 0.07847163248709,
      "learning_rate": 1.913230064710256e-07,
      "logits/chosen": -3.537649154663086,
      "logits/rejected": -3.4662158489227295,
      "logps/chosen": -15.92737865447998,
      "logps/rejected": -60.82484817504883,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.630243301391602,
      "rewards/margins": 4.630243301391602,
      "rewards/rejected": 0.0,
      "step": 2541
    },
    {
      "epoch": 14.201117318435754,
      "grad_norm": 0.05421644134448587,
      "learning_rate": 1.9015540641237562e-07,
      "logits/chosen": -3.481093168258667,
      "logits/rejected": -3.4041590690612793,
      "logps/chosen": -0.16377080976963043,
      "logps/rejected": -98.47212982177734,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0120701789855957,
      "rewards/margins": 3.0120701789855957,
      "rewards/rejected": 0.0,
      "step": 2542
    },
    {
      "epoch": 14.206703910614525,
      "grad_norm": 0.10357746232698054,
      "learning_rate": 1.8899123919415202e-07,
      "logits/chosen": -3.1703240871429443,
      "logits/rejected": -3.1634860038757324,
      "logps/chosen": -4.666523456573486,
      "logps/rejected": -42.341678619384766,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9960360527038574,
      "rewards/margins": 2.9960360527038574,
      "rewards/rejected": 0.0,
      "step": 2543
    },
    {
      "epoch": 14.212290502793296,
      "grad_norm": 0.07086372537506068,
      "learning_rate": 1.878305065465158e-07,
      "logits/chosen": -3.5112967491149902,
      "logits/rejected": -3.672274351119995,
      "logps/chosen": -8.819843292236328,
      "logps/rejected": -88.54414367675781,
      "loss": 0.1196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.225799083709717,
      "rewards/margins": 3.225799083709717,
      "rewards/rejected": 0.0,
      "step": 2544
    },
    {
      "epoch": 14.217877094972067,
      "grad_norm": 0.15694130162140896,
      "learning_rate": 1.8667321019452095e-07,
      "logits/chosen": -3.505821466445923,
      "logits/rejected": -3.293067693710327,
      "logps/chosen": -0.04184804856777191,
      "logps/rejected": -79.25289916992188,
      "loss": 0.0866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7695503234863281,
      "rewards/margins": 1.7695503234863281,
      "rewards/rejected": 0.0,
      "step": 2545
    },
    {
      "epoch": 14.223463687150838,
      "grad_norm": 0.06386908273067705,
      "learning_rate": 1.855193518581172e-07,
      "logits/chosen": -3.6064510345458984,
      "logits/rejected": -3.596398115158081,
      "logps/chosen": -0.034129995852708817,
      "logps/rejected": -55.82867431640625,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5340256690979004,
      "rewards/margins": 1.5340256690979004,
      "rewards/rejected": 0.0,
      "step": 2546
    },
    {
      "epoch": 14.22905027932961,
      "grad_norm": 0.07495553037600615,
      "learning_rate": 1.8436893325214312e-07,
      "logits/chosen": -3.4225704669952393,
      "logits/rejected": -3.4533064365386963,
      "logps/chosen": -0.07437299937009811,
      "logps/rejected": -90.46488189697266,
      "loss": 0.0955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9553345441818237,
      "rewards/margins": 1.9553345441818237,
      "rewards/rejected": 0.0,
      "step": 2547
    },
    {
      "epoch": 14.23463687150838,
      "grad_norm": 0.06731466388400151,
      "learning_rate": 1.832219560863255e-07,
      "logits/chosen": -3.5013442039489746,
      "logits/rejected": -3.5395750999450684,
      "logps/chosen": -0.659659743309021,
      "logps/rejected": -67.65909576416016,
      "loss": 0.0828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4359612464904785,
      "rewards/margins": 3.4359612464904785,
      "rewards/rejected": 0.0,
      "step": 2548
    },
    {
      "epoch": 14.240223463687151,
      "grad_norm": 0.07314935300547357,
      "learning_rate": 1.820784220652766e-07,
      "logits/chosen": -3.767683267593384,
      "logits/rejected": -3.6912949085235596,
      "logps/chosen": -21.15695571899414,
      "logps/rejected": -30.914417266845703,
      "loss": 0.0658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.633927345275879,
      "rewards/margins": 4.633927345275879,
      "rewards/rejected": 0.0,
      "step": 2549
    },
    {
      "epoch": 14.245810055865922,
      "grad_norm": 0.09506597952050798,
      "learning_rate": 1.8093833288849173e-07,
      "logits/chosen": -3.331636428833008,
      "logits/rejected": -3.279085874557495,
      "logps/chosen": -0.3950047791004181,
      "logps/rejected": -50.50664520263672,
      "loss": 0.1077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1661458015441895,
      "rewards/margins": 3.1661458015441895,
      "rewards/rejected": 0.0,
      "step": 2550
    },
    {
      "epoch": 14.251396648044693,
      "grad_norm": 0.08339786061567567,
      "learning_rate": 1.798016902503466e-07,
      "logits/chosen": -3.4243831634521484,
      "logits/rejected": -3.277414321899414,
      "logps/chosen": -1.187961220741272,
      "logps/rejected": -98.99130249023438,
      "loss": 0.0986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4333362579345703,
      "rewards/margins": 3.4333362579345703,
      "rewards/rejected": 0.0,
      "step": 2551
    },
    {
      "epoch": 14.256983240223464,
      "grad_norm": 0.07167607824989813,
      "learning_rate": 1.7866849584009415e-07,
      "logits/chosen": -3.5047965049743652,
      "logits/rejected": -3.4253458976745605,
      "logps/chosen": -1.2214423418045044,
      "logps/rejected": -57.77812194824219,
      "loss": 0.0718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.211000442504883,
      "rewards/margins": 4.211000442504883,
      "rewards/rejected": 0.0,
      "step": 2552
    },
    {
      "epoch": 14.262569832402235,
      "grad_norm": 0.06468912681945323,
      "learning_rate": 1.7753875134186415e-07,
      "logits/chosen": -3.608921766281128,
      "logits/rejected": -3.613231658935547,
      "logps/chosen": -0.9310357570648193,
      "logps/rejected": -46.378639221191406,
      "loss": 0.1008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7504615783691406,
      "rewards/margins": 2.7504615783691406,
      "rewards/rejected": 0.0,
      "step": 2553
    },
    {
      "epoch": 14.268156424581006,
      "grad_norm": 0.09552850980991535,
      "learning_rate": 1.764124584346566e-07,
      "logits/chosen": -3.3504645824432373,
      "logits/rejected": -3.301530599594116,
      "logps/chosen": -0.14444191753864288,
      "logps/rejected": -65.51722717285156,
      "loss": 0.1115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0284500122070312,
      "rewards/margins": 2.0284500122070312,
      "rewards/rejected": 0.0,
      "step": 2554
    },
    {
      "epoch": 14.273743016759777,
      "grad_norm": 0.06282166441285134,
      "learning_rate": 1.7528961879234447e-07,
      "logits/chosen": -3.3970272541046143,
      "logits/rejected": -3.54575252532959,
      "logps/chosen": -1.5491836071014404,
      "logps/rejected": -74.45542907714844,
      "loss": 0.1143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1419806480407715,
      "rewards/margins": 3.1419806480407715,
      "rewards/rejected": 0.0,
      "step": 2555
    },
    {
      "epoch": 14.279329608938548,
      "grad_norm": 0.08826782466144162,
      "learning_rate": 1.7417023408366706e-07,
      "logits/chosen": -3.459824323654175,
      "logits/rejected": -3.3296825885772705,
      "logps/chosen": -2.3146004676818848,
      "logps/rejected": -85.70074462890625,
      "loss": 0.0717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.127386093139648,
      "rewards/margins": 4.127386093139648,
      "rewards/rejected": 0.0,
      "step": 2556
    },
    {
      "epoch": 14.28491620111732,
      "grad_norm": 0.06883731419048664,
      "learning_rate": 1.730543059722295e-07,
      "logits/chosen": -3.444694757461548,
      "logits/rejected": -3.5121119022369385,
      "logps/chosen": -8.249305725097656,
      "logps/rejected": -93.48934936523438,
      "loss": 0.0886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.63862681388855,
      "rewards/margins": 3.63862681388855,
      "rewards/rejected": 0.0,
      "step": 2557
    },
    {
      "epoch": 14.29050279329609,
      "grad_norm": 0.070894715511883,
      "learning_rate": 1.7194183611649957e-07,
      "logits/chosen": -3.337170362472534,
      "logits/rejected": -3.408329963684082,
      "logps/chosen": -0.3728465437889099,
      "logps/rejected": -142.5657958984375,
      "loss": 0.0845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7874574661254883,
      "rewards/margins": 2.7874574661254883,
      "rewards/rejected": 0.0,
      "step": 2558
    },
    {
      "epoch": 14.296089385474861,
      "grad_norm": 0.1893246865710598,
      "learning_rate": 1.7083282616980558e-07,
      "logits/chosen": -3.3024370670318604,
      "logits/rejected": -3.416799306869507,
      "logps/chosen": -0.20801487565040588,
      "logps/rejected": -98.42364501953125,
      "loss": 0.1134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4981093406677246,
      "rewards/margins": 2.4981093406677246,
      "rewards/rejected": 0.0,
      "step": 2559
    },
    {
      "epoch": 14.30167597765363,
      "grad_norm": 0.07664520450712667,
      "learning_rate": 1.697272777803341e-07,
      "logits/chosen": -3.4587299823760986,
      "logits/rejected": -3.5126571655273438,
      "logps/chosen": -0.40235698223114014,
      "logps/rejected": -81.27101135253906,
      "loss": 0.1163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.279655933380127,
      "rewards/margins": 3.279655933380127,
      "rewards/rejected": 0.0,
      "step": 2560
    },
    {
      "epoch": 14.30167597765363,
      "eval_logits/chosen": -3.3419296741485596,
      "eval_logits/rejected": -3.4683098793029785,
      "eval_logps/chosen": -47.39228439331055,
      "eval_logps/rejected": -74.82671356201172,
      "eval_loss": 2.476539373397827,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7322242259979248,
      "eval_rewards/margins": -1.7322242259979248,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.7093,
      "eval_samples_per_second": 12.058,
      "eval_steps_per_second": 0.389,
      "step": 2560
    },
    {
      "epoch": 14.307262569832401,
      "grad_norm": 0.07496531730826456,
      "learning_rate": 1.6862519259112613e-07,
      "logits/chosen": -3.1079070568084717,
      "logits/rejected": -3.1072399616241455,
      "logps/chosen": -10.338077545166016,
      "logps/rejected": -85.41987609863281,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6540472507476807,
      "rewards/margins": 3.6540472507476807,
      "rewards/rejected": 0.0,
      "step": 2561
    },
    {
      "epoch": 14.312849162011172,
      "grad_norm": 0.0650640293245698,
      "learning_rate": 1.675265722400779e-07,
      "logits/chosen": -3.2798333168029785,
      "logits/rejected": -3.325429916381836,
      "logps/chosen": -0.09625471383333206,
      "logps/rejected": -60.60560989379883,
      "loss": 0.1047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1207308769226074,
      "rewards/margins": 2.1207308769226074,
      "rewards/rejected": 0.0,
      "step": 2562
    },
    {
      "epoch": 14.318435754189943,
      "grad_norm": 0.240347245649094,
      "learning_rate": 1.664314183599333e-07,
      "logits/chosen": -3.3197522163391113,
      "logits/rejected": -3.5476183891296387,
      "logps/chosen": -0.3420514464378357,
      "logps/rejected": -56.597007751464844,
      "loss": 0.1011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2493674755096436,
      "rewards/margins": 2.2493674755096436,
      "rewards/rejected": 0.0,
      "step": 2563
    },
    {
      "epoch": 14.324022346368714,
      "grad_norm": 0.07848615972274683,
      "learning_rate": 1.6533973257828766e-07,
      "logits/chosen": -3.5880696773529053,
      "logits/rejected": -3.5556066036224365,
      "logps/chosen": -0.3983520269393921,
      "logps/rejected": -90.93360137939453,
      "loss": 0.1084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2907261848449707,
      "rewards/margins": 2.2907261848449707,
      "rewards/rejected": 0.0,
      "step": 2564
    },
    {
      "epoch": 14.329608938547485,
      "grad_norm": 0.06443510685569594,
      "learning_rate": 1.6425151651757898e-07,
      "logits/chosen": -3.312352180480957,
      "logits/rejected": -3.5113353729248047,
      "logps/chosen": -0.7183350324630737,
      "logps/rejected": -67.61034393310547,
      "loss": 0.0878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2718629837036133,
      "rewards/margins": 3.2718629837036133,
      "rewards/rejected": 0.0,
      "step": 2565
    },
    {
      "epoch": 14.335195530726256,
      "grad_norm": 0.07994137844173765,
      "learning_rate": 1.6316677179509104e-07,
      "logits/chosen": -3.2537567615509033,
      "logits/rejected": -3.1024444103240967,
      "logps/chosen": -0.19721224904060364,
      "logps/rejected": -62.46946716308594,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.728640556335449,
      "rewards/margins": 2.728640556335449,
      "rewards/rejected": 0.0,
      "step": 2566
    },
    {
      "epoch": 14.340782122905027,
      "grad_norm": 0.3627518604877921,
      "learning_rate": 1.6208550002294766e-07,
      "logits/chosen": -3.126181125640869,
      "logits/rejected": -3.2870686054229736,
      "logps/chosen": -0.5504027605056763,
      "logps/rejected": -115.98076629638672,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1933603286743164,
      "rewards/margins": 2.1933603286743164,
      "rewards/rejected": 0.0,
      "step": 2567
    },
    {
      "epoch": 14.346368715083798,
      "grad_norm": 0.0957702707468743,
      "learning_rate": 1.6100770280811084e-07,
      "logits/chosen": -3.3722851276397705,
      "logits/rejected": -3.490171432495117,
      "logps/chosen": -1.0246591567993164,
      "logps/rejected": -66.98036193847656,
      "loss": 0.1293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2603726387023926,
      "rewards/margins": 3.2603726387023926,
      "rewards/rejected": 0.0,
      "step": 2568
    },
    {
      "epoch": 14.35195530726257,
      "grad_norm": 0.07558014184842302,
      "learning_rate": 1.599333817523796e-07,
      "logits/chosen": -3.3719429969787598,
      "logits/rejected": -3.3973193168640137,
      "logps/chosen": -1.2939848899841309,
      "logps/rejected": -89.76663970947266,
      "loss": 0.0778,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.900472640991211,
      "rewards/margins": 3.900472640991211,
      "rewards/rejected": 0.0,
      "step": 2569
    },
    {
      "epoch": 14.35754189944134,
      "grad_norm": 0.08621268212392799,
      "learning_rate": 1.5886253845238593e-07,
      "logits/chosen": -3.428781270980835,
      "logits/rejected": -3.279366970062256,
      "logps/chosen": -2.790952682495117,
      "logps/rejected": -97.81484985351562,
      "loss": 0.0637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.349658966064453,
      "rewards/margins": 4.349658966064453,
      "rewards/rejected": 0.0,
      "step": 2570
    },
    {
      "epoch": 14.363128491620111,
      "grad_norm": 0.07403122672075528,
      "learning_rate": 1.5779517449959397e-07,
      "logits/chosen": -3.5159897804260254,
      "logits/rejected": -3.507391929626465,
      "logps/chosen": -0.8858562707901001,
      "logps/rejected": -57.501731872558594,
      "loss": 0.0913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1030948162078857,
      "rewards/margins": 3.1030948162078857,
      "rewards/rejected": 0.0,
      "step": 2571
    },
    {
      "epoch": 14.368715083798882,
      "grad_norm": 0.10074738749975395,
      "learning_rate": 1.5673129148029627e-07,
      "logits/chosen": -3.447173833847046,
      "logits/rejected": -3.5314886569976807,
      "logps/chosen": -0.1544211208820343,
      "logps/rejected": -82.07405853271484,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7711081504821777,
      "rewards/margins": 2.7711081504821777,
      "rewards/rejected": 0.0,
      "step": 2572
    },
    {
      "epoch": 14.374301675977653,
      "grad_norm": 0.08684888356796237,
      "learning_rate": 1.5567089097561328e-07,
      "logits/chosen": -3.3900651931762695,
      "logits/rejected": -3.7440178394317627,
      "logps/chosen": -0.7848489284515381,
      "logps/rejected": -74.51779174804688,
      "loss": 0.1031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1314315795898438,
      "rewards/margins": 3.1314315795898438,
      "rewards/rejected": 0.0,
      "step": 2573
    },
    {
      "epoch": 14.379888268156424,
      "grad_norm": 0.10267792120211053,
      "learning_rate": 1.5461397456148802e-07,
      "logits/chosen": -3.216275215148926,
      "logits/rejected": -3.5065999031066895,
      "logps/chosen": -0.05487659573554993,
      "logps/rejected": -142.13941955566406,
      "loss": 0.1161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0793251991271973,
      "rewards/margins": 2.0793251991271973,
      "rewards/rejected": 0.0,
      "step": 2574
    },
    {
      "epoch": 14.385474860335195,
      "grad_norm": 0.1535575795762717,
      "learning_rate": 1.5356054380868707e-07,
      "logits/chosen": -3.603776454925537,
      "logits/rejected": -3.6794791221618652,
      "logps/chosen": -0.29706066846847534,
      "logps/rejected": -81.89429473876953,
      "loss": 0.1063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9508416652679443,
      "rewards/margins": 2.9508416652679443,
      "rewards/rejected": 0.0,
      "step": 2575
    },
    {
      "epoch": 14.391061452513966,
      "grad_norm": 0.06050580361647226,
      "learning_rate": 1.5251060028279612e-07,
      "logits/chosen": -3.446075439453125,
      "logits/rejected": -3.5177643299102783,
      "logps/chosen": -0.8321819305419922,
      "logps/rejected": -73.25489807128906,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3444302082061768,
      "rewards/margins": 3.3444302082061768,
      "rewards/rejected": 0.0,
      "step": 2576
    },
    {
      "epoch": 14.396648044692737,
      "grad_norm": 0.22637281991374175,
      "learning_rate": 1.5146414554421812e-07,
      "logits/chosen": -3.4229063987731934,
      "logits/rejected": -3.480694055557251,
      "logps/chosen": -0.1472569704055786,
      "logps/rejected": -101.4334716796875,
      "loss": 0.1131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.257418394088745,
      "rewards/margins": 2.257418394088745,
      "rewards/rejected": 0.0,
      "step": 2577
    },
    {
      "epoch": 14.402234636871508,
      "grad_norm": 0.11807285924759534,
      "learning_rate": 1.5042118114817122e-07,
      "logits/chosen": -3.5964837074279785,
      "logits/rejected": -3.456418037414551,
      "logps/chosen": -0.08364443480968475,
      "logps/rejected": -71.90811920166016,
      "loss": 0.0875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8505768775939941,
      "rewards/margins": 1.8505768775939941,
      "rewards/rejected": 0.0,
      "step": 2578
    },
    {
      "epoch": 14.40782122905028,
      "grad_norm": 0.07373934083792344,
      "learning_rate": 1.4938170864468636e-07,
      "logits/chosen": -3.365337371826172,
      "logits/rejected": -3.5417628288269043,
      "logps/chosen": -0.33562973141670227,
      "logps/rejected": -80.64109802246094,
      "loss": 0.0749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3192451000213623,
      "rewards/margins": 2.3192451000213623,
      "rewards/rejected": 0.0,
      "step": 2579
    },
    {
      "epoch": 14.41340782122905,
      "grad_norm": 0.08441541326018585,
      "learning_rate": 1.483457295786045e-07,
      "logits/chosen": -3.3298568725585938,
      "logits/rejected": -3.3701534271240234,
      "logps/chosen": -0.058742716908454895,
      "logps/rejected": -114.47935485839844,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7969670295715332,
      "rewards/margins": 1.7969670295715332,
      "rewards/rejected": 0.0,
      "step": 2580
    },
    {
      "epoch": 14.41340782122905,
      "eval_logits/chosen": -3.3429760932922363,
      "eval_logits/rejected": -3.468442440032959,
      "eval_logps/chosen": -47.36948776245117,
      "eval_logps/rejected": -74.74958801269531,
      "eval_loss": 2.4738755226135254,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7299442291259766,
      "eval_rewards/margins": -1.7299442291259766,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6265,
      "eval_samples_per_second": 12.097,
      "eval_steps_per_second": 0.39,
      "step": 2580
    },
    {
      "epoch": 14.418994413407821,
      "grad_norm": 0.07578522234032789,
      "learning_rate": 1.473132454895751e-07,
      "logits/chosen": -3.5203044414520264,
      "logits/rejected": -3.4596407413482666,
      "logps/chosen": -3.910560131072998,
      "logps/rejected": -68.02816772460938,
      "loss": 0.076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.492376327514648,
      "rewards/margins": 4.492376327514648,
      "rewards/rejected": 0.0,
      "step": 2581
    },
    {
      "epoch": 14.424581005586592,
      "grad_norm": 0.06582248943063825,
      "learning_rate": 1.462842579120538e-07,
      "logits/chosen": -3.5192108154296875,
      "logits/rejected": -3.586561441421509,
      "logps/chosen": -0.1896495670080185,
      "logps/rejected": -62.026161193847656,
      "loss": 0.0828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.496875286102295,
      "rewards/margins": 2.496875286102295,
      "rewards/rejected": 0.0,
      "step": 2582
    },
    {
      "epoch": 14.430167597765363,
      "grad_norm": 0.12042837919496653,
      "learning_rate": 1.4525876837529872e-07,
      "logits/chosen": -3.3460960388183594,
      "logits/rejected": -3.3318324089050293,
      "logps/chosen": -0.07539049535989761,
      "logps/rejected": -160.94198608398438,
      "loss": 0.0781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.253368377685547,
      "rewards/margins": 2.253368377685547,
      "rewards/rejected": 0.0,
      "step": 2583
    },
    {
      "epoch": 14.435754189944134,
      "grad_norm": 0.1322497432155837,
      "learning_rate": 1.4423677840337047e-07,
      "logits/chosen": -3.5443003177642822,
      "logits/rejected": -3.575939893722534,
      "logps/chosen": -0.20412245392799377,
      "logps/rejected": -64.3873291015625,
      "loss": 0.0844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0160598754882812,
      "rewards/margins": 3.0160598754882812,
      "rewards/rejected": 0.0,
      "step": 2584
    },
    {
      "epoch": 14.441340782122905,
      "grad_norm": 0.0673136618938892,
      "learning_rate": 1.4321828951512822e-07,
      "logits/chosen": -3.5089728832244873,
      "logits/rejected": -3.565366744995117,
      "logps/chosen": -0.9091699719429016,
      "logps/rejected": -91.00198364257812,
      "loss": 0.0704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2481818199157715,
      "rewards/margins": 3.2481818199157715,
      "rewards/rejected": 0.0,
      "step": 2585
    },
    {
      "epoch": 14.446927374301676,
      "grad_norm": 0.07060328894188236,
      "learning_rate": 1.422033032242273e-07,
      "logits/chosen": -3.427642822265625,
      "logits/rejected": -3.535456895828247,
      "logps/chosen": -0.1371862292289734,
      "logps/rejected": -84.53546142578125,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1193323135375977,
      "rewards/margins": 2.1193323135375977,
      "rewards/rejected": 0.0,
      "step": 2586
    },
    {
      "epoch": 14.452513966480447,
      "grad_norm": 0.05718542349374118,
      "learning_rate": 1.411918210391186e-07,
      "logits/chosen": -3.2205755710601807,
      "logits/rejected": -3.1049375534057617,
      "logps/chosen": -16.401416778564453,
      "logps/rejected": -83.85708618164062,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.261296272277832,
      "rewards/margins": 3.261296272277832,
      "rewards/rejected": 0.0,
      "step": 2587
    },
    {
      "epoch": 14.458100558659218,
      "grad_norm": 0.06984223680878394,
      "learning_rate": 1.401838444630446e-07,
      "logits/chosen": -3.5347201824188232,
      "logits/rejected": -3.3737621307373047,
      "logps/chosen": -3.0213863849639893,
      "logps/rejected": -60.221534729003906,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2954442501068115,
      "rewards/margins": 3.2954442501068115,
      "rewards/rejected": 0.0,
      "step": 2588
    },
    {
      "epoch": 14.46368715083799,
      "grad_norm": 0.07647582827781133,
      "learning_rate": 1.3917937499403788e-07,
      "logits/chosen": -3.2113921642303467,
      "logits/rejected": -3.2245078086853027,
      "logps/chosen": -2.060941219329834,
      "logps/rejected": -112.8257827758789,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8452658653259277,
      "rewards/margins": 3.8452658653259277,
      "rewards/rejected": 0.0,
      "step": 2589
    },
    {
      "epoch": 14.46927374301676,
      "grad_norm": 0.09209427908106865,
      "learning_rate": 1.3817841412491873e-07,
      "logits/chosen": -3.062732696533203,
      "logits/rejected": -3.2233948707580566,
      "logps/chosen": -6.071165084838867,
      "logps/rejected": -73.3856430053711,
      "loss": 0.0827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.149545192718506,
      "rewards/margins": 4.149545192718506,
      "rewards/rejected": 0.0,
      "step": 2590
    },
    {
      "epoch": 14.474860335195531,
      "grad_norm": 0.10768704057498271,
      "learning_rate": 1.371809633432944e-07,
      "logits/chosen": -3.1780784130096436,
      "logits/rejected": -3.200840711593628,
      "logps/chosen": -1.0536103248596191,
      "logps/rejected": -97.46714782714844,
      "loss": 0.1111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.255228281021118,
      "rewards/margins": 3.255228281021118,
      "rewards/rejected": 0.0,
      "step": 2591
    },
    {
      "epoch": 14.480446927374302,
      "grad_norm": 0.08915463260073055,
      "learning_rate": 1.3618702413155299e-07,
      "logits/chosen": -3.4323949813842773,
      "logits/rejected": -3.5986506938934326,
      "logps/chosen": -1.1817846298217773,
      "logps/rejected": -83.2958984375,
      "loss": 0.0737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8584084510803223,
      "rewards/margins": 3.8584084510803223,
      "rewards/rejected": 0.0,
      "step": 2592
    },
    {
      "epoch": 14.486033519553073,
      "grad_norm": 0.07054311842572435,
      "learning_rate": 1.3519659796686623e-07,
      "logits/chosen": -3.5474748611450195,
      "logits/rejected": -3.452868700027466,
      "logps/chosen": -0.48985755443573,
      "logps/rejected": -72.26704406738281,
      "loss": 0.1054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.605041980743408,
      "rewards/margins": 3.605041980743408,
      "rewards/rejected": 0.0,
      "step": 2593
    },
    {
      "epoch": 14.491620111731844,
      "grad_norm": 0.08547905008276417,
      "learning_rate": 1.342096863211828e-07,
      "logits/chosen": -3.5042128562927246,
      "logits/rejected": -3.283742904663086,
      "logps/chosen": -1.6115163564682007,
      "logps/rejected": -94.72529602050781,
      "loss": 0.0956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.608293056488037,
      "rewards/margins": 2.608293056488037,
      "rewards/rejected": 0.0,
      "step": 2594
    },
    {
      "epoch": 14.497206703910614,
      "grad_norm": 0.06414790224325509,
      "learning_rate": 1.3322629066123e-07,
      "logits/chosen": -3.6285667419433594,
      "logits/rejected": -3.5495502948760986,
      "logps/chosen": -0.1786433607339859,
      "logps/rejected": -68.10738372802734,
      "loss": 0.1021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0756349563598633,
      "rewards/margins": 3.0756349563598633,
      "rewards/rejected": 0.0,
      "step": 2595
    },
    {
      "epoch": 14.502793296089386,
      "grad_norm": 0.05341029105647344,
      "learning_rate": 1.3224641244850873e-07,
      "logits/chosen": -3.5933358669281006,
      "logits/rejected": -3.600435972213745,
      "logps/chosen": -0.1139393299818039,
      "logps/rejected": -79.21420288085938,
      "loss": 0.0988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3094069957733154,
      "rewards/margins": 2.3094069957733154,
      "rewards/rejected": 0.0,
      "step": 2596
    },
    {
      "epoch": 14.508379888268156,
      "grad_norm": 0.1707674677726083,
      "learning_rate": 1.3127005313929215e-07,
      "logits/chosen": -3.6307599544525146,
      "logits/rejected": -3.443329095840454,
      "logps/chosen": -0.02352992445230484,
      "logps/rejected": -75.89917755126953,
      "loss": 0.1215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6138557195663452,
      "rewards/margins": 1.6138557195663452,
      "rewards/rejected": 0.0,
      "step": 2597
    },
    {
      "epoch": 14.513966480446927,
      "grad_norm": 0.10373485621549917,
      "learning_rate": 1.3029721418462454e-07,
      "logits/chosen": -3.379626512527466,
      "logits/rejected": -3.3297088146209717,
      "logps/chosen": -0.6566970348358154,
      "logps/rejected": -84.28819274902344,
      "loss": 0.091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2794528007507324,
      "rewards/margins": 3.2794528007507324,
      "rewards/rejected": 0.0,
      "step": 2598
    },
    {
      "epoch": 14.519553072625698,
      "grad_norm": 0.06423937369277799,
      "learning_rate": 1.2932789703031746e-07,
      "logits/chosen": -3.530385971069336,
      "logits/rejected": -3.4926130771636963,
      "logps/chosen": -0.06337545812129974,
      "logps/rejected": -53.490257263183594,
      "loss": 0.0905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.199716806411743,
      "rewards/margins": 2.199716806411743,
      "rewards/rejected": 0.0,
      "step": 2599
    },
    {
      "epoch": 14.525139664804469,
      "grad_norm": 0.24299733019647315,
      "learning_rate": 1.2836210311694936e-07,
      "logits/chosen": -3.4721124172210693,
      "logits/rejected": -3.6074414253234863,
      "logps/chosen": -0.19456113874912262,
      "logps/rejected": -92.68525695800781,
      "loss": 0.0799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7687079906463623,
      "rewards/margins": 2.7687079906463623,
      "rewards/rejected": 0.0,
      "step": 2600
    },
    {
      "epoch": 14.525139664804469,
      "eval_logits/chosen": -3.346135377883911,
      "eval_logits/rejected": -3.4710068702697754,
      "eval_logps/chosen": -47.481224060058594,
      "eval_logps/rejected": -75.2069091796875,
      "eval_loss": 2.482699394226074,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7411178350448608,
      "eval_rewards/margins": -1.7411178350448608,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6637,
      "eval_samples_per_second": 12.079,
      "eval_steps_per_second": 0.39,
      "step": 2600
    },
    {
      "epoch": 14.53072625698324,
      "grad_norm": 0.06495327066168807,
      "learning_rate": 1.27399833879861e-07,
      "logits/chosen": -3.5416831970214844,
      "logits/rejected": -3.5689384937286377,
      "logps/chosen": -0.028855998069047928,
      "logps/rejected": -68.96515655517578,
      "loss": 0.1066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6751576662063599,
      "rewards/margins": 1.6751576662063599,
      "rewards/rejected": 0.0,
      "step": 2601
    },
    {
      "epoch": 14.53631284916201,
      "grad_norm": 0.06689999698767533,
      "learning_rate": 1.2644109074915702e-07,
      "logits/chosen": -2.858088731765747,
      "logits/rejected": -2.896815776824951,
      "logps/chosen": -30.291173934936523,
      "logps/rejected": -57.86850357055664,
      "loss": 0.0583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.070109844207764,
      "rewards/margins": 5.070109844207764,
      "rewards/rejected": 0.0,
      "step": 2602
    },
    {
      "epoch": 14.541899441340782,
      "grad_norm": 0.05199243442389231,
      "learning_rate": 1.2548587514969956e-07,
      "logits/chosen": -3.495915412902832,
      "logits/rejected": -3.4499809741973877,
      "logps/chosen": -1.064984917640686,
      "logps/rejected": -68.33692169189453,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.760465145111084,
      "rewards/margins": 2.760465145111084,
      "rewards/rejected": 0.0,
      "step": 2603
    },
    {
      "epoch": 14.547486033519553,
      "grad_norm": 0.06973289279455157,
      "learning_rate": 1.245341885011095e-07,
      "logits/chosen": -3.3653347492218018,
      "logits/rejected": -3.336766004562378,
      "logps/chosen": -11.929577827453613,
      "logps/rejected": -104.96206665039062,
      "loss": 0.1063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.716737747192383,
      "rewards/margins": 4.716737747192383,
      "rewards/rejected": 0.0,
      "step": 2604
    },
    {
      "epoch": 14.553072625698324,
      "grad_norm": 0.04910350842288955,
      "learning_rate": 1.235860322177629e-07,
      "logits/chosen": -3.314415693283081,
      "logits/rejected": -3.2730624675750732,
      "logps/chosen": -2.8860576152801514,
      "logps/rejected": -69.026611328125,
      "loss": 0.0888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6454014778137207,
      "rewards/margins": 3.6454014778137207,
      "rewards/rejected": 0.0,
      "step": 2605
    },
    {
      "epoch": 14.558659217877095,
      "grad_norm": 0.06169721630697898,
      "learning_rate": 1.226414077087884e-07,
      "logits/chosen": -3.7138872146606445,
      "logits/rejected": -3.6684582233428955,
      "logps/chosen": -0.26720130443573,
      "logps/rejected": -58.233890533447266,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0274887084960938,
      "rewards/margins": 3.0274887084960938,
      "rewards/rejected": 0.0,
      "step": 2606
    },
    {
      "epoch": 14.564245810055866,
      "grad_norm": 0.06733394754296279,
      "learning_rate": 1.2170031637806666e-07,
      "logits/chosen": -3.2943501472473145,
      "logits/rejected": -3.352858304977417,
      "logps/chosen": -0.15617495775222778,
      "logps/rejected": -52.88414764404297,
      "loss": 0.1194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.14019775390625,
      "rewards/margins": 2.14019775390625,
      "rewards/rejected": 0.0,
      "step": 2607
    },
    {
      "epoch": 14.569832402234637,
      "grad_norm": 0.07945268491474473,
      "learning_rate": 1.2076275962422668e-07,
      "logits/chosen": -3.5958192348480225,
      "logits/rejected": -3.4434046745300293,
      "logps/chosen": -6.617356777191162,
      "logps/rejected": -64.54447937011719,
      "loss": 0.0986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.937424182891846,
      "rewards/margins": 4.937424182891846,
      "rewards/rejected": 0.0,
      "step": 2608
    },
    {
      "epoch": 14.575418994413408,
      "grad_norm": 0.06803477706345462,
      "learning_rate": 1.1982873884064466e-07,
      "logits/chosen": -3.585059404373169,
      "logits/rejected": -3.6313860416412354,
      "logps/chosen": -0.2317335605621338,
      "logps/rejected": -78.32341003417969,
      "loss": 0.09,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.816413402557373,
      "rewards/margins": 2.816413402557373,
      "rewards/rejected": 0.0,
      "step": 2609
    },
    {
      "epoch": 14.581005586592179,
      "grad_norm": 0.05848898762186865,
      "learning_rate": 1.1889825541544192e-07,
      "logits/chosen": -3.5138089656829834,
      "logits/rejected": -3.661830186843872,
      "logps/chosen": -0.025997797027230263,
      "logps/rejected": -79.28936767578125,
      "loss": 0.1186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8725581169128418,
      "rewards/margins": 1.8725581169128418,
      "rewards/rejected": 0.0,
      "step": 2610
    },
    {
      "epoch": 14.58659217877095,
      "grad_norm": 0.05466196703733353,
      "learning_rate": 1.1797131073148338e-07,
      "logits/chosen": -3.5453920364379883,
      "logits/rejected": -3.6538894176483154,
      "logps/chosen": -0.1973976045846939,
      "logps/rejected": -91.31842041015625,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6543917655944824,
      "rewards/margins": 2.6543917655944824,
      "rewards/rejected": 0.0,
      "step": 2611
    },
    {
      "epoch": 14.59217877094972,
      "grad_norm": 0.07370575774420754,
      "learning_rate": 1.1704790616637235e-07,
      "logits/chosen": -3.407444477081299,
      "logits/rejected": -3.4567854404449463,
      "logps/chosen": -0.07540037482976913,
      "logps/rejected": -109.11024475097656,
      "loss": 0.1034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4017632007598877,
      "rewards/margins": 2.4017632007598877,
      "rewards/rejected": 0.0,
      "step": 2612
    },
    {
      "epoch": 14.597765363128492,
      "grad_norm": 0.14489272256149655,
      "learning_rate": 1.1612804309245385e-07,
      "logits/chosen": -3.6931216716766357,
      "logits/rejected": -3.764753818511963,
      "logps/chosen": -3.2332754135131836,
      "logps/rejected": -43.3342399597168,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.335442543029785,
      "rewards/margins": 3.335442543029785,
      "rewards/rejected": 0.0,
      "step": 2613
    },
    {
      "epoch": 14.603351955307263,
      "grad_norm": 0.1202684194440469,
      "learning_rate": 1.1521172287680765e-07,
      "logits/chosen": -3.4050259590148926,
      "logits/rejected": -3.4243550300598145,
      "logps/chosen": -0.3130713701248169,
      "logps/rejected": -59.4523811340332,
      "loss": 0.0885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6001055240631104,
      "rewards/margins": 2.6001055240631104,
      "rewards/rejected": 0.0,
      "step": 2614
    },
    {
      "epoch": 14.608938547486034,
      "grad_norm": 0.10597712427720471,
      "learning_rate": 1.1429894688124859e-07,
      "logits/chosen": -3.304499387741089,
      "logits/rejected": -3.2872536182403564,
      "logps/chosen": -0.16215583682060242,
      "logps/rejected": -80.24067687988281,
      "loss": 0.0782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3922858238220215,
      "rewards/margins": 2.3922858238220215,
      "rewards/rejected": 0.0,
      "step": 2615
    },
    {
      "epoch": 14.614525139664805,
      "grad_norm": 0.15827517866715896,
      "learning_rate": 1.1338971646232489e-07,
      "logits/chosen": -3.545638084411621,
      "logits/rejected": -3.5647149085998535,
      "logps/chosen": -0.1700972020626068,
      "logps/rejected": -102.33625030517578,
      "loss": 0.087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.756819248199463,
      "rewards/margins": 2.756819248199463,
      "rewards/rejected": 0.0,
      "step": 2616
    },
    {
      "epoch": 14.620111731843576,
      "grad_norm": 0.16207880867375152,
      "learning_rate": 1.1248403297131421e-07,
      "logits/chosen": -3.368192195892334,
      "logits/rejected": -3.23091721534729,
      "logps/chosen": -1.1771401166915894,
      "logps/rejected": -88.04542541503906,
      "loss": 0.1139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8200724124908447,
      "rewards/margins": 2.8200724124908447,
      "rewards/rejected": 0.0,
      "step": 2617
    },
    {
      "epoch": 14.625698324022347,
      "grad_norm": 0.11122833312394394,
      "learning_rate": 1.1158189775422407e-07,
      "logits/chosen": -3.3079264163970947,
      "logits/rejected": -3.4250168800354004,
      "logps/chosen": -10.751974105834961,
      "logps/rejected": -79.64700317382812,
      "loss": 0.1024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.299909591674805,
      "rewards/margins": 4.299909591674805,
      "rewards/rejected": 0.0,
      "step": 2618
    },
    {
      "epoch": 14.631284916201118,
      "grad_norm": 0.272404915750668,
      "learning_rate": 1.1068331215178752e-07,
      "logits/chosen": -3.194859266281128,
      "logits/rejected": -3.524075746536255,
      "logps/chosen": -0.5779706835746765,
      "logps/rejected": -91.58537292480469,
      "loss": 0.1353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.002816677093506,
      "rewards/margins": 2.002816677093506,
      "rewards/rejected": 0.0,
      "step": 2619
    },
    {
      "epoch": 14.636871508379889,
      "grad_norm": 0.07150443880391816,
      "learning_rate": 1.097882774994638e-07,
      "logits/chosen": -3.368964672088623,
      "logits/rejected": -3.267707109451294,
      "logps/chosen": -0.16600646078586578,
      "logps/rejected": -68.03684997558594,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8017001152038574,
      "rewards/margins": 2.8017001152038574,
      "rewards/rejected": 0.0,
      "step": 2620
    },
    {
      "epoch": 14.636871508379889,
      "eval_logits/chosen": -3.3458588123321533,
      "eval_logits/rejected": -3.470435380935669,
      "eval_logps/chosen": -47.39256286621094,
      "eval_logps/rejected": -75.0083236694336,
      "eval_loss": 2.4753599166870117,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7322521209716797,
      "eval_rewards/margins": -1.7322521209716797,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5701,
      "eval_samples_per_second": 12.124,
      "eval_steps_per_second": 0.391,
      "step": 2620
    },
    {
      "epoch": 14.64245810055866,
      "grad_norm": 0.13466775724257504,
      "learning_rate": 1.088967951274325e-07,
      "logits/chosen": -3.5946295261383057,
      "logits/rejected": -3.791116952896118,
      "logps/chosen": -1.434589147567749,
      "logps/rejected": -71.76211547851562,
      "loss": 0.1035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.476717948913574,
      "rewards/margins": 3.476717948913574,
      "rewards/rejected": 0.0,
      "step": 2621
    },
    {
      "epoch": 14.64804469273743,
      "grad_norm": 0.06654801976208459,
      "learning_rate": 1.080088663605966e-07,
      "logits/chosen": -3.512598752975464,
      "logits/rejected": -3.383091926574707,
      "logps/chosen": -0.12942200899124146,
      "logps/rejected": -76.5213623046875,
      "loss": 0.096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.503091335296631,
      "rewards/margins": 2.503091335296631,
      "rewards/rejected": 0.0,
      "step": 2622
    },
    {
      "epoch": 14.653631284916202,
      "grad_norm": 0.12208550821693923,
      "learning_rate": 1.0712449251857493e-07,
      "logits/chosen": -3.6037330627441406,
      "logits/rejected": -3.653477668762207,
      "logps/chosen": -0.25987350940704346,
      "logps/rejected": -64.83638000488281,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7501866817474365,
      "rewards/margins": 2.7501866817474365,
      "rewards/rejected": 0.0,
      "step": 2623
    },
    {
      "epoch": 14.659217877094973,
      "grad_norm": 0.055131842000041405,
      "learning_rate": 1.0624367491570531e-07,
      "logits/chosen": -3.1469919681549072,
      "logits/rejected": -3.213998794555664,
      "logps/chosen": -0.7240663170814514,
      "logps/rejected": -57.863285064697266,
      "loss": 0.0884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9318222999572754,
      "rewards/margins": 2.9318222999572754,
      "rewards/rejected": 0.0,
      "step": 2624
    },
    {
      "epoch": 14.664804469273744,
      "grad_norm": 0.06567629064230661,
      "learning_rate": 1.053664148610395e-07,
      "logits/chosen": -3.517260789871216,
      "logits/rejected": -3.4059646129608154,
      "logps/chosen": -2.2850539684295654,
      "logps/rejected": -96.35608673095703,
      "loss": 0.1208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.531297206878662,
      "rewards/margins": 3.531297206878662,
      "rewards/rejected": 0.0,
      "step": 2625
    },
    {
      "epoch": 14.670391061452515,
      "grad_norm": 0.06372259336412726,
      "learning_rate": 1.0449271365834208e-07,
      "logits/chosen": -3.5848896503448486,
      "logits/rejected": -3.613410234451294,
      "logps/chosen": -0.06434054672718048,
      "logps/rejected": -57.23888397216797,
      "loss": 0.1138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2051987648010254,
      "rewards/margins": 2.2051987648010254,
      "rewards/rejected": 0.0,
      "step": 2626
    },
    {
      "epoch": 14.675977653631286,
      "grad_norm": 0.1288710278539372,
      "learning_rate": 1.0362257260608882e-07,
      "logits/chosen": -3.2837536334991455,
      "logits/rejected": -3.3601667881011963,
      "logps/chosen": -0.03569909557700157,
      "logps/rejected": -70.64374542236328,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6739699840545654,
      "rewards/margins": 1.6739699840545654,
      "rewards/rejected": 0.0,
      "step": 2627
    },
    {
      "epoch": 14.681564245810057,
      "grad_norm": 0.09430166046580284,
      "learning_rate": 1.0275599299746419e-07,
      "logits/chosen": -3.4433157444000244,
      "logits/rejected": -3.6257472038269043,
      "logps/chosen": -3.794346809387207,
      "logps/rejected": -59.453651428222656,
      "loss": 0.0876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.033989906311035,
      "rewards/margins": 4.033989906311035,
      "rewards/rejected": 0.0,
      "step": 2628
    },
    {
      "epoch": 14.687150837988828,
      "grad_norm": 0.1137866548434106,
      "learning_rate": 1.0189297612035964e-07,
      "logits/chosen": -3.2459001541137695,
      "logits/rejected": -3.204583168029785,
      "logps/chosen": -1.905285120010376,
      "logps/rejected": -46.09019470214844,
      "loss": 0.0854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.808163166046143,
      "rewards/margins": 4.808163166046143,
      "rewards/rejected": 0.0,
      "step": 2629
    },
    {
      "epoch": 14.692737430167599,
      "grad_norm": 0.05923121109918802,
      "learning_rate": 1.010335232573717e-07,
      "logits/chosen": -3.328399419784546,
      "logits/rejected": -3.3373606204986572,
      "logps/chosen": -0.2736271321773529,
      "logps/rejected": -53.43663024902344,
      "loss": 0.0954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6785402297973633,
      "rewards/margins": 2.6785402297973633,
      "rewards/rejected": 0.0,
      "step": 2630
    },
    {
      "epoch": 14.69832402234637,
      "grad_norm": 0.09329972724908436,
      "learning_rate": 1.0017763568580142e-07,
      "logits/chosen": -3.523043155670166,
      "logits/rejected": -3.3812344074249268,
      "logps/chosen": -1.2626618146896362,
      "logps/rejected": -47.99482727050781,
      "loss": 0.0786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4588637351989746,
      "rewards/margins": 3.4588637351989746,
      "rewards/rejected": 0.0,
      "step": 2631
    },
    {
      "epoch": 14.703910614525139,
      "grad_norm": 0.22669716811898136,
      "learning_rate": 9.932531467764905e-08,
      "logits/chosen": -3.5106489658355713,
      "logits/rejected": -3.473675489425659,
      "logps/chosen": -1.0356266498565674,
      "logps/rejected": -85.11006164550781,
      "loss": 0.1012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5531091690063477,
      "rewards/margins": 3.5531091690063477,
      "rewards/rejected": 0.0,
      "step": 2632
    },
    {
      "epoch": 14.70949720670391,
      "grad_norm": 0.05618902340832726,
      "learning_rate": 9.847656149961582e-08,
      "logits/chosen": -3.59480881690979,
      "logits/rejected": -3.533278226852417,
      "logps/chosen": -0.12047456204891205,
      "logps/rejected": -104.28570556640625,
      "loss": 0.1227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.091501235961914,
      "rewards/margins": 2.091501235961914,
      "rewards/rejected": 0.0,
      "step": 2633
    },
    {
      "epoch": 14.71508379888268,
      "grad_norm": 0.05873797627236693,
      "learning_rate": 9.763137741310047e-08,
      "logits/chosen": -3.4859795570373535,
      "logits/rejected": -3.3338239192962646,
      "logps/chosen": -1.0505775213241577,
      "logps/rejected": -86.0008316040039,
      "loss": 0.0705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6677308082580566,
      "rewards/margins": 3.6677308082580566,
      "rewards/rejected": 0.0,
      "step": 2634
    },
    {
      "epoch": 14.720670391061452,
      "grad_norm": 0.0706228172918715,
      "learning_rate": 9.678976367419657e-08,
      "logits/chosen": -3.0917398929595947,
      "logits/rejected": -2.9784817695617676,
      "logps/chosen": -22.601089477539062,
      "logps/rejected": -83.79568481445312,
      "loss": 0.0931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.430174827575684,
      "rewards/margins": 5.430174827575684,
      "rewards/rejected": 0.0,
      "step": 2635
    },
    {
      "epoch": 14.726256983240223,
      "grad_norm": 0.06008321292167997,
      "learning_rate": 9.59517215336922e-08,
      "logits/chosen": -3.624481439590454,
      "logits/rejected": -3.5343544483184814,
      "logps/chosen": -0.2945241630077362,
      "logps/rejected": -79.02870178222656,
      "loss": 0.0863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5602242946624756,
      "rewards/margins": 2.5602242946624756,
      "rewards/rejected": 0.0,
      "step": 2636
    },
    {
      "epoch": 14.731843575418994,
      "grad_norm": 0.06725890719944953,
      "learning_rate": 9.511725223706719e-08,
      "logits/chosen": -3.4666054248809814,
      "logits/rejected": -3.6201088428497314,
      "logps/chosen": -0.09248079359531403,
      "logps/rejected": -82.14547729492188,
      "loss": 0.0652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.045114517211914,
      "rewards/margins": 2.045114517211914,
      "rewards/rejected": 0.0,
      "step": 2637
    },
    {
      "epoch": 14.737430167597765,
      "grad_norm": 0.11656630097370878,
      "learning_rate": 9.428635702449145e-08,
      "logits/chosen": -3.4363443851470947,
      "logits/rejected": -3.615079641342163,
      "logps/chosen": -0.02577294409275055,
      "logps/rejected": -130.27716064453125,
      "loss": 0.1291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3656597137451172,
      "rewards/margins": 1.3656597137451172,
      "rewards/rejected": 0.0,
      "step": 2638
    },
    {
      "epoch": 14.743016759776536,
      "grad_norm": 0.06964071729707114,
      "learning_rate": 9.345903713082305e-08,
      "logits/chosen": -3.6492671966552734,
      "logits/rejected": -3.5973997116088867,
      "logps/chosen": -8.540973663330078,
      "logps/rejected": -72.86698913574219,
      "loss": 0.0761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7155160903930664,
      "rewards/margins": 3.7155160903930664,
      "rewards/rejected": 0.0,
      "step": 2639
    },
    {
      "epoch": 14.748603351955307,
      "grad_norm": 0.09536239245131492,
      "learning_rate": 9.263529378560788e-08,
      "logits/chosen": -3.610262870788574,
      "logits/rejected": -3.5260281562805176,
      "logps/chosen": -1.4583308696746826,
      "logps/rejected": -74.21035766601562,
      "loss": 0.0752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7441887855529785,
      "rewards/margins": 3.7441887855529785,
      "rewards/rejected": 0.0,
      "step": 2640
    },
    {
      "epoch": 14.748603351955307,
      "eval_logits/chosen": -3.3422389030456543,
      "eval_logits/rejected": -3.4682681560516357,
      "eval_logps/chosen": -47.63288116455078,
      "eval_logps/rejected": -75.13614654541016,
      "eval_loss": 2.487725257873535,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7562835216522217,
      "eval_rewards/margins": -1.7562835216522217,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6174,
      "eval_samples_per_second": 12.101,
      "eval_steps_per_second": 0.39,
      "step": 2640
    },
    {
      "epoch": 14.754189944134078,
      "grad_norm": 0.07533978057945347,
      "learning_rate": 9.181512821307331e-08,
      "logits/chosen": -3.4110636711120605,
      "logits/rejected": -3.5582003593444824,
      "logps/chosen": -0.08827461302280426,
      "logps/rejected": -114.66342163085938,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6376378536224365,
      "rewards/margins": 1.6376378536224365,
      "rewards/rejected": 0.0,
      "step": 2641
    },
    {
      "epoch": 14.759776536312849,
      "grad_norm": 0.07465440507947627,
      "learning_rate": 9.099854163213323e-08,
      "logits/chosen": -3.6079976558685303,
      "logits/rejected": -3.6273248195648193,
      "logps/chosen": -0.0392402783036232,
      "logps/rejected": -87.34199523925781,
      "loss": 0.117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.754267930984497,
      "rewards/margins": 1.754267930984497,
      "rewards/rejected": 0.0,
      "step": 2642
    },
    {
      "epoch": 14.76536312849162,
      "grad_norm": 0.0699192083260912,
      "learning_rate": 9.018553525637963e-08,
      "logits/chosen": -3.6556737422943115,
      "logits/rejected": -3.557772636413574,
      "logps/chosen": -0.7348961234092712,
      "logps/rejected": -84.02926635742188,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5943751335144043,
      "rewards/margins": 3.5943751335144043,
      "rewards/rejected": 0.0,
      "step": 2643
    },
    {
      "epoch": 14.77094972067039,
      "grad_norm": 0.11680103754457195,
      "learning_rate": 8.937611029408516e-08,
      "logits/chosen": -3.555018186569214,
      "logits/rejected": -3.384613037109375,
      "logps/chosen": -1.3891406059265137,
      "logps/rejected": -67.84953308105469,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.604404926300049,
      "rewards/margins": 3.604404926300049,
      "rewards/rejected": 0.0,
      "step": 2644
    },
    {
      "epoch": 14.776536312849162,
      "grad_norm": 0.08561932042254783,
      "learning_rate": 8.857026794819951e-08,
      "logits/chosen": -3.6449930667877197,
      "logits/rejected": -3.6480629444122314,
      "logps/chosen": -1.1543183326721191,
      "logps/rejected": -47.816165924072266,
      "loss": 0.1203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4715805053710938,
      "rewards/margins": 3.4715805053710938,
      "rewards/rejected": 0.0,
      "step": 2645
    },
    {
      "epoch": 14.782122905027933,
      "grad_norm": 0.11960117964550036,
      "learning_rate": 8.7768009416348e-08,
      "logits/chosen": -3.534020185470581,
      "logits/rejected": -3.375974178314209,
      "logps/chosen": -1.1405922174453735,
      "logps/rejected": -119.12454223632812,
      "loss": 0.1031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.363970994949341,
      "rewards/margins": 3.363970994949341,
      "rewards/rejected": 0.0,
      "step": 2646
    },
    {
      "epoch": 14.787709497206704,
      "grad_norm": 0.12273639861444587,
      "learning_rate": 8.696933589082912e-08,
      "logits/chosen": -3.484541654586792,
      "logits/rejected": -3.570465087890625,
      "logps/chosen": -0.11714895069599152,
      "logps/rejected": -80.4317626953125,
      "loss": 0.114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.351327896118164,
      "rewards/margins": 2.351327896118164,
      "rewards/rejected": 0.0,
      "step": 2647
    },
    {
      "epoch": 14.793296089385475,
      "grad_norm": 0.06830685230010003,
      "learning_rate": 8.617424855861423e-08,
      "logits/chosen": -3.6699135303497314,
      "logits/rejected": -3.6187894344329834,
      "logps/chosen": -0.2920590341091156,
      "logps/rejected": -119.6656494140625,
      "loss": 0.0796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.728156089782715,
      "rewards/margins": 2.728156089782715,
      "rewards/rejected": 0.0,
      "step": 2648
    },
    {
      "epoch": 14.798882681564246,
      "grad_norm": 0.2700215059465368,
      "learning_rate": 8.53827486013456e-08,
      "logits/chosen": -3.3204617500305176,
      "logits/rejected": -3.3800442218780518,
      "logps/chosen": -0.045188046991825104,
      "logps/rejected": -84.39704895019531,
      "loss": 0.1322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8734195232391357,
      "rewards/margins": 1.8734195232391357,
      "rewards/rejected": 0.0,
      "step": 2649
    },
    {
      "epoch": 14.804469273743017,
      "grad_norm": 0.0933587446384116,
      "learning_rate": 8.459483719533174e-08,
      "logits/chosen": -3.4048595428466797,
      "logits/rejected": -3.4742562770843506,
      "logps/chosen": -0.6673104166984558,
      "logps/rejected": -78.40400695800781,
      "loss": 0.0939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1480531692504883,
      "rewards/margins": 3.1480531692504883,
      "rewards/rejected": 0.0,
      "step": 2650
    },
    {
      "epoch": 14.810055865921788,
      "grad_norm": 0.14223540566346363,
      "learning_rate": 8.381051551155039e-08,
      "logits/chosen": -3.6089019775390625,
      "logits/rejected": -3.035285472869873,
      "logps/chosen": -0.23008176684379578,
      "logps/rejected": -187.97396850585938,
      "loss": 0.1027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.143134593963623,
      "rewards/margins": 2.143134593963623,
      "rewards/rejected": 0.0,
      "step": 2651
    },
    {
      "epoch": 14.815642458100559,
      "grad_norm": 0.06727459893231484,
      "learning_rate": 8.302978471564221e-08,
      "logits/chosen": -3.56488299369812,
      "logits/rejected": -3.583134889602661,
      "logps/chosen": -0.41604286432266235,
      "logps/rejected": -111.74696350097656,
      "loss": 0.116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9591410160064697,
      "rewards/margins": 2.9591410160064697,
      "rewards/rejected": 0.0,
      "step": 2652
    },
    {
      "epoch": 14.82122905027933,
      "grad_norm": 0.07701518348299334,
      "learning_rate": 8.225264596791294e-08,
      "logits/chosen": -3.443305015563965,
      "logits/rejected": -3.605511426925659,
      "logps/chosen": -0.12965776026248932,
      "logps/rejected": -86.55521392822266,
      "loss": 0.12,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.17421293258667,
      "rewards/margins": 2.17421293258667,
      "rewards/rejected": 0.0,
      "step": 2653
    },
    {
      "epoch": 14.8268156424581,
      "grad_norm": 0.06107637479830724,
      "learning_rate": 8.147910042332924e-08,
      "logits/chosen": -3.728997230529785,
      "logits/rejected": -3.6401546001434326,
      "logps/chosen": -15.926197052001953,
      "logps/rejected": -66.34703063964844,
      "loss": 0.1084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9119303226470947,
      "rewards/margins": 2.9119303226470947,
      "rewards/rejected": 0.0,
      "step": 2654
    },
    {
      "epoch": 14.832402234636872,
      "grad_norm": 0.054943552367813,
      "learning_rate": 8.070914923151734e-08,
      "logits/chosen": -3.5528881549835205,
      "logits/rejected": -3.465118885040283,
      "logps/chosen": -0.09529821574687958,
      "logps/rejected": -77.11373138427734,
      "loss": 0.1022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.481663465499878,
      "rewards/margins": 2.481663465499878,
      "rewards/rejected": 0.0,
      "step": 2655
    },
    {
      "epoch": 14.837988826815643,
      "grad_norm": 0.1705242354628252,
      "learning_rate": 7.994279353676165e-08,
      "logits/chosen": -3.550952672958374,
      "logits/rejected": -3.522303819656372,
      "logps/chosen": -31.664588928222656,
      "logps/rejected": -92.17240905761719,
      "loss": 0.0955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 6.173187255859375,
      "rewards/margins": 6.173187255859375,
      "rewards/rejected": 0.0,
      "step": 2656
    },
    {
      "epoch": 14.843575418994414,
      "grad_norm": 0.11129511469055318,
      "learning_rate": 7.91800344780036e-08,
      "logits/chosen": -3.3968915939331055,
      "logits/rejected": -3.6042144298553467,
      "logps/chosen": -0.04132615402340889,
      "logps/rejected": -96.49880981445312,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4672287702560425,
      "rewards/margins": 1.4672287702560425,
      "rewards/rejected": 0.0,
      "step": 2657
    },
    {
      "epoch": 14.849162011173185,
      "grad_norm": 0.0628643059743185,
      "learning_rate": 7.842087318883862e-08,
      "logits/chosen": -3.6007611751556396,
      "logits/rejected": -3.569828748703003,
      "logps/chosen": -33.77226638793945,
      "logps/rejected": -106.77887725830078,
      "loss": 0.092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.324838638305664,
      "rewards/margins": 4.324838638305664,
      "rewards/rejected": 0.0,
      "step": 2658
    },
    {
      "epoch": 14.854748603351956,
      "grad_norm": 0.10985801342071894,
      "learning_rate": 7.766531079751532e-08,
      "logits/chosen": -3.7090210914611816,
      "logits/rejected": -3.656320810317993,
      "logps/chosen": -8.634781837463379,
      "logps/rejected": -128.2339630126953,
      "loss": 0.1011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.939038038253784,
      "rewards/margins": 2.939038038253784,
      "rewards/rejected": 0.0,
      "step": 2659
    },
    {
      "epoch": 14.860335195530727,
      "grad_norm": 0.06264740725236316,
      "learning_rate": 7.69133484269352e-08,
      "logits/chosen": -3.2198615074157715,
      "logits/rejected": -3.335296869277954,
      "logps/chosen": -9.646119117736816,
      "logps/rejected": -104.94625854492188,
      "loss": 0.0972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.900778293609619,
      "rewards/margins": 3.900778293609619,
      "rewards/rejected": 0.0,
      "step": 2660
    },
    {
      "epoch": 14.860335195530727,
      "eval_logits/chosen": -3.3415565490722656,
      "eval_logits/rejected": -3.4674975872039795,
      "eval_logps/chosen": -47.48728561401367,
      "eval_logps/rejected": -75.2568359375,
      "eval_loss": 2.493227958679199,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7417246103286743,
      "eval_rewards/margins": -1.7417246103286743,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6579,
      "eval_samples_per_second": 12.082,
      "eval_steps_per_second": 0.39,
      "step": 2660
    },
    {
      "epoch": 14.865921787709498,
      "grad_norm": 0.1425657168428455,
      "learning_rate": 7.616498719464654e-08,
      "logits/chosen": -3.187406539916992,
      "logits/rejected": -3.2696382999420166,
      "logps/chosen": -0.8430418968200684,
      "logps/rejected": -71.918212890625,
      "loss": 0.0891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3113296031951904,
      "rewards/margins": 2.3113296031951904,
      "rewards/rejected": 0.0,
      "step": 2661
    },
    {
      "epoch": 14.871508379888269,
      "grad_norm": 0.0769094999555517,
      "learning_rate": 7.542022821284883e-08,
      "logits/chosen": -3.561718225479126,
      "logits/rejected": -3.719122886657715,
      "logps/chosen": -28.937883377075195,
      "logps/rejected": -147.16522216796875,
      "loss": 0.0844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0690674781799316,
      "rewards/margins": 3.0690674781799316,
      "rewards/rejected": 0.0,
      "step": 2662
    },
    {
      "epoch": 14.87709497206704,
      "grad_norm": 0.0972045635559283,
      "learning_rate": 7.467907258838614e-08,
      "logits/chosen": -3.6359214782714844,
      "logits/rejected": -3.6314361095428467,
      "logps/chosen": -11.905926704406738,
      "logps/rejected": -79.72250366210938,
      "loss": 0.1256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8299429416656494,
      "rewards/margins": 2.8299429416656494,
      "rewards/rejected": 0.0,
      "step": 2663
    },
    {
      "epoch": 14.88268156424581,
      "grad_norm": 0.05631216596363446,
      "learning_rate": 7.394152142274735e-08,
      "logits/chosen": -3.4242868423461914,
      "logits/rejected": -3.470958948135376,
      "logps/chosen": -0.02999250218272209,
      "logps/rejected": -87.89634704589844,
      "loss": 0.0946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7633392810821533,
      "rewards/margins": 1.7633392810821533,
      "rewards/rejected": 0.0,
      "step": 2664
    },
    {
      "epoch": 14.888268156424582,
      "grad_norm": 0.2463763104843701,
      "learning_rate": 7.320757581206511e-08,
      "logits/chosen": -3.4373927116394043,
      "logits/rejected": -3.284749746322632,
      "logps/chosen": -2.7186105251312256,
      "logps/rejected": -115.5733413696289,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7186107635498047,
      "rewards/margins": 2.7186107635498047,
      "rewards/rejected": 0.0,
      "step": 2665
    },
    {
      "epoch": 14.893854748603353,
      "grad_norm": 0.061395793962294325,
      "learning_rate": 7.247723684711383e-08,
      "logits/chosen": -3.549309015274048,
      "logits/rejected": -3.40303111076355,
      "logps/chosen": -11.68626880645752,
      "logps/rejected": -79.62433624267578,
      "loss": 0.0823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.410923480987549,
      "rewards/margins": 4.410923480987549,
      "rewards/rejected": 0.0,
      "step": 2666
    },
    {
      "epoch": 14.899441340782122,
      "grad_norm": 0.10562386805564088,
      "learning_rate": 7.17505056133061e-08,
      "logits/chosen": -3.479433298110962,
      "logits/rejected": -3.6191952228546143,
      "logps/chosen": -0.11753205955028534,
      "logps/rejected": -108.7557144165039,
      "loss": 0.089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4644861221313477,
      "rewards/margins": 2.4644861221313477,
      "rewards/rejected": 0.0,
      "step": 2667
    },
    {
      "epoch": 14.905027932960895,
      "grad_norm": 0.0693978765947422,
      "learning_rate": 7.102738319069464e-08,
      "logits/chosen": -3.3882253170013428,
      "logits/rejected": -3.371307611465454,
      "logps/chosen": -0.036642685532569885,
      "logps/rejected": -98.25254821777344,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7713758945465088,
      "rewards/margins": 1.7713758945465088,
      "rewards/rejected": 0.0,
      "step": 2668
    },
    {
      "epoch": 14.910614525139664,
      "grad_norm": 0.08114447161344256,
      "learning_rate": 7.030787065396866e-08,
      "logits/chosen": -3.396533250808716,
      "logits/rejected": -3.4911179542541504,
      "logps/chosen": -2.572953939437866,
      "logps/rejected": -78.99909973144531,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.01948356628418,
      "rewards/margins": 4.01948356628418,
      "rewards/rejected": 0.0,
      "step": 2669
    },
    {
      "epoch": 14.916201117318435,
      "grad_norm": 0.09121046636145479,
      "learning_rate": 6.959196907245086e-08,
      "logits/chosen": -3.7260522842407227,
      "logits/rejected": -3.7704861164093018,
      "logps/chosen": -10.816338539123535,
      "logps/rejected": -95.3752670288086,
      "loss": 0.0679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.2229180335998535,
      "rewards/margins": 4.2229180335998535,
      "rewards/rejected": 0.0,
      "step": 2670
    },
    {
      "epoch": 14.921787709497206,
      "grad_norm": 0.06621109882637734,
      "learning_rate": 6.887967951009905e-08,
      "logits/chosen": -3.3882720470428467,
      "logits/rejected": -3.414973497390747,
      "logps/chosen": -0.06177481636404991,
      "logps/rejected": -119.70152282714844,
      "loss": 0.0869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.168581247329712,
      "rewards/margins": 2.168581247329712,
      "rewards/rejected": 0.0,
      "step": 2671
    },
    {
      "epoch": 14.927374301675977,
      "grad_norm": 0.09169689597571246,
      "learning_rate": 6.817100302550255e-08,
      "logits/chosen": -3.4632678031921387,
      "logits/rejected": -3.665860891342163,
      "logps/chosen": -0.09308983385562897,
      "logps/rejected": -134.44033813476562,
      "loss": 0.1073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.236053705215454,
      "rewards/margins": 2.236053705215454,
      "rewards/rejected": 0.0,
      "step": 2672
    },
    {
      "epoch": 14.932960893854748,
      "grad_norm": 0.06540266464898115,
      "learning_rate": 6.746594067188056e-08,
      "logits/chosen": -3.2963645458221436,
      "logits/rejected": -3.132025718688965,
      "logps/chosen": -2.651299476623535,
      "logps/rejected": -58.60000991821289,
      "loss": 0.1029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.020965099334717,
      "rewards/margins": 3.020965099334717,
      "rewards/rejected": 0.0,
      "step": 2673
    },
    {
      "epoch": 14.938547486033519,
      "grad_norm": 0.241120709981297,
      "learning_rate": 6.676449349708125e-08,
      "logits/chosen": -3.2946462631225586,
      "logits/rejected": -3.527331590652466,
      "logps/chosen": -21.93337059020996,
      "logps/rejected": -70.92479705810547,
      "loss": 0.0957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9592156410217285,
      "rewards/margins": 3.9592156410217285,
      "rewards/rejected": 0.0,
      "step": 2674
    },
    {
      "epoch": 14.94413407821229,
      "grad_norm": 0.07842534724392755,
      "learning_rate": 6.606666254358047e-08,
      "logits/chosen": -3.4131996631622314,
      "logits/rejected": -3.433109760284424,
      "logps/chosen": -0.17740947008132935,
      "logps/rejected": -115.56074523925781,
      "loss": 0.0857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.08317232131958,
      "rewards/margins": 2.08317232131958,
      "rewards/rejected": 0.0,
      "step": 2675
    },
    {
      "epoch": 14.949720670391061,
      "grad_norm": 0.09246361636941196,
      "learning_rate": 6.53724488484786e-08,
      "logits/chosen": -3.5814456939697266,
      "logits/rejected": -3.4657914638519287,
      "logps/chosen": -1.5611193180084229,
      "logps/rejected": -57.01099395751953,
      "loss": 0.108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.497338056564331,
      "rewards/margins": 3.497338056564331,
      "rewards/rejected": 0.0,
      "step": 2676
    },
    {
      "epoch": 14.955307262569832,
      "grad_norm": 0.06856733509301878,
      "learning_rate": 6.46818534435012e-08,
      "logits/chosen": -3.424607992172241,
      "logits/rejected": -3.328423023223877,
      "logps/chosen": -0.38313883543014526,
      "logps/rejected": -61.35697937011719,
      "loss": 0.0661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4650425910949707,
      "rewards/margins": 3.4650425910949707,
      "rewards/rejected": 0.0,
      "step": 2677
    },
    {
      "epoch": 14.960893854748603,
      "grad_norm": 0.08119345121844006,
      "learning_rate": 6.399487735499643e-08,
      "logits/chosen": -3.275883913040161,
      "logits/rejected": -3.1007730960845947,
      "logps/chosen": -0.22194182872772217,
      "logps/rejected": -69.80036926269531,
      "loss": 0.0855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4422695636749268,
      "rewards/margins": 2.4422695636749268,
      "rewards/rejected": 0.0,
      "step": 2678
    },
    {
      "epoch": 14.966480446927374,
      "grad_norm": 0.250641418464037,
      "learning_rate": 6.331152160393206e-08,
      "logits/chosen": -3.16677188873291,
      "logits/rejected": -3.2099499702453613,
      "logps/chosen": -0.3182055950164795,
      "logps/rejected": -64.22069549560547,
      "loss": 0.0971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4339427947998047,
      "rewards/margins": 2.4339427947998047,
      "rewards/rejected": 0.0,
      "step": 2679
    },
    {
      "epoch": 14.972067039106145,
      "grad_norm": 0.11173192919056635,
      "learning_rate": 6.263178720589763e-08,
      "logits/chosen": -3.4870736598968506,
      "logits/rejected": -3.5388107299804688,
      "logps/chosen": -1.0155028104782104,
      "logps/rejected": -66.85826110839844,
      "loss": 0.1108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1286306381225586,
      "rewards/margins": 3.1286306381225586,
      "rewards/rejected": 0.0,
      "step": 2680
    },
    {
      "epoch": 14.972067039106145,
      "eval_logits/chosen": -3.3475863933563232,
      "eval_logits/rejected": -3.471416473388672,
      "eval_logps/chosen": -47.374942779541016,
      "eval_logps/rejected": -75.1053695678711,
      "eval_loss": 2.486539602279663,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7304906845092773,
      "eval_rewards/margins": -1.7304906845092773,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.705,
      "eval_samples_per_second": 12.06,
      "eval_steps_per_second": 0.389,
      "step": 2680
    },
    {
      "epoch": 14.977653631284916,
      "grad_norm": 0.09291348209288353,
      "learning_rate": 6.195567517109841e-08,
      "logits/chosen": -3.3699326515197754,
      "logits/rejected": -3.558612585067749,
      "logps/chosen": -14.661766052246094,
      "logps/rejected": -98.6075439453125,
      "loss": 0.083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.255365371704102,
      "rewards/margins": 4.255365371704102,
      "rewards/rejected": 0.0,
      "step": 2681
    },
    {
      "epoch": 14.983240223463687,
      "grad_norm": 0.17312062735795453,
      "learning_rate": 6.12831865043581e-08,
      "logits/chosen": -3.413655996322632,
      "logits/rejected": -3.544975519180298,
      "logps/chosen": -0.0500541552901268,
      "logps/rejected": -106.30564880371094,
      "loss": 0.0848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7969796657562256,
      "rewards/margins": 1.7969796657562256,
      "rewards/rejected": 0.0,
      "step": 2682
    },
    {
      "epoch": 14.988826815642458,
      "grad_norm": 0.06894691878345026,
      "learning_rate": 6.061432220511449e-08,
      "logits/chosen": -3.265346050262451,
      "logits/rejected": -3.5909037590026855,
      "logps/chosen": -1.1652634143829346,
      "logps/rejected": -121.17091369628906,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7069835662841797,
      "rewards/margins": 2.7069835662841797,
      "rewards/rejected": 0.0,
      "step": 2683
    },
    {
      "epoch": 14.994413407821229,
      "grad_norm": 0.10945739047916746,
      "learning_rate": 5.994908326741877e-08,
      "logits/chosen": -3.592958688735962,
      "logits/rejected": -3.598598003387451,
      "logps/chosen": -0.5325120091438293,
      "logps/rejected": -88.70783996582031,
      "loss": 0.0643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3994975090026855,
      "rewards/margins": 3.3994975090026855,
      "rewards/rejected": 0.0,
      "step": 2684
    },
    {
      "epoch": 15.0,
      "grad_norm": 0.09216654281214443,
      "learning_rate": 5.928747067993429e-08,
      "logits/chosen": -3.4150915145874023,
      "logits/rejected": -3.4750094413757324,
      "logps/chosen": -0.11701145768165588,
      "logps/rejected": -52.98584747314453,
      "loss": 0.1087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5973095893859863,
      "rewards/margins": 2.5973095893859863,
      "rewards/rejected": 0.0,
      "step": 2685
    },
    {
      "epoch": 15.005586592178771,
      "grad_norm": 0.06087350082146417,
      "learning_rate": 5.862948542593616e-08,
      "logits/chosen": -3.583808660507202,
      "logits/rejected": -3.7303545475006104,
      "logps/chosen": -0.35043954849243164,
      "logps/rejected": -80.69517517089844,
      "loss": 0.0718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8679871559143066,
      "rewards/margins": 2.8679871559143066,
      "rewards/rejected": 0.0,
      "step": 2686
    },
    {
      "epoch": 15.011173184357542,
      "grad_norm": 0.08301206341390331,
      "learning_rate": 5.7975128483306884e-08,
      "logits/chosen": -3.566920280456543,
      "logits/rejected": -3.578704595565796,
      "logps/chosen": -2.0168848037719727,
      "logps/rejected": -86.14628601074219,
      "loss": 0.0973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4306559562683105,
      "rewards/margins": 3.4306559562683105,
      "rewards/rejected": 0.0,
      "step": 2687
    },
    {
      "epoch": 15.016759776536313,
      "grad_norm": 0.06758086910417789,
      "learning_rate": 5.732440082453688e-08,
      "logits/chosen": -3.3018648624420166,
      "logits/rejected": -3.598667860031128,
      "logps/chosen": -0.41568154096603394,
      "logps/rejected": -156.07833862304688,
      "loss": 0.0733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8274617195129395,
      "rewards/margins": 2.8274617195129395,
      "rewards/rejected": 0.0,
      "step": 2688
    },
    {
      "epoch": 15.022346368715084,
      "grad_norm": 0.07585054879195069,
      "learning_rate": 5.6677303416724784e-08,
      "logits/chosen": -3.2712154388427734,
      "logits/rejected": -3.3105082511901855,
      "logps/chosen": -1.1800408363342285,
      "logps/rejected": -83.71031188964844,
      "loss": 0.0848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.784479856491089,
      "rewards/margins": 3.784479856491089,
      "rewards/rejected": 0.0,
      "step": 2689
    },
    {
      "epoch": 15.027932960893855,
      "grad_norm": 0.13596395481336312,
      "learning_rate": 5.6033837221571016e-08,
      "logits/chosen": -3.262941360473633,
      "logits/rejected": -3.2470247745513916,
      "logps/chosen": -1.84309983253479,
      "logps/rejected": -75.82217407226562,
      "loss": 0.1133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.421543598175049,
      "rewards/margins": 2.421543598175049,
      "rewards/rejected": 0.0,
      "step": 2690
    },
    {
      "epoch": 15.033519553072626,
      "grad_norm": 0.0546493210844619,
      "learning_rate": 5.5394003195382295e-08,
      "logits/chosen": -3.5566465854644775,
      "logits/rejected": -3.600414752960205,
      "logps/chosen": -3.0827577114105225,
      "logps/rejected": -67.9485092163086,
      "loss": 0.0932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.339794397354126,
      "rewards/margins": 3.339794397354126,
      "rewards/rejected": 0.0,
      "step": 2691
    },
    {
      "epoch": 15.039106145251397,
      "grad_norm": 0.09412767966749257,
      "learning_rate": 5.475780228906519e-08,
      "logits/chosen": -3.456383466720581,
      "logits/rejected": -3.6109044551849365,
      "logps/chosen": -1.473923921585083,
      "logps/rejected": -79.66427612304688,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.686516761779785,
      "rewards/margins": 3.686516761779785,
      "rewards/rejected": 0.0,
      "step": 2692
    },
    {
      "epoch": 15.044692737430168,
      "grad_norm": 0.14283405662442247,
      "learning_rate": 5.4125235448127534e-08,
      "logits/chosen": -3.5348892211914062,
      "logits/rejected": -3.314619302749634,
      "logps/chosen": -0.09158764779567719,
      "logps/rejected": -46.61731719970703,
      "loss": 0.1178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.012991428375244,
      "rewards/margins": 2.012991428375244,
      "rewards/rejected": 0.0,
      "step": 2693
    },
    {
      "epoch": 15.050279329608939,
      "grad_norm": 0.05952288122344854,
      "learning_rate": 5.3496303612676214e-08,
      "logits/chosen": -3.5566611289978027,
      "logits/rejected": -3.2337896823883057,
      "logps/chosen": -8.666064262390137,
      "logps/rejected": -86.9752426147461,
      "loss": 0.13,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.156078815460205,
      "rewards/margins": 3.156078815460205,
      "rewards/rejected": 0.0,
      "step": 2694
    },
    {
      "epoch": 15.05586592178771,
      "grad_norm": 0.081284246538001,
      "learning_rate": 5.287100771741688e-08,
      "logits/chosen": -3.416889190673828,
      "logits/rejected": -3.4450767040252686,
      "logps/chosen": -1.879902720451355,
      "logps/rejected": -67.12012481689453,
      "loss": 0.0961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5146350860595703,
      "rewards/margins": 3.5146350860595703,
      "rewards/rejected": 0.0,
      "step": 2695
    },
    {
      "epoch": 15.061452513966481,
      "grad_norm": 0.06811505708106129,
      "learning_rate": 5.224934869164977e-08,
      "logits/chosen": -3.4806461334228516,
      "logits/rejected": -3.256423234939575,
      "logps/chosen": -12.347026824951172,
      "logps/rejected": -89.2240982055664,
      "loss": 0.0776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.484251022338867,
      "rewards/margins": 4.484251022338867,
      "rewards/rejected": 0.0,
      "step": 2696
    },
    {
      "epoch": 15.067039106145252,
      "grad_norm": 0.08877571521057044,
      "learning_rate": 5.163132745927058e-08,
      "logits/chosen": -3.5734453201293945,
      "logits/rejected": -3.622756242752075,
      "logps/chosen": -7.0094194412231445,
      "logps/rejected": -88.48245239257812,
      "loss": 0.078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.2164201736450195,
      "rewards/margins": 4.2164201736450195,
      "rewards/rejected": 0.0,
      "step": 2697
    },
    {
      "epoch": 15.072625698324023,
      "grad_norm": 0.06334463613564785,
      "learning_rate": 5.101694493876985e-08,
      "logits/chosen": -3.4060721397399902,
      "logits/rejected": -3.4259488582611084,
      "logps/chosen": -0.3380793333053589,
      "logps/rejected": -68.28422546386719,
      "loss": 0.0888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.737694263458252,
      "rewards/margins": 2.737694263458252,
      "rewards/rejected": 0.0,
      "step": 2698
    },
    {
      "epoch": 15.078212290502794,
      "grad_norm": 0.19474529171798852,
      "learning_rate": 5.0406202043228604e-08,
      "logits/chosen": -3.6640498638153076,
      "logits/rejected": -3.5242481231689453,
      "logps/chosen": -1.010623574256897,
      "logps/rejected": -81.3812484741211,
      "loss": 0.0736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5768966674804688,
      "rewards/margins": 3.5768966674804688,
      "rewards/rejected": 0.0,
      "step": 2699
    },
    {
      "epoch": 15.083798882681565,
      "grad_norm": 0.08741194973368072,
      "learning_rate": 4.9799099680320204e-08,
      "logits/chosen": -3.5028762817382812,
      "logits/rejected": -3.68888521194458,
      "logps/chosen": -0.08457759022712708,
      "logps/rejected": -100.74397277832031,
      "loss": 0.0931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.284221649169922,
      "rewards/margins": 2.284221649169922,
      "rewards/rejected": 0.0,
      "step": 2700
    },
    {
      "epoch": 15.083798882681565,
      "eval_logits/chosen": -3.3423047065734863,
      "eval_logits/rejected": -3.4681403636932373,
      "eval_logps/chosen": -47.4738883972168,
      "eval_logps/rejected": -75.25563049316406,
      "eval_loss": 2.484579086303711,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.740384817123413,
      "eval_rewards/margins": -1.740384817123413,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.556,
      "eval_samples_per_second": 12.13,
      "eval_steps_per_second": 0.391,
      "step": 2700
    },
    {
      "epoch": 15.089385474860336,
      "grad_norm": 0.09031880356856822,
      "learning_rate": 4.919563875230654e-08,
      "logits/chosen": -3.389329671859741,
      "logits/rejected": -3.411900281906128,
      "logps/chosen": -6.7606048583984375,
      "logps/rejected": -85.23677062988281,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.321913480758667,
      "rewards/margins": 3.321913480758667,
      "rewards/rejected": 0.0,
      "step": 2701
    },
    {
      "epoch": 15.094972067039107,
      "grad_norm": 0.18165478461516643,
      "learning_rate": 4.859582015603798e-08,
      "logits/chosen": -3.4334006309509277,
      "logits/rejected": -3.5534050464630127,
      "logps/chosen": -1.944549560546875,
      "logps/rejected": -112.60130310058594,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6350369453430176,
      "rewards/margins": 3.6350369453430176,
      "rewards/rejected": 0.0,
      "step": 2702
    },
    {
      "epoch": 15.100558659217878,
      "grad_norm": 0.06862972618114446,
      "learning_rate": 4.7999644782951714e-08,
      "logits/chosen": -3.5935728549957275,
      "logits/rejected": -3.5551836490631104,
      "logps/chosen": -0.18208704888820648,
      "logps/rejected": -68.35399627685547,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.311584949493408,
      "rewards/margins": 2.311584949493408,
      "rewards/rejected": 0.0,
      "step": 2703
    },
    {
      "epoch": 15.106145251396647,
      "grad_norm": 0.08350692622730652,
      "learning_rate": 4.740711351907096e-08,
      "logits/chosen": -3.2683041095733643,
      "logits/rejected": -3.0936245918273926,
      "logps/chosen": -0.060960300266742706,
      "logps/rejected": -136.77874755859375,
      "loss": 0.1169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2177979946136475,
      "rewards/margins": 2.2177979946136475,
      "rewards/rejected": 0.0,
      "step": 2704
    },
    {
      "epoch": 15.111731843575418,
      "grad_norm": 0.05927182967046026,
      "learning_rate": 4.681822724500212e-08,
      "logits/chosen": -3.2745108604431152,
      "logits/rejected": -3.4096338748931885,
      "logps/chosen": -0.4198507070541382,
      "logps/rejected": -79.2154541015625,
      "loss": 0.1,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5407586097717285,
      "rewards/margins": 3.5407586097717285,
      "rewards/rejected": 0.0,
      "step": 2705
    },
    {
      "epoch": 15.11731843575419,
      "grad_norm": 0.07760647300119268,
      "learning_rate": 4.623298683593569e-08,
      "logits/chosen": -3.6807544231414795,
      "logits/rejected": -3.5814812183380127,
      "logps/chosen": -0.09206473082304001,
      "logps/rejected": -65.00735473632812,
      "loss": 0.1028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4786179065704346,
      "rewards/margins": 2.4786179065704346,
      "rewards/rejected": 0.0,
      "step": 2706
    },
    {
      "epoch": 15.12290502793296,
      "grad_norm": 0.06246078433514727,
      "learning_rate": 4.5651393161642857e-08,
      "logits/chosen": -3.6627581119537354,
      "logits/rejected": -3.768402338027954,
      "logps/chosen": -0.16511279344558716,
      "logps/rejected": -127.64789581298828,
      "loss": 0.0957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7223448753356934,
      "rewards/margins": 2.7223448753356934,
      "rewards/rejected": 0.0,
      "step": 2707
    },
    {
      "epoch": 15.128491620111731,
      "grad_norm": 0.07237055064471519,
      "learning_rate": 4.5073447086474986e-08,
      "logits/chosen": -3.360015869140625,
      "logits/rejected": -3.4609780311584473,
      "logps/chosen": -1.6811405420303345,
      "logps/rejected": -47.24462127685547,
      "loss": 0.0606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4801242351531982,
      "rewards/margins": 3.4801242351531982,
      "rewards/rejected": 0.0,
      "step": 2708
    },
    {
      "epoch": 15.134078212290502,
      "grad_norm": 0.05635938606411965,
      "learning_rate": 4.4499149469363626e-08,
      "logits/chosen": -3.52665114402771,
      "logits/rejected": -3.6285622119903564,
      "logps/chosen": -0.045854613184928894,
      "logps/rejected": -87.06652069091797,
      "loss": 0.1042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8562848567962646,
      "rewards/margins": 1.8562848567962646,
      "rewards/rejected": 0.0,
      "step": 2709
    },
    {
      "epoch": 15.139664804469273,
      "grad_norm": 0.16150946991642842,
      "learning_rate": 4.392850116381658e-08,
      "logits/chosen": -3.554158926010132,
      "logits/rejected": -3.5773589611053467,
      "logps/chosen": -3.679598093032837,
      "logps/rejected": -68.16761779785156,
      "loss": 0.0834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2606945037841797,
      "rewards/margins": 3.2606945037841797,
      "rewards/rejected": 0.0,
      "step": 2710
    },
    {
      "epoch": 15.145251396648044,
      "grad_norm": 0.08133004746936791,
      "learning_rate": 4.3361503017919636e-08,
      "logits/chosen": -3.632436990737915,
      "logits/rejected": -3.6254689693450928,
      "logps/chosen": -0.04333137348294258,
      "logps/rejected": -46.70853805541992,
      "loss": 0.1072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7058504819869995,
      "rewards/margins": 1.7058504819869995,
      "rewards/rejected": 0.0,
      "step": 2711
    },
    {
      "epoch": 15.150837988826815,
      "grad_norm": 0.18918201085407266,
      "learning_rate": 4.279815587433234e-08,
      "logits/chosen": -3.3841419219970703,
      "logits/rejected": -3.602185010910034,
      "logps/chosen": -0.11444330960512161,
      "logps/rejected": -74.76443481445312,
      "loss": 0.1057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.339547634124756,
      "rewards/margins": 2.339547634124756,
      "rewards/rejected": 0.0,
      "step": 2712
    },
    {
      "epoch": 15.156424581005586,
      "grad_norm": 0.0703600303443826,
      "learning_rate": 4.223846057028913e-08,
      "logits/chosen": -3.4758052825927734,
      "logits/rejected": -3.617095470428467,
      "logps/chosen": -2.030487537384033,
      "logps/rejected": -84.22758483886719,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.991548538208008,
      "rewards/margins": 2.991548538208008,
      "rewards/rejected": 0.0,
      "step": 2713
    },
    {
      "epoch": 15.162011173184357,
      "grad_norm": 0.062126883486016846,
      "learning_rate": 4.1682417937596585e-08,
      "logits/chosen": -3.68786358833313,
      "logits/rejected": -3.459568500518799,
      "logps/chosen": -0.12375984340906143,
      "logps/rejected": -67.18240356445312,
      "loss": 0.0958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.70992374420166,
      "rewards/margins": 2.70992374420166,
      "rewards/rejected": 0.0,
      "step": 2714
    },
    {
      "epoch": 15.167597765363128,
      "grad_norm": 0.08288112856903099,
      "learning_rate": 4.113002880263395e-08,
      "logits/chosen": -3.409911870956421,
      "logits/rejected": -3.5906543731689453,
      "logps/chosen": -0.33500030636787415,
      "logps/rejected": -90.11952209472656,
      "loss": 0.0878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6977152824401855,
      "rewards/margins": 2.6977152824401855,
      "rewards/rejected": 0.0,
      "step": 2715
    },
    {
      "epoch": 15.1731843575419,
      "grad_norm": 0.10108338595517911,
      "learning_rate": 4.058129398634897e-08,
      "logits/chosen": -3.45843243598938,
      "logits/rejected": -3.464501142501831,
      "logps/chosen": -3.3450522422790527,
      "logps/rejected": -94.4803466796875,
      "loss": 0.1031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.909184455871582,
      "rewards/margins": 2.909184455871582,
      "rewards/rejected": 0.0,
      "step": 2716
    },
    {
      "epoch": 15.17877094972067,
      "grad_norm": 0.06581121285162035,
      "learning_rate": 4.003621430425986e-08,
      "logits/chosen": -3.3761396408081055,
      "logits/rejected": -3.4580764770507812,
      "logps/chosen": -0.1303006112575531,
      "logps/rejected": -87.07098388671875,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2808055877685547,
      "rewards/margins": 2.2808055877685547,
      "rewards/rejected": 0.0,
      "step": 2717
    },
    {
      "epoch": 15.184357541899441,
      "grad_norm": 0.08105624248318959,
      "learning_rate": 3.949479056645194e-08,
      "logits/chosen": -3.5160114765167236,
      "logits/rejected": -3.483417510986328,
      "logps/chosen": -0.2158122956752777,
      "logps/rejected": -75.03404235839844,
      "loss": 0.0899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6620888710021973,
      "rewards/margins": 2.6620888710021973,
      "rewards/rejected": 0.0,
      "step": 2718
    },
    {
      "epoch": 15.189944134078212,
      "grad_norm": 0.06989964934204956,
      "learning_rate": 3.895702357757686e-08,
      "logits/chosen": -3.2785120010375977,
      "logits/rejected": -2.989724636077881,
      "logps/chosen": -0.9528423547744751,
      "logps/rejected": -138.31771850585938,
      "loss": 0.0969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9725356101989746,
      "rewards/margins": 2.9725356101989746,
      "rewards/rejected": 0.0,
      "step": 2719
    },
    {
      "epoch": 15.195530726256983,
      "grad_norm": 0.1836530777575577,
      "learning_rate": 3.842291413685278e-08,
      "logits/chosen": -3.345991611480713,
      "logits/rejected": -3.5073277950286865,
      "logps/chosen": -2.3083364963531494,
      "logps/rejected": -66.37335968017578,
      "loss": 0.0796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3770580291748047,
      "rewards/margins": 3.3770580291748047,
      "rewards/rejected": 0.0,
      "step": 2720
    },
    {
      "epoch": 15.195530726256983,
      "eval_logits/chosen": -3.3545639514923096,
      "eval_logits/rejected": -3.4777474403381348,
      "eval_logps/chosen": -47.24371337890625,
      "eval_logps/rejected": -75.16525268554688,
      "eval_loss": 2.4848501682281494,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.71736741065979,
      "eval_rewards/margins": -1.71736741065979,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6086,
      "eval_samples_per_second": 12.105,
      "eval_steps_per_second": 0.39,
      "step": 2720
    },
    {
      "epoch": 15.201117318435754,
      "grad_norm": 0.06126580640264751,
      "learning_rate": 3.789246303806116e-08,
      "logits/chosen": -3.5398809909820557,
      "logits/rejected": -3.5752997398376465,
      "logps/chosen": -0.054307445883750916,
      "logps/rejected": -74.51615905761719,
      "loss": 0.0886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0310511589050293,
      "rewards/margins": 2.0310511589050293,
      "rewards/rejected": 0.0,
      "step": 2721
    },
    {
      "epoch": 15.206703910614525,
      "grad_norm": 0.10624224673415666,
      "learning_rate": 3.7365671069546636e-08,
      "logits/chosen": -3.3427658081054688,
      "logits/rejected": -3.2591917514801025,
      "logps/chosen": -7.422860622406006,
      "logps/rejected": -81.40379333496094,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4582910537719727,
      "rewards/margins": 3.4582910537719727,
      "rewards/rejected": 0.0,
      "step": 2722
    },
    {
      "epoch": 15.212290502793296,
      "grad_norm": 0.06855994422591813,
      "learning_rate": 3.684253901421603e-08,
      "logits/chosen": -3.02646803855896,
      "logits/rejected": -3.089282512664795,
      "logps/chosen": -5.582579135894775,
      "logps/rejected": -95.753662109375,
      "loss": 0.0875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1530637741088867,
      "rewards/margins": 3.1530637741088867,
      "rewards/rejected": 0.0,
      "step": 2723
    },
    {
      "epoch": 15.217877094972067,
      "grad_norm": 0.13541892652780776,
      "learning_rate": 3.6323067649537414e-08,
      "logits/chosen": -3.5867161750793457,
      "logits/rejected": -3.606198787689209,
      "logps/chosen": -3.5121426582336426,
      "logps/rejected": -57.06910705566406,
      "loss": 0.0739,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7086739540100098,
      "rewards/margins": 3.7086739540100098,
      "rewards/rejected": 0.0,
      "step": 2724
    },
    {
      "epoch": 15.223463687150838,
      "grad_norm": 0.09222469234876053,
      "learning_rate": 3.580725774753657e-08,
      "logits/chosen": -3.4243361949920654,
      "logits/rejected": -3.327516794204712,
      "logps/chosen": -0.20425766706466675,
      "logps/rejected": -111.10508728027344,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4035916328430176,
      "rewards/margins": 2.4035916328430176,
      "rewards/rejected": 0.0,
      "step": 2725
    },
    {
      "epoch": 15.22905027932961,
      "grad_norm": 0.101751705448414,
      "learning_rate": 3.529511007479946e-08,
      "logits/chosen": -3.401578426361084,
      "logits/rejected": -3.590961217880249,
      "logps/chosen": -1.8007445335388184,
      "logps/rejected": -75.44445037841797,
      "loss": 0.0681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.479214668273926,
      "rewards/margins": 4.479214668273926,
      "rewards/rejected": 0.0,
      "step": 2726
    },
    {
      "epoch": 15.23463687150838,
      "grad_norm": 0.057191340874770784,
      "learning_rate": 3.4786625392469184e-08,
      "logits/chosen": -3.591913938522339,
      "logits/rejected": -3.5862879753112793,
      "logps/chosen": -1.5128735303878784,
      "logps/rejected": -49.18549346923828,
      "loss": 0.0956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0055229663848877,
      "rewards/margins": 3.0055229663848877,
      "rewards/rejected": 0.0,
      "step": 2727
    },
    {
      "epoch": 15.240223463687151,
      "grad_norm": 0.05685414590238335,
      "learning_rate": 3.4281804456243464e-08,
      "logits/chosen": -3.3648815155029297,
      "logits/rejected": -3.536132335662842,
      "logps/chosen": -1.4901360273361206,
      "logps/rejected": -79.00518035888672,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.447396993637085,
      "rewards/margins": 3.447396993637085,
      "rewards/rejected": 0.0,
      "step": 2728
    },
    {
      "epoch": 15.245810055865922,
      "grad_norm": 0.06651136406440658,
      "learning_rate": 3.378064801637687e-08,
      "logits/chosen": -3.540707588195801,
      "logits/rejected": -3.7749557495117188,
      "logps/chosen": -1.375655174255371,
      "logps/rejected": -104.90602111816406,
      "loss": 0.0833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5636982917785645,
      "rewards/margins": 3.5636982917785645,
      "rewards/rejected": 0.0,
      "step": 2729
    },
    {
      "epoch": 15.251396648044693,
      "grad_norm": 0.07076828393457149,
      "learning_rate": 3.3283156817677245e-08,
      "logits/chosen": -3.4476537704467773,
      "logits/rejected": -3.69547700881958,
      "logps/chosen": -24.80058479309082,
      "logps/rejected": -83.67581939697266,
      "loss": 0.0686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.811678886413574,
      "rewards/margins": 3.811678886413574,
      "rewards/rejected": 0.0,
      "step": 2730
    },
    {
      "epoch": 15.256983240223464,
      "grad_norm": 0.08330272652047108,
      "learning_rate": 3.2789331599504535e-08,
      "logits/chosen": -3.595304250717163,
      "logits/rejected": -3.4066169261932373,
      "logps/chosen": -16.0871524810791,
      "logps/rejected": -98.7870101928711,
      "loss": 0.0967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.1950364112854,
      "rewards/margins": 4.1950364112854,
      "rewards/rejected": 0.0,
      "step": 2731
    },
    {
      "epoch": 15.262569832402235,
      "grad_norm": 0.07763059488966433,
      "learning_rate": 3.229917309577141e-08,
      "logits/chosen": -3.613851547241211,
      "logits/rejected": -3.4626972675323486,
      "logps/chosen": -2.7565975189208984,
      "logps/rejected": -99.91390991210938,
      "loss": 0.1081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3294169902801514,
      "rewards/margins": 3.3294169902801514,
      "rewards/rejected": 0.0,
      "step": 2732
    },
    {
      "epoch": 15.268156424581006,
      "grad_norm": 0.16590583824957325,
      "learning_rate": 3.1812682034940426e-08,
      "logits/chosen": -3.462803602218628,
      "logits/rejected": -3.5950090885162354,
      "logps/chosen": -1.60796058177948,
      "logps/rejected": -41.91818618774414,
      "loss": 0.072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.99067759513855,
      "rewards/margins": 3.99067759513855,
      "rewards/rejected": 0.0,
      "step": 2733
    },
    {
      "epoch": 15.273743016759777,
      "grad_norm": 0.10949747410067326,
      "learning_rate": 3.1329859140024063e-08,
      "logits/chosen": -3.205247640609741,
      "logits/rejected": -3.090736150741577,
      "logps/chosen": -16.828203201293945,
      "logps/rejected": -59.75727844238281,
      "loss": 0.077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.367287635803223,
      "rewards/margins": 4.367287635803223,
      "rewards/rejected": 0.0,
      "step": 2734
    },
    {
      "epoch": 15.279329608938548,
      "grad_norm": 0.12113779461998471,
      "learning_rate": 3.0850705128583345e-08,
      "logits/chosen": -3.5528180599212646,
      "logits/rejected": -3.424255609512329,
      "logps/chosen": -0.21694202721118927,
      "logps/rejected": -54.87000274658203,
      "loss": 0.0808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5473153591156006,
      "rewards/margins": 2.5473153591156006,
      "rewards/rejected": 0.0,
      "step": 2735
    },
    {
      "epoch": 15.28491620111732,
      "grad_norm": 0.07381994024890844,
      "learning_rate": 3.0375220712725864e-08,
      "logits/chosen": -3.380708694458008,
      "logits/rejected": -3.4931678771972656,
      "logps/chosen": -0.4469834566116333,
      "logps/rejected": -59.51420974731445,
      "loss": 0.0899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2473835945129395,
      "rewards/margins": 3.2473835945129395,
      "rewards/rejected": 0.0,
      "step": 2736
    },
    {
      "epoch": 15.29050279329609,
      "grad_norm": 0.2291337382434782,
      "learning_rate": 2.99034065991069e-08,
      "logits/chosen": -3.363121747970581,
      "logits/rejected": -3.347712516784668,
      "logps/chosen": -0.06317540258169174,
      "logps/rejected": -53.10482406616211,
      "loss": 0.1235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.05393123626709,
      "rewards/margins": 2.05393123626709,
      "rewards/rejected": 0.0,
      "step": 2737
    },
    {
      "epoch": 15.296089385474861,
      "grad_norm": 0.08157406806216373,
      "learning_rate": 2.9435263488925836e-08,
      "logits/chosen": -3.6473026275634766,
      "logits/rejected": -3.6163363456726074,
      "logps/chosen": -0.29996827244758606,
      "logps/rejected": -47.41431427001953,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.739543914794922,
      "rewards/margins": 2.739543914794922,
      "rewards/rejected": 0.0,
      "step": 2738
    },
    {
      "epoch": 15.30167597765363,
      "grad_norm": 0.05868257162957692,
      "learning_rate": 2.8970792077926414e-08,
      "logits/chosen": -3.3479833602905273,
      "logits/rejected": -3.419679880142212,
      "logps/chosen": -4.800485610961914,
      "logps/rejected": -90.10478210449219,
      "loss": 0.1208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.912367582321167,
      "rewards/margins": 3.912367582321167,
      "rewards/rejected": 0.0,
      "step": 2739
    },
    {
      "epoch": 15.307262569832401,
      "grad_norm": 0.06700226443746646,
      "learning_rate": 2.8509993056396456e-08,
      "logits/chosen": -3.344224452972412,
      "logits/rejected": -3.3780016899108887,
      "logps/chosen": -2.526299476623535,
      "logps/rejected": -69.8238754272461,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.343440055847168,
      "rewards/margins": 3.343440055847168,
      "rewards/rejected": 0.0,
      "step": 2740
    },
    {
      "epoch": 15.307262569832401,
      "eval_logits/chosen": -3.3435757160186768,
      "eval_logits/rejected": -3.469086170196533,
      "eval_logps/chosen": -47.509788513183594,
      "eval_logps/rejected": -75.1581039428711,
      "eval_loss": 2.493318557739258,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7439746856689453,
      "eval_rewards/margins": -1.7439746856689453,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6905,
      "eval_samples_per_second": 12.067,
      "eval_steps_per_second": 0.389,
      "step": 2740
    },
    {
      "epoch": 15.312849162011172,
      "grad_norm": 0.07389535308687781,
      "learning_rate": 2.805286710916483e-08,
      "logits/chosen": -3.374464750289917,
      "logits/rejected": -3.5629935264587402,
      "logps/chosen": -0.3510968089103699,
      "logps/rejected": -67.61271667480469,
      "loss": 0.0836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4273200035095215,
      "rewards/margins": 3.4273200035095215,
      "rewards/rejected": 0.0,
      "step": 2741
    },
    {
      "epoch": 15.318435754189943,
      "grad_norm": 0.08004366963191681,
      "learning_rate": 2.7599414915602806e-08,
      "logits/chosen": -3.4946272373199463,
      "logits/rejected": -3.2735142707824707,
      "logps/chosen": -0.11969605833292007,
      "logps/rejected": -92.78634643554688,
      "loss": 0.0885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8176844120025635,
      "rewards/margins": 2.8176844120025635,
      "rewards/rejected": 0.0,
      "step": 2742
    },
    {
      "epoch": 15.324022346368714,
      "grad_norm": 0.07630451201887872,
      "learning_rate": 2.7149637149620212e-08,
      "logits/chosen": -3.5323894023895264,
      "logits/rejected": -3.313035011291504,
      "logps/chosen": -2.589784860610962,
      "logps/rejected": -118.15804290771484,
      "loss": 0.1042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4905800819396973,
      "rewards/margins": 3.4905800819396973,
      "rewards/rejected": 0.0,
      "step": 2743
    },
    {
      "epoch": 15.329608938547485,
      "grad_norm": 0.1327379415635715,
      "learning_rate": 2.670353447966789e-08,
      "logits/chosen": -3.5632622241973877,
      "logits/rejected": -3.5468931198120117,
      "logps/chosen": -2.0292165279388428,
      "logps/rejected": -47.46084213256836,
      "loss": 0.1005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.984321117401123,
      "rewards/margins": 3.984321117401123,
      "rewards/rejected": 0.0,
      "step": 2744
    },
    {
      "epoch": 15.335195530726256,
      "grad_norm": 0.10445680652661272,
      "learning_rate": 2.6261107568732736e-08,
      "logits/chosen": -3.383681297302246,
      "logits/rejected": -3.3328068256378174,
      "logps/chosen": -0.09252248704433441,
      "logps/rejected": -64.81755065917969,
      "loss": 0.1115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7739174365997314,
      "rewards/margins": 1.7739174365997314,
      "rewards/rejected": 0.0,
      "step": 2745
    },
    {
      "epoch": 15.340782122905027,
      "grad_norm": 0.06589342440117354,
      "learning_rate": 2.5822357074340997e-08,
      "logits/chosen": -3.338146448135376,
      "logits/rejected": -3.282992124557495,
      "logps/chosen": -0.14448070526123047,
      "logps/rejected": -37.31623077392578,
      "loss": 0.0786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5821564197540283,
      "rewards/margins": 2.5821564197540283,
      "rewards/rejected": 0.0,
      "step": 2746
    },
    {
      "epoch": 15.346368715083798,
      "grad_norm": 0.06995687429479645,
      "learning_rate": 2.5387283648553307e-08,
      "logits/chosen": -3.5783727169036865,
      "logits/rejected": -3.45378041267395,
      "logps/chosen": -1.5255447626113892,
      "logps/rejected": -102.89134216308594,
      "loss": 0.0924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6155757904052734,
      "rewards/margins": 3.6155757904052734,
      "rewards/rejected": 0.0,
      "step": 2747
    },
    {
      "epoch": 15.35195530726257,
      "grad_norm": 0.06858677506746594,
      "learning_rate": 2.4955887937966605e-08,
      "logits/chosen": -3.3724491596221924,
      "logits/rejected": -3.4346840381622314,
      "logps/chosen": -0.17388297617435455,
      "logps/rejected": -77.27851867675781,
      "loss": 0.1083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2769391536712646,
      "rewards/margins": 2.2769391536712646,
      "rewards/rejected": 0.0,
      "step": 2748
    },
    {
      "epoch": 15.35754189944134,
      "grad_norm": 0.12859428267871326,
      "learning_rate": 2.452817058371193e-08,
      "logits/chosen": -3.5100245475769043,
      "logits/rejected": -3.4809508323669434,
      "logps/chosen": -0.2950904369354248,
      "logps/rejected": -82.99752807617188,
      "loss": 0.1121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.849607467651367,
      "rewards/margins": 2.849607467651367,
      "rewards/rejected": 0.0,
      "step": 2749
    },
    {
      "epoch": 15.363128491620111,
      "grad_norm": 0.1029563302835106,
      "learning_rate": 2.4104132221453026e-08,
      "logits/chosen": -3.3181262016296387,
      "logits/rejected": -3.4023361206054688,
      "logps/chosen": -1.9698055982589722,
      "logps/rejected": -73.15696716308594,
      "loss": 0.1056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.130910873413086,
      "rewards/margins": 4.130910873413086,
      "rewards/rejected": 0.0,
      "step": 2750
    },
    {
      "epoch": 15.368715083798882,
      "grad_norm": 0.11687629302131461,
      "learning_rate": 2.368377348138634e-08,
      "logits/chosen": -3.6389386653900146,
      "logits/rejected": -3.4950997829437256,
      "logps/chosen": -22.216842651367188,
      "logps/rejected": -56.7461051940918,
      "loss": 0.1141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.852404594421387,
      "rewards/margins": 5.852404594421387,
      "rewards/rejected": 0.0,
      "step": 2751
    },
    {
      "epoch": 15.374301675977653,
      "grad_norm": 0.06580704264218035,
      "learning_rate": 2.32670949882402e-08,
      "logits/chosen": -3.4066579341888428,
      "logits/rejected": -3.437929630279541,
      "logps/chosen": -0.08646802604198456,
      "logps/rejected": -68.40959167480469,
      "loss": 0.0947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.208587169647217,
      "rewards/margins": 2.208587169647217,
      "rewards/rejected": 0.0,
      "step": 2752
    },
    {
      "epoch": 15.379888268156424,
      "grad_norm": 0.05930564310285875,
      "learning_rate": 2.2854097361273143e-08,
      "logits/chosen": -3.5222647190093994,
      "logits/rejected": -3.404839038848877,
      "logps/chosen": -0.15493589639663696,
      "logps/rejected": -112.32102966308594,
      "loss": 0.1214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.292752265930176,
      "rewards/margins": 2.292752265930176,
      "rewards/rejected": 0.0,
      "step": 2753
    },
    {
      "epoch": 15.385474860335195,
      "grad_norm": 0.07794634496171665,
      "learning_rate": 2.2444781214272515e-08,
      "logits/chosen": -3.2579805850982666,
      "logits/rejected": -3.260159730911255,
      "logps/chosen": -0.3095542788505554,
      "logps/rejected": -89.8855209350586,
      "loss": 0.1033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.854715347290039,
      "rewards/margins": 2.854715347290039,
      "rewards/rejected": 0.0,
      "step": 2754
    },
    {
      "epoch": 15.391061452513966,
      "grad_norm": 0.06167847228311244,
      "learning_rate": 2.2039147155555884e-08,
      "logits/chosen": -3.615595817565918,
      "logits/rejected": -3.565816879272461,
      "logps/chosen": -0.20485813915729523,
      "logps/rejected": -207.8339080810547,
      "loss": 0.0748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5359389781951904,
      "rewards/margins": 2.5359389781951904,
      "rewards/rejected": 0.0,
      "step": 2755
    },
    {
      "epoch": 15.396648044692737,
      "grad_norm": 0.149833483015002,
      "learning_rate": 2.1637195787966858e-08,
      "logits/chosen": -3.4892988204956055,
      "logits/rejected": -3.4084830284118652,
      "logps/chosen": -0.1109364926815033,
      "logps/rejected": -57.66490936279297,
      "loss": 0.0764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5150861740112305,
      "rewards/margins": 2.5150861740112305,
      "rewards/rejected": 0.0,
      "step": 2756
    },
    {
      "epoch": 15.402234636871508,
      "grad_norm": 0.10936000445680952,
      "learning_rate": 2.1238927708877588e-08,
      "logits/chosen": -3.2009801864624023,
      "logits/rejected": -3.2409074306488037,
      "logps/chosen": -3.7064664363861084,
      "logps/rejected": -67.48663330078125,
      "loss": 0.1132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.4407243728637695,
      "rewards/margins": 4.4407243728637695,
      "rewards/rejected": 0.0,
      "step": 2757
    },
    {
      "epoch": 15.40782122905028,
      "grad_norm": 0.12165730381804428,
      "learning_rate": 2.0844343510184605e-08,
      "logits/chosen": -3.584834575653076,
      "logits/rejected": -3.705653667449951,
      "logps/chosen": -0.032418519258499146,
      "logps/rejected": -116.51278686523438,
      "loss": 0.0867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7090098857879639,
      "rewards/margins": 1.7090098857879639,
      "rewards/rejected": 0.0,
      "step": 2758
    },
    {
      "epoch": 15.41340782122905,
      "grad_norm": 0.07582916307638235,
      "learning_rate": 2.0453443778310766e-08,
      "logits/chosen": -3.3884212970733643,
      "logits/rejected": -3.1947591304779053,
      "logps/chosen": -0.13315390050411224,
      "logps/rejected": -107.76301574707031,
      "loss": 0.1295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.23248291015625,
      "rewards/margins": 2.23248291015625,
      "rewards/rejected": 0.0,
      "step": 2759
    },
    {
      "epoch": 15.418994413407821,
      "grad_norm": 0.06524394951676055,
      "learning_rate": 2.006622909420275e-08,
      "logits/chosen": -3.5978565216064453,
      "logits/rejected": -3.481435537338257,
      "logps/chosen": -4.604059219360352,
      "logps/rejected": -56.594539642333984,
      "loss": 0.0968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8490729331970215,
      "rewards/margins": 3.8490729331970215,
      "rewards/rejected": 0.0,
      "step": 2760
    },
    {
      "epoch": 15.418994413407821,
      "eval_logits/chosen": -3.3392422199249268,
      "eval_logits/rejected": -3.4659619331359863,
      "eval_logps/chosen": -47.50263595581055,
      "eval_logps/rejected": -75.40838623046875,
      "eval_loss": 2.4894070625305176,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7432596683502197,
      "eval_rewards/margins": -1.7432596683502197,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.5708,
      "eval_samples_per_second": 12.123,
      "eval_steps_per_second": 0.391,
      "step": 2760
    },
    {
      "epoch": 15.424581005586592,
      "grad_norm": 0.08304427697161672,
      "learning_rate": 1.9682700033329948e-08,
      "logits/chosen": -3.4937007427215576,
      "logits/rejected": -3.627150297164917,
      "logps/chosen": -0.9644902944564819,
      "logps/rejected": -69.6111831665039,
      "loss": 0.0942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7468364238739014,
      "rewards/margins": 3.7468364238739014,
      "rewards/rejected": 0.0,
      "step": 2761
    },
    {
      "epoch": 15.430167597765363,
      "grad_norm": 0.0765038315203457,
      "learning_rate": 1.9302857165685585e-08,
      "logits/chosen": -3.6887435913085938,
      "logits/rejected": -3.5693037509918213,
      "logps/chosen": -0.5923054218292236,
      "logps/rejected": -125.23566436767578,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3524510860443115,
      "rewards/margins": 3.3524510860443115,
      "rewards/rejected": 0.0,
      "step": 2762
    },
    {
      "epoch": 15.435754189944134,
      "grad_norm": 0.0703720718274148,
      "learning_rate": 1.892670105578309e-08,
      "logits/chosen": -3.3976500034332275,
      "logits/rejected": -3.2741739749908447,
      "logps/chosen": -7.296231269836426,
      "logps/rejected": -37.72069549560547,
      "loss": 0.0607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.1767730712890625,
      "rewards/margins": 4.1767730712890625,
      "rewards/rejected": 0.0,
      "step": 2763
    },
    {
      "epoch": 15.441340782122905,
      "grad_norm": 0.0879363257076842,
      "learning_rate": 1.8554232262658346e-08,
      "logits/chosen": -3.415048837661743,
      "logits/rejected": -3.287208318710327,
      "logps/chosen": -1.5808649063110352,
      "logps/rejected": -81.44414520263672,
      "loss": 0.0736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.252791881561279,
      "rewards/margins": 4.252791881561279,
      "rewards/rejected": 0.0,
      "step": 2764
    },
    {
      "epoch": 15.446927374301676,
      "grad_norm": 0.06887763478360065,
      "learning_rate": 1.8185451339865212e-08,
      "logits/chosen": -3.4631943702697754,
      "logits/rejected": -3.566100597381592,
      "logps/chosen": -9.574996948242188,
      "logps/rejected": -75.28001403808594,
      "loss": 0.0879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2247214317321777,
      "rewards/margins": 3.2247214317321777,
      "rewards/rejected": 0.0,
      "step": 2765
    },
    {
      "epoch": 15.452513966480447,
      "grad_norm": 0.06330668284616198,
      "learning_rate": 1.7820358835478612e-08,
      "logits/chosen": -3.4284586906433105,
      "logits/rejected": -3.366476058959961,
      "logps/chosen": -0.0781894251704216,
      "logps/rejected": -108.28744506835938,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.913622498512268,
      "rewards/margins": 1.913622498512268,
      "rewards/rejected": 0.0,
      "step": 2766
    },
    {
      "epoch": 15.458100558659218,
      "grad_norm": 0.08047665515453299,
      "learning_rate": 1.7458955292090908e-08,
      "logits/chosen": -3.481201171875,
      "logits/rejected": -3.5143234729766846,
      "logps/chosen": -0.09581702947616577,
      "logps/rejected": -63.904850006103516,
      "loss": 0.0965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5141196250915527,
      "rewards/margins": 2.5141196250915527,
      "rewards/rejected": 0.0,
      "step": 2767
    },
    {
      "epoch": 15.46368715083799,
      "grad_norm": 0.10811609665993614,
      "learning_rate": 1.7101241246811894e-08,
      "logits/chosen": -3.3514134883880615,
      "logits/rejected": -3.391160011291504,
      "logps/chosen": -0.1599455326795578,
      "logps/rejected": -78.9266128540039,
      "loss": 0.0893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7463021278381348,
      "rewards/margins": 2.7463021278381348,
      "rewards/rejected": 0.0,
      "step": 2768
    },
    {
      "epoch": 15.46927374301676,
      "grad_norm": 0.0682076133695015,
      "learning_rate": 1.6747217231268543e-08,
      "logits/chosen": -3.4743459224700928,
      "logits/rejected": -3.355234146118164,
      "logps/chosen": -1.4955674409866333,
      "logps/rejected": -73.40723419189453,
      "loss": 0.0848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.652853488922119,
      "rewards/margins": 3.652853488922119,
      "rewards/rejected": 0.0,
      "step": 2769
    },
    {
      "epoch": 15.474860335195531,
      "grad_norm": 0.06309760504676679,
      "learning_rate": 1.6396883771603588e-08,
      "logits/chosen": -3.2838194370269775,
      "logits/rejected": -3.263381004333496,
      "logps/chosen": -9.456883430480957,
      "logps/rejected": -69.0884780883789,
      "loss": 0.076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.127040863037109,
      "rewards/margins": 4.127040863037109,
      "rewards/rejected": 0.0,
      "step": 2770
    },
    {
      "epoch": 15.480446927374302,
      "grad_norm": 0.054826777585961134,
      "learning_rate": 1.605024138847472e-08,
      "logits/chosen": -3.283132314682007,
      "logits/rejected": -3.4017345905303955,
      "logps/chosen": -0.10158102214336395,
      "logps/rejected": -120.74005126953125,
      "loss": 0.0873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.150954008102417,
      "rewards/margins": 2.150954008102417,
      "rewards/rejected": 0.0,
      "step": 2771
    },
    {
      "epoch": 15.486033519553073,
      "grad_norm": 0.13479172395651814,
      "learning_rate": 1.570729059705428e-08,
      "logits/chosen": -3.4332175254821777,
      "logits/rejected": -3.39937686920166,
      "logps/chosen": -0.903067946434021,
      "logps/rejected": -51.955352783203125,
      "loss": 0.0916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8097057342529297,
      "rewards/margins": 2.8097057342529297,
      "rewards/rejected": 0.0,
      "step": 2772
    },
    {
      "epoch": 15.491620111731844,
      "grad_norm": 0.1012491950676247,
      "learning_rate": 1.5368031907028737e-08,
      "logits/chosen": -3.5245187282562256,
      "logits/rejected": -3.635437488555908,
      "logps/chosen": -0.6184836626052856,
      "logps/rejected": -65.00921630859375,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5047786235809326,
      "rewards/margins": 3.5047786235809326,
      "rewards/rejected": 0.0,
      "step": 2773
    },
    {
      "epoch": 15.497206703910614,
      "grad_norm": 0.06971825175548804,
      "learning_rate": 1.5032465822596154e-08,
      "logits/chosen": -3.6696712970733643,
      "logits/rejected": -3.6703004837036133,
      "logps/chosen": -0.6613969206809998,
      "logps/rejected": -87.042724609375,
      "loss": 0.1065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.924109935760498,
      "rewards/margins": 2.924109935760498,
      "rewards/rejected": 0.0,
      "step": 2774
    },
    {
      "epoch": 15.502793296089386,
      "grad_norm": 0.10893872908738948,
      "learning_rate": 1.4700592842467887e-08,
      "logits/chosen": -3.662618398666382,
      "logits/rejected": -3.581494092941284,
      "logps/chosen": -1.2745782136917114,
      "logps/rejected": -92.68180847167969,
      "loss": 0.1049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1417999267578125,
      "rewards/margins": 3.1417999267578125,
      "rewards/rejected": 0.0,
      "step": 2775
    },
    {
      "epoch": 15.508379888268156,
      "grad_norm": 0.08857123629459687,
      "learning_rate": 1.4372413459866618e-08,
      "logits/chosen": -3.377340078353882,
      "logits/rejected": -3.435826301574707,
      "logps/chosen": -0.5195276141166687,
      "logps/rejected": -90.61245727539062,
      "loss": 0.0949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.983135223388672,
      "rewards/margins": 2.983135223388672,
      "rewards/rejected": 0.0,
      "step": 2776
    },
    {
      "epoch": 15.513966480446927,
      "grad_norm": 0.0688387264591557,
      "learning_rate": 1.4047928162524705e-08,
      "logits/chosen": -3.251783609390259,
      "logits/rejected": -3.354464054107666,
      "logps/chosen": -0.029843183234333992,
      "logps/rejected": -122.26451873779297,
      "loss": 0.1104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4985909461975098,
      "rewards/margins": 1.4985909461975098,
      "rewards/rejected": 0.0,
      "step": 2777
    },
    {
      "epoch": 15.519553072625698,
      "grad_norm": 0.06413986472618324,
      "learning_rate": 1.3727137432685556e-08,
      "logits/chosen": -3.0542056560516357,
      "logits/rejected": -3.134302854537964,
      "logps/chosen": -0.44078221917152405,
      "logps/rejected": -73.28562927246094,
      "loss": 0.0929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.164726495742798,
      "rewards/margins": 3.164726495742798,
      "rewards/rejected": 0.0,
      "step": 2778
    },
    {
      "epoch": 15.525139664804469,
      "grad_norm": 0.06543441419332167,
      "learning_rate": 1.3410041747100866e-08,
      "logits/chosen": -3.5490193367004395,
      "logits/rejected": -3.6224846839904785,
      "logps/chosen": -0.1136002391576767,
      "logps/rejected": -107.57380676269531,
      "loss": 0.1103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4688148498535156,
      "rewards/margins": 2.4688148498535156,
      "rewards/rejected": 0.0,
      "step": 2779
    },
    {
      "epoch": 15.53072625698324,
      "grad_norm": 0.09817146211962548,
      "learning_rate": 1.3096641577031998e-08,
      "logits/chosen": -3.4680042266845703,
      "logits/rejected": -3.7258074283599854,
      "logps/chosen": -0.2619788646697998,
      "logps/rejected": -91.61324310302734,
      "loss": 0.1185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5260047912597656,
      "rewards/margins": 2.5260047912597656,
      "rewards/rejected": 0.0,
      "step": 2780
    },
    {
      "epoch": 15.53072625698324,
      "eval_logits/chosen": -3.3420352935791016,
      "eval_logits/rejected": -3.467559337615967,
      "eval_logps/chosen": -47.40123748779297,
      "eval_logps/rejected": -75.30181884765625,
      "eval_loss": 2.4996888637542725,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.733119249343872,
      "eval_rewards/margins": -1.733119249343872,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6005,
      "eval_samples_per_second": 12.109,
      "eval_steps_per_second": 0.391,
      "step": 2780
    },
    {
      "epoch": 15.53631284916201,
      "grad_norm": 0.0636826337496596,
      "learning_rate": 1.2786937388246656e-08,
      "logits/chosen": -3.3405275344848633,
      "logits/rejected": -3.541997194290161,
      "logps/chosen": -1.3829888105392456,
      "logps/rejected": -92.88412475585938,
      "loss": 0.0824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.338512897491455,
      "rewards/margins": 3.338512897491455,
      "rewards/rejected": 0.0,
      "step": 2781
    },
    {
      "epoch": 15.541899441340782,
      "grad_norm": 0.07672215254571678,
      "learning_rate": 1.2480929641020823e-08,
      "logits/chosen": -3.207190752029419,
      "logits/rejected": -3.3045361042022705,
      "logps/chosen": -0.3114842176437378,
      "logps/rejected": -78.98859405517578,
      "loss": 0.095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6992363929748535,
      "rewards/margins": 2.6992363929748535,
      "rewards/rejected": 0.0,
      "step": 2782
    },
    {
      "epoch": 15.547486033519553,
      "grad_norm": 0.06467994222286068,
      "learning_rate": 1.2178618790136543e-08,
      "logits/chosen": -3.5716335773468018,
      "logits/rejected": -3.4618449211120605,
      "logps/chosen": -0.21837478876113892,
      "logps/rejected": -88.88046264648438,
      "loss": 0.0692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.225508689880371,
      "rewards/margins": 3.225508689880371,
      "rewards/rejected": 0.0,
      "step": 2783
    },
    {
      "epoch": 15.553072625698324,
      "grad_norm": 0.0739176383698719,
      "learning_rate": 1.1880005284881368e-08,
      "logits/chosen": -3.3839354515075684,
      "logits/rejected": -3.38396954536438,
      "logps/chosen": -0.336498498916626,
      "logps/rejected": -91.82330322265625,
      "loss": 0.0844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2020554542541504,
      "rewards/margins": 3.2020554542541504,
      "rewards/rejected": 0.0,
      "step": 2784
    },
    {
      "epoch": 15.558659217877095,
      "grad_norm": 0.10991158708524605,
      "learning_rate": 1.1585089569048635e-08,
      "logits/chosen": -3.306359052658081,
      "logits/rejected": -3.3765058517456055,
      "logps/chosen": -0.20256057381629944,
      "logps/rejected": -88.52799224853516,
      "loss": 0.1066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6608099937438965,
      "rewards/margins": 2.6608099937438965,
      "rewards/rejected": 0.0,
      "step": 2785
    },
    {
      "epoch": 15.564245810055866,
      "grad_norm": 0.07138452896848355,
      "learning_rate": 1.1293872080934965e-08,
      "logits/chosen": -3.331258773803711,
      "logits/rejected": -3.31390643119812,
      "logps/chosen": -0.654541552066803,
      "logps/rejected": -90.73831176757812,
      "loss": 0.0921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.140639066696167,
      "rewards/margins": 3.140639066696167,
      "rewards/rejected": 0.0,
      "step": 2786
    },
    {
      "epoch": 15.569832402234637,
      "grad_norm": 0.20615513302508595,
      "learning_rate": 1.1006353253341651e-08,
      "logits/chosen": -3.684760332107544,
      "logits/rejected": -3.5166618824005127,
      "logps/chosen": -0.7343639731407166,
      "logps/rejected": -59.59184265136719,
      "loss": 0.1095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.925171136856079,
      "rewards/margins": 2.925171136856079,
      "rewards/rejected": 0.0,
      "step": 2787
    },
    {
      "epoch": 15.575418994413408,
      "grad_norm": 0.06891527228473046,
      "learning_rate": 1.0722533513572996e-08,
      "logits/chosen": -3.5007989406585693,
      "logits/rejected": -3.5372331142425537,
      "logps/chosen": -3.565725326538086,
      "logps/rejected": -47.86988067626953,
      "loss": 0.0907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.474992275238037,
      "rewards/margins": 4.474992275238037,
      "rewards/rejected": 0.0,
      "step": 2788
    },
    {
      "epoch": 15.581005586592179,
      "grad_norm": 0.07525180018089062,
      "learning_rate": 1.0442413283435759e-08,
      "logits/chosen": -2.758923053741455,
      "logits/rejected": -2.81191086769104,
      "logps/chosen": -12.105801582336426,
      "logps/rejected": -68.31151580810547,
      "loss": 0.0936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.91522216796875,
      "rewards/margins": 4.91522216796875,
      "rewards/rejected": 0.0,
      "step": 2789
    },
    {
      "epoch": 15.58659217877095,
      "grad_norm": 0.09000236076207628,
      "learning_rate": 1.0165992979238037e-08,
      "logits/chosen": -3.396803855895996,
      "logits/rejected": -3.338219165802002,
      "logps/chosen": -3.4699511528015137,
      "logps/rejected": -77.7493896484375,
      "loss": 0.1193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8900277614593506,
      "rewards/margins": 3.8900277614593506,
      "rewards/rejected": 0.0,
      "step": 2790
    },
    {
      "epoch": 15.59217877094972,
      "grad_norm": 0.18350557163173342,
      "learning_rate": 9.893273011790105e-09,
      "logits/chosen": -3.4088072776794434,
      "logits/rejected": -3.585818290710449,
      "logps/chosen": -0.18339017033576965,
      "logps/rejected": -97.97665405273438,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.917022705078125,
      "rewards/margins": 2.917022705078125,
      "rewards/rejected": 0.0,
      "step": 2791
    },
    {
      "epoch": 15.597765363128492,
      "grad_norm": 0.08104013050648716,
      "learning_rate": 9.624253786402193e-09,
      "logits/chosen": -3.3339433670043945,
      "logits/rejected": -3.28316330909729,
      "logps/chosen": -0.1418805569410324,
      "logps/rejected": -62.73424530029297,
      "loss": 0.0729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9360082149505615,
      "rewards/margins": 2.9360082149505615,
      "rewards/rejected": 0.0,
      "step": 2792
    },
    {
      "epoch": 15.603351955307263,
      "grad_norm": 0.10867705584094947,
      "learning_rate": 9.358935702884763e-09,
      "logits/chosen": -3.4274606704711914,
      "logits/rejected": -3.45723295211792,
      "logps/chosen": -1.611250400543213,
      "logps/rejected": -91.81990814208984,
      "loss": 0.0938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5292749404907227,
      "rewards/margins": 3.5292749404907227,
      "rewards/rejected": 0.0,
      "step": 2793
    },
    {
      "epoch": 15.608938547486034,
      "grad_norm": 0.13613889365730977,
      "learning_rate": 9.097319155547957e-09,
      "logits/chosen": -3.409119129180908,
      "logits/rejected": -3.3991682529449463,
      "logps/chosen": -0.23953001201152802,
      "logps/rejected": -70.9903793334961,
      "loss": 0.1144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2229185104370117,
      "rewards/margins": 2.2229185104370117,
      "rewards/rejected": 0.0,
      "step": 2794
    },
    {
      "epoch": 15.614525139664805,
      "grad_norm": 0.21183831940644393,
      "learning_rate": 8.839404533200201e-09,
      "logits/chosen": -3.4511196613311768,
      "logits/rejected": -3.3919196128845215,
      "logps/chosen": -0.09194304049015045,
      "logps/rejected": -80.68867492675781,
      "loss": 0.1005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4174046516418457,
      "rewards/margins": 2.4174046516418457,
      "rewards/rejected": 0.0,
      "step": 2795
    },
    {
      "epoch": 15.620111731843576,
      "grad_norm": 0.07870643501111128,
      "learning_rate": 8.585192219148775e-09,
      "logits/chosen": -3.5079879760742188,
      "logits/rejected": -3.4023990631103516,
      "logps/chosen": -0.055559802800416946,
      "logps/rejected": -95.88575744628906,
      "loss": 0.1146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1010630130767822,
      "rewards/margins": 2.1010630130767822,
      "rewards/rejected": 0.0,
      "step": 2796
    },
    {
      "epoch": 15.625698324022347,
      "grad_norm": 0.06848750162414868,
      "learning_rate": 8.334682591198406e-09,
      "logits/chosen": -3.680471420288086,
      "logits/rejected": -3.81967830657959,
      "logps/chosen": -0.05435600131750107,
      "logps/rejected": -61.04711151123047,
      "loss": 0.1121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9788801670074463,
      "rewards/margins": 1.9788801670074463,
      "rewards/rejected": 0.0,
      "step": 2797
    },
    {
      "epoch": 15.631284916201118,
      "grad_norm": 0.11033279851747729,
      "learning_rate": 8.087876021651287e-09,
      "logits/chosen": -3.2740135192871094,
      "logits/rejected": -3.360989809036255,
      "logps/chosen": -0.05257813259959221,
      "logps/rejected": -68.323974609375,
      "loss": 0.0904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0660409927368164,
      "rewards/margins": 2.0660409927368164,
      "rewards/rejected": 0.0,
      "step": 2798
    },
    {
      "epoch": 15.636871508379889,
      "grad_norm": 0.06883518378720632,
      "learning_rate": 7.844772877305395e-09,
      "logits/chosen": -3.448732852935791,
      "logits/rejected": -3.3789007663726807,
      "logps/chosen": -1.2360402345657349,
      "logps/rejected": -52.799503326416016,
      "loss": 0.0801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0636680126190186,
      "rewards/margins": 3.0636680126190186,
      "rewards/rejected": 0.0,
      "step": 2799
    },
    {
      "epoch": 15.64245810055866,
      "grad_norm": 0.07347724072547576,
      "learning_rate": 7.605373519455616e-09,
      "logits/chosen": -3.414083480834961,
      "logits/rejected": -3.396712064743042,
      "logps/chosen": -1.4669040441513062,
      "logps/rejected": -64.02394104003906,
      "loss": 0.1196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0104446411132812,
      "rewards/margins": 3.0104446411132812,
      "rewards/rejected": 0.0,
      "step": 2800
    },
    {
      "epoch": 15.64245810055866,
      "eval_logits/chosen": -3.3413100242614746,
      "eval_logits/rejected": -3.4676036834716797,
      "eval_logps/chosen": -47.37445831298828,
      "eval_logps/rejected": -75.23528289794922,
      "eval_loss": 2.492432117462158,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7304418087005615,
      "eval_rewards/margins": -1.7304418087005615,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 25.6236,
      "eval_samples_per_second": 12.098,
      "eval_steps_per_second": 0.39,
      "step": 2800
    },
    {
      "epoch": 15.64804469273743,
      "grad_norm": 0.06605473958524372,
      "learning_rate": 7.369678303892069e-09,
      "logits/chosen": -3.167696952819824,
      "logits/rejected": -3.1558542251586914,
      "logps/chosen": -1.4753198623657227,
      "logps/rejected": -184.23814392089844,
      "loss": 0.1004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.507171630859375,
      "rewards/margins": 3.507171630859375,
      "rewards/rejected": 0.0,
      "step": 2801
    },
    {
      "epoch": 15.653631284916202,
      "grad_norm": 0.07707161961020655,
      "learning_rate": 7.137687580899555e-09,
      "logits/chosen": -3.7952585220336914,
      "logits/rejected": -3.4413225650787354,
      "logps/chosen": -0.3136916160583496,
      "logps/rejected": -83.6711654663086,
      "loss": 0.088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3850274085998535,
      "rewards/margins": 2.3850274085998535,
      "rewards/rejected": 0.0,
      "step": 2802
    },
    {
      "epoch": 15.659217877094973,
      "grad_norm": 0.0929502451394159,
      "learning_rate": 6.909401695257834e-09,
      "logits/chosen": -3.4854655265808105,
      "logits/rejected": -3.5775043964385986,
      "logps/chosen": -0.07359762489795685,
      "logps/rejected": -70.69830322265625,
      "loss": 0.0916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.413515567779541,
      "rewards/margins": 2.413515567779541,
      "rewards/rejected": 0.0,
      "step": 2803
    },
    {
      "epoch": 15.664804469273744,
      "grad_norm": 0.06619135540239196,
      "learning_rate": 6.684820986240515e-09,
      "logits/chosen": -3.4305644035339355,
      "logits/rejected": -3.6688780784606934,
      "logps/chosen": -2.010016918182373,
      "logps/rejected": -99.08497619628906,
      "loss": 0.0629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.82488751411438,
      "rewards/margins": 3.82488751411438,
      "rewards/rejected": 0.0,
      "step": 2804
    },
    {
      "epoch": 15.670391061452515,
      "grad_norm": 0.05859261055967558,
      "learning_rate": 6.463945787614223e-09,
      "logits/chosen": -3.304037094116211,
      "logits/rejected": -3.3344905376434326,
      "logps/chosen": -0.1681138575077057,
      "logps/rejected": -60.66477966308594,
      "loss": 0.0917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.681652545928955,
      "rewards/margins": 2.681652545928955,
      "rewards/rejected": 0.0,
      "step": 2805
    },
    {
      "epoch": 15.675977653631286,
      "grad_norm": 0.23231635279263277,
      "learning_rate": 6.246776427639156e-09,
      "logits/chosen": -3.6290698051452637,
      "logits/rejected": -3.638810157775879,
      "logps/chosen": -0.6224958896636963,
      "logps/rejected": -74.41159057617188,
      "loss": 0.1027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.260887384414673,
      "rewards/margins": 3.260887384414673,
      "rewards/rejected": 0.0,
      "step": 2806
    },
    {
      "epoch": 15.681564245810057,
      "grad_norm": 0.10459496539914614,
      "learning_rate": 6.033313229067139e-09,
      "logits/chosen": -3.3099114894866943,
      "logits/rejected": -3.2708089351654053,
      "logps/chosen": -0.896016001701355,
      "logps/rejected": -84.5750503540039,
      "loss": 0.1185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.80500864982605,
      "rewards/margins": 3.80500864982605,
      "rewards/rejected": 0.0,
      "step": 2807
    },
    {
      "epoch": 15.687150837988828,
      "grad_norm": 0.06712543399590276,
      "learning_rate": 5.823556509143013e-09,
      "logits/chosen": -3.497455596923828,
      "logits/rejected": -3.418752431869507,
      "logps/chosen": -0.07571820169687271,
      "logps/rejected": -123.8826904296875,
      "loss": 0.0884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2072691917419434,
      "rewards/margins": 2.2072691917419434,
      "rewards/rejected": 0.0,
      "step": 2808
    },
    {
      "epoch": 15.692737430167599,
      "grad_norm": 0.09381758064883929,
      "learning_rate": 5.617506579602694e-09,
      "logits/chosen": -3.1149485111236572,
      "logits/rejected": -3.2121708393096924,
      "logps/chosen": -1.1464471817016602,
      "logps/rejected": -65.04843139648438,
      "loss": 0.0826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.659022331237793,
      "rewards/margins": 3.659022331237793,
      "rewards/rejected": 0.0,
      "step": 2809
    },
    {
      "epoch": 15.69832402234637,
      "grad_norm": 0.08157931300750791,
      "learning_rate": 5.415163746672336e-09,
      "logits/chosen": -3.27819561958313,
      "logits/rejected": -3.260833263397217,
      "logps/chosen": -0.31416991353034973,
      "logps/rejected": -65.88528442382812,
      "loss": 0.1275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.55147647857666,
      "rewards/margins": 2.55147647857666,
      "rewards/rejected": 0.0,
      "step": 2810
    },
    {
      "epoch": 15.703910614525139,
      "grad_norm": 0.11199004334898396,
      "learning_rate": 5.216528311070279e-09,
      "logits/chosen": -3.2737841606140137,
      "logits/rejected": -3.4769816398620605,
      "logps/chosen": -0.06868614256381989,
      "logps/rejected": -72.49754333496094,
      "loss": 0.1084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.068758010864258,
      "rewards/margins": 2.068758010864258,
      "rewards/rejected": 0.0,
      "step": 2811
    },
    {
      "epoch": 15.70949720670391,
      "grad_norm": 0.06574788385104496,
      "learning_rate": 5.0216005680034374e-09,
      "logits/chosen": -3.630953073501587,
      "logits/rejected": -3.4982993602752686,
      "logps/chosen": -0.41153740882873535,
      "logps/rejected": -64.26884460449219,
      "loss": 0.1033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3772568702697754,
      "rewards/margins": 3.3772568702697754,
      "rewards/rejected": 0.0,
      "step": 2812
    },
    {
      "epoch": 15.71508379888268,
      "grad_norm": 0.07769784031295746,
      "learning_rate": 4.830380807169799e-09,
      "logits/chosen": -3.5292623043060303,
      "logits/rejected": -3.4918453693389893,
      "logps/chosen": -1.0959370136260986,
      "logps/rejected": -109.1531753540039,
      "loss": 0.0728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.429105043411255,
      "rewards/margins": 3.429105043411255,
      "rewards/rejected": 0.0,
      "step": 2813
    },
    {
      "epoch": 15.720670391061452,
      "grad_norm": 0.07337043490726505,
      "learning_rate": 4.642869312755649e-09,
      "logits/chosen": -3.4188151359558105,
      "logits/rejected": -3.6130895614624023,
      "logps/chosen": -0.04764718562364578,
      "logps/rejected": -67.7523193359375,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6957037448883057,
      "rewards/margins": 1.6957037448883057,
      "rewards/rejected": 0.0,
      "step": 2814
    },
    {
      "epoch": 15.726256983240223,
      "grad_norm": 0.06405154619776614,
      "learning_rate": 4.459066363436404e-09,
      "logits/chosen": -3.502201795578003,
      "logits/rejected": -3.4674811363220215,
      "logps/chosen": -11.656913757324219,
      "logps/rejected": -107.47445678710938,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.477845668792725,
      "rewards/margins": 4.477845668792725,
      "rewards/rejected": 0.0,
      "step": 2815
    },
    {
      "epoch": 15.731843575418994,
      "grad_norm": 0.05659276447414945,
      "learning_rate": 4.278972232376055e-09,
      "logits/chosen": -3.3710808753967285,
      "logits/rejected": -3.3932273387908936,
      "logps/chosen": -0.1366906613111496,
      "logps/rejected": -50.82596969604492,
      "loss": 0.1307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1214940547943115,
      "rewards/margins": 2.1214940547943115,
      "rewards/rejected": 0.0,
      "step": 2816
    },
    {
      "epoch": 15.737430167597765,
      "grad_norm": 0.057258417677475845,
      "learning_rate": 4.1025871872266145e-09,
      "logits/chosen": -3.6876072883605957,
      "logits/rejected": -3.711169481277466,
      "logps/chosen": -0.13256098330020905,
      "logps/rejected": -80.47181701660156,
      "loss": 0.0984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.216733455657959,
      "rewards/margins": 2.216733455657959,
      "rewards/rejected": 0.0,
      "step": 2817
    },
    {
      "epoch": 15.743016759776536,
      "grad_norm": 0.059026905921715334,
      "learning_rate": 3.929911490127558e-09,
      "logits/chosen": -3.800696611404419,
      "logits/rejected": -3.6680636405944824,
      "logps/chosen": -0.6521201133728027,
      "logps/rejected": -63.48240661621094,
      "loss": 0.1007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1984426975250244,
      "rewards/margins": 3.1984426975250244,
      "rewards/rejected": 0.0,
      "step": 2818
    },
    {
      "epoch": 15.748603351955307,
      "grad_norm": 0.06441718385991946,
      "learning_rate": 3.760945397705828e-09,
      "logits/chosen": -3.0818707942962646,
      "logits/rejected": -3.0276100635528564,
      "logps/chosen": -6.450782775878906,
      "logps/rejected": -54.81761932373047,
      "loss": 0.0811,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.797600030899048,
      "rewards/margins": 3.797600030899048,
      "rewards/rejected": 0.0,
      "step": 2819
    },
    {
      "epoch": 15.754189944134078,
      "grad_norm": 0.0654099672178021,
      "learning_rate": 3.5956891610752777e-09,
      "logits/chosen": -3.49802303314209,
      "logits/rejected": -3.627603769302368,
      "logps/chosen": -0.2851520776748657,
      "logps/rejected": -64.04364013671875,
      "loss": 0.1048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.957000255584717,
      "rewards/margins": 2.957000255584717,
      "rewards/rejected": 0.0,
      "step": 2820
    },
    {
      "epoch": 15.754189944134078,
      "eval_logits/chosen": -3.3453285694122314,
      "eval_logits/rejected": -3.470130205154419,
      "eval_logps/chosen": -47.282676696777344,
      "eval_logps/rejected": -75.15850830078125,
      "eval_loss": 2.49228835105896,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.721264123916626,
      "eval_rewards/margins": -1.721264123916626,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 36.4244,
      "eval_samples_per_second": 8.511,
      "eval_steps_per_second": 0.275,
      "step": 2820
    },
    {
      "epoch": 15.759776536312849,
      "grad_norm": 0.06027012321963651,
      "learning_rate": 3.434143025835834e-09,
      "logits/chosen": -3.4904024600982666,
      "logits/rejected": -3.3491783142089844,
      "logps/chosen": -0.17782026529312134,
      "logps/rejected": -122.28035736083984,
      "loss": 0.0701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.451840400695801,
      "rewards/margins": 2.451840400695801,
      "rewards/rejected": 0.0,
      "step": 2821
    },
    {
      "epoch": 15.76536312849162,
      "grad_norm": 0.08220870713369734,
      "learning_rate": 3.2763072320737833e-09,
      "logits/chosen": -3.491356134414673,
      "logits/rejected": -3.706437587738037,
      "logps/chosen": -23.931806564331055,
      "logps/rejected": -55.03112030029297,
      "loss": 0.1292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.1812744140625,
      "rewards/margins": 4.1812744140625,
      "rewards/rejected": 0.0,
      "step": 2822
    },
    {
      "epoch": 15.77094972067039,
      "grad_norm": 0.24158642402104927,
      "learning_rate": 3.1221820143614857e-09,
      "logits/chosen": -3.3773059844970703,
      "logits/rejected": -3.4749419689178467,
      "logps/chosen": -5.872084140777588,
      "logps/rejected": -59.177833557128906,
      "loss": 0.1232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.323142051696777,
      "rewards/margins": 4.323142051696777,
      "rewards/rejected": 0.0,
      "step": 2823
    },
    {
      "epoch": 15.776536312849162,
      "grad_norm": 0.08141725807547504,
      "learning_rate": 2.9717676017562703e-09,
      "logits/chosen": -3.4394278526306152,
      "logits/rejected": -3.3938910961151123,
      "logps/chosen": -2.221015214920044,
      "logps/rejected": -77.76676940917969,
      "loss": 0.115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.610126495361328,
      "rewards/margins": 3.610126495361328,
      "rewards/rejected": 0.0,
      "step": 2824
    },
    {
      "epoch": 15.782122905027933,
      "grad_norm": 0.10226776299196899,
      "learning_rate": 2.8250642178007103e-09,
      "logits/chosen": -3.219860792160034,
      "logits/rejected": -3.335422992706299,
      "logps/chosen": -0.2947622537612915,
      "logps/rejected": -98.76092529296875,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5582022666931152,
      "rewards/margins": 2.5582022666931152,
      "rewards/rejected": 0.0,
      "step": 2825
    },
    {
      "epoch": 15.787709497206704,
      "grad_norm": 0.07587724545679266,
      "learning_rate": 2.6820720805223465e-09,
      "logits/chosen": -3.440401792526245,
      "logits/rejected": -3.2990269660949707,
      "logps/chosen": -12.039679527282715,
      "logps/rejected": -72.43476867675781,
      "loss": 0.0747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.0225653648376465,
      "rewards/margins": 4.0225653648376465,
      "rewards/rejected": 0.0,
      "step": 2826
    },
    {
      "epoch": 15.793296089385475,
      "grad_norm": 0.06909941433158935,
      "learning_rate": 2.5427914024328537e-09,
      "logits/chosen": -3.6128461360931396,
      "logits/rejected": -3.5705482959747314,
      "logps/chosen": -2.212287425994873,
      "logps/rejected": -80.34994506835938,
      "loss": 0.071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.425081253051758,
      "rewards/margins": 4.425081253051758,
      "rewards/rejected": 0.0,
      "step": 2827
    },
    {
      "epoch": 15.798882681564246,
      "grad_norm": 0.128538326774322,
      "learning_rate": 2.407222390528041e-09,
      "logits/chosen": -3.5346062183380127,
      "logits/rejected": -3.662837028503418,
      "logps/chosen": -0.6916574239730835,
      "logps/rejected": -108.77653503417969,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.402315616607666,
      "rewards/margins": 3.402315616607666,
      "rewards/rejected": 0.0,
      "step": 2828
    },
    {
      "epoch": 15.804469273743017,
      "grad_norm": 0.17632242739002668,
      "learning_rate": 2.2753652462878528e-09,
      "logits/chosen": -3.074406147003174,
      "logits/rejected": -3.057480573654175,
      "logps/chosen": -6.808490753173828,
      "logps/rejected": -59.173702239990234,
      "loss": 0.102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.54844331741333,
      "rewards/margins": 3.54844331741333,
      "rewards/rejected": 0.0,
      "step": 2829
    },
    {
      "epoch": 15.810055865921788,
      "grad_norm": 0.07338791651107965,
      "learning_rate": 2.1472201656752567e-09,
      "logits/chosen": -3.415936231613159,
      "logits/rejected": -3.388049364089966,
      "logps/chosen": -1.3033119440078735,
      "logps/rejected": -53.284812927246094,
      "loss": 0.0894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.292821407318115,
      "rewards/margins": 4.292821407318115,
      "rewards/rejected": 0.0,
      "step": 2830
    },
    {
      "epoch": 15.815642458100559,
      "grad_norm": 0.22818702230870594,
      "learning_rate": 2.0227873391368e-09,
      "logits/chosen": -3.545337438583374,
      "logits/rejected": -3.4466400146484375,
      "logps/chosen": -0.04984913021326065,
      "logps/rejected": -146.43247985839844,
      "loss": 0.1447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6555644273757935,
      "rewards/margins": 1.6555644273757935,
      "rewards/rejected": 0.0,
      "step": 2831
    },
    {
      "epoch": 15.82122905027933,
      "grad_norm": 0.27224431845210206,
      "learning_rate": 1.9020669516015e-09,
      "logits/chosen": -3.5220894813537598,
      "logits/rejected": -3.378898859024048,
      "logps/chosen": -43.75907897949219,
      "logps/rejected": -82.16192626953125,
      "loss": 0.0943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.143968105316162,
      "rewards/margins": 4.143968105316162,
      "rewards/rejected": 0.0,
      "step": 2832
    },
    {
      "epoch": 15.8268156424581,
      "grad_norm": 0.06252052177367119,
      "learning_rate": 1.7850591824819518e-09,
      "logits/chosen": -3.4360110759735107,
      "logits/rejected": -3.3904173374176025,
      "logps/chosen": -0.2722325921058655,
      "logps/rejected": -90.50654602050781,
      "loss": 0.1005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.086320400238037,
      "rewards/margins": 3.086320400238037,
      "rewards/rejected": 0.0,
      "step": 2833
    },
    {
      "epoch": 15.832402234636872,
      "grad_norm": 0.07582915447593554,
      "learning_rate": 1.6717642056721106e-09,
      "logits/chosen": -3.3195483684539795,
      "logits/rejected": -3.483537435531616,
      "logps/chosen": -0.32701942324638367,
      "logps/rejected": -119.918701171875,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.014735698699951,
      "rewards/margins": 3.014735698699951,
      "rewards/rejected": 0.0,
      "step": 2834
    },
    {
      "epoch": 15.837988826815643,
      "grad_norm": 0.05894056733578767,
      "learning_rate": 1.5621821895484002e-09,
      "logits/chosen": -3.339810609817505,
      "logits/rejected": -3.22098970413208,
      "logps/chosen": -0.16541801393032074,
      "logps/rejected": -131.99359130859375,
      "loss": 0.0782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5087485313415527,
      "rewards/margins": 2.5087485313415527,
      "rewards/rejected": 0.0,
      "step": 2835
    },
    {
      "epoch": 15.843575418994414,
      "grad_norm": 0.05792275025803481,
      "learning_rate": 1.456313296969436e-09,
      "logits/chosen": -3.3697242736816406,
      "logits/rejected": -3.2525293827056885,
      "logps/chosen": -1.1539714336395264,
      "logps/rejected": -97.24539184570312,
      "loss": 0.1166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.718949794769287,
      "rewards/margins": 2.718949794769287,
      "rewards/rejected": 0.0,
      "step": 2836
    },
    {
      "epoch": 15.849162011173185,
      "grad_norm": 0.06169083290782013,
      "learning_rate": 1.35415768527547e-09,
      "logits/chosen": -3.3522002696990967,
      "logits/rejected": -3.469846248626709,
      "logps/chosen": -0.12439636886119843,
      "logps/rejected": -86.8050537109375,
      "loss": 0.1244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3501925468444824,
      "rewards/margins": 2.3501925468444824,
      "rewards/rejected": 0.0,
      "step": 2837
    },
    {
      "epoch": 15.854748603351956,
      "grad_norm": 0.15076821130289314,
      "learning_rate": 1.2557155062875581e-09,
      "logits/chosen": -3.3721554279327393,
      "logits/rejected": -3.474015474319458,
      "logps/chosen": -0.15480196475982666,
      "logps/rejected": -66.68931579589844,
      "loss": 0.1186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.340919017791748,
      "rewards/margins": 2.340919017791748,
      "rewards/rejected": 0.0,
      "step": 2838
    },
    {
      "epoch": 15.860335195530727,
      "grad_norm": 0.05469865384815534,
      "learning_rate": 1.1609869063081147e-09,
      "logits/chosen": -3.537051200866699,
      "logits/rejected": -3.7184009552001953,
      "logps/chosen": -0.12188741564750671,
      "logps/rejected": -58.157615661621094,
      "loss": 0.1461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.422609329223633,
      "rewards/margins": 2.422609329223633,
      "rewards/rejected": 0.0,
      "step": 2839
    },
    {
      "epoch": 15.865921787709498,
      "grad_norm": 0.08475694437440483,
      "learning_rate": 1.0699720261214685e-09,
      "logits/chosen": -3.403956174850464,
      "logits/rejected": -3.5428097248077393,
      "logps/chosen": -0.28007930517196655,
      "logps/rejected": -67.26319885253906,
      "loss": 0.0864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.009378433227539,
      "rewards/margins": 3.009378433227539,
      "rewards/rejected": 0.0,
      "step": 2840
    },
    {
      "epoch": 15.865921787709498,
      "eval_logits/chosen": -3.3372886180877686,
      "eval_logits/rejected": -3.464087724685669,
      "eval_logps/chosen": -47.31254959106445,
      "eval_logps/rejected": -75.32481384277344,
      "eval_loss": 2.490699052810669,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7242504358291626,
      "eval_rewards/margins": -1.7242504358291626,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 34.2706,
      "eval_samples_per_second": 9.046,
      "eval_steps_per_second": 0.292,
      "step": 2840
    },
    {
      "epoch": 15.871508379888269,
      "grad_norm": 0.07014872250919763,
      "learning_rate": 9.826710009910866e-10,
      "logits/chosen": -3.66642689704895,
      "logits/rejected": -3.7725396156311035,
      "logps/chosen": -0.2694419026374817,
      "logps/rejected": -53.383628845214844,
      "loss": 0.0965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.286777973175049,
      "rewards/margins": 2.286777973175049,
      "rewards/rejected": 0.0,
      "step": 2841
    },
    {
      "epoch": 15.87709497206704,
      "grad_norm": 0.11400650236094123,
      "learning_rate": 8.990839606623502e-10,
      "logits/chosen": -3.351201295852661,
      "logits/rejected": -3.077425479888916,
      "logps/chosen": -0.0689784586429596,
      "logps/rejected": -161.99246215820312,
      "loss": 0.0892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8020633459091187,
      "rewards/margins": 1.8020633459091187,
      "rewards/rejected": 0.0,
      "step": 2842
    },
    {
      "epoch": 15.88268156424581,
      "grad_norm": 0.11143603570226611,
      "learning_rate": 8.192110293606115e-10,
      "logits/chosen": -3.339954376220703,
      "logits/rejected": -3.242433786392212,
      "logps/chosen": -0.22348976135253906,
      "logps/rejected": -39.238990783691406,
      "loss": 0.1124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.599144458770752,
      "rewards/margins": 2.599144458770752,
      "rewards/rejected": 0.0,
      "step": 2843
    },
    {
      "epoch": 15.888268156424582,
      "grad_norm": 0.08716775915165767,
      "learning_rate": 7.430523257909161e-10,
      "logits/chosen": -3.364763021469116,
      "logits/rejected": -3.4702746868133545,
      "logps/chosen": -8.824444770812988,
      "logps/rejected": -75.41344451904297,
      "loss": 0.0634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.999072551727295,
      "rewards/margins": 3.999072551727295,
      "rewards/rejected": 0.0,
      "step": 2844
    },
    {
      "epoch": 15.893854748603353,
      "grad_norm": 0.0862922606579798,
      "learning_rate": 6.706079631385587e-10,
      "logits/chosen": -3.5571086406707764,
      "logits/rejected": -3.65500545501709,
      "logps/chosen": -0.5697550773620605,
      "logps/rejected": -58.712013244628906,
      "loss": 0.085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.732456922531128,
      "rewards/margins": 2.732456922531128,
      "rewards/rejected": 0.0,
      "step": 2845
    },
    {
      "epoch": 15.899441340782122,
      "grad_norm": 0.07098422570954431,
      "learning_rate": 6.018780490690823e-10,
      "logits/chosen": -3.646716594696045,
      "logits/rejected": -3.6624855995178223,
      "logps/chosen": -0.8158391714096069,
      "logps/rejected": -90.78672790527344,
      "loss": 0.1109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8477234840393066,
      "rewards/margins": 2.8477234840393066,
      "rewards/rejected": 0.0,
      "step": 2846
    },
    {
      "epoch": 15.905027932960895,
      "grad_norm": 0.0658744376599127,
      "learning_rate": 5.368626857274461e-10,
      "logits/chosen": -3.289581298828125,
      "logits/rejected": -3.3102762699127197,
      "logps/chosen": -0.6068215370178223,
      "logps/rejected": -98.23741149902344,
      "loss": 0.0933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.485121011734009,
      "rewards/margins": 3.485121011734009,
      "rewards/rejected": 0.0,
      "step": 2847
    },
    {
      "epoch": 15.910614525139664,
      "grad_norm": 0.08174994607918143,
      "learning_rate": 4.755619697377478e-10,
      "logits/chosen": -3.3266208171844482,
      "logits/rejected": -3.416579246520996,
      "logps/chosen": -8.912286758422852,
      "logps/rejected": -84.00599670410156,
      "loss": 0.0906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.620667457580566,
      "rewards/margins": 4.620667457580566,
      "rewards/rejected": 0.0,
      "step": 2848
    },
    {
      "epoch": 15.916201117318435,
      "grad_norm": 0.06801986976669967,
      "learning_rate": 4.1797599220405605e-10,
      "logits/chosen": -3.312364101409912,
      "logits/rejected": -3.2477965354919434,
      "logps/chosen": -0.15498216450214386,
      "logps/rejected": -91.32539367675781,
      "loss": 0.1134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8150181770324707,
      "rewards/margins": 2.8150181770324707,
      "rewards/rejected": 0.0,
      "step": 2849
    },
    {
      "epoch": 15.921787709497206,
      "grad_norm": 0.08930325859729565,
      "learning_rate": 3.641048387090229e-10,
      "logits/chosen": -3.436702013015747,
      "logits/rejected": -3.3798465728759766,
      "logps/chosen": -1.3915913105010986,
      "logps/rejected": -64.94581604003906,
      "loss": 0.0955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.318284034729004,
      "rewards/margins": 3.318284034729004,
      "rewards/rejected": 0.0,
      "step": 2850
    },
    {
      "epoch": 15.927374301675977,
      "grad_norm": 0.05643388239622793,
      "learning_rate": 3.1394858931527164e-10,
      "logits/chosen": -3.434492826461792,
      "logits/rejected": -3.5812230110168457,
      "logps/chosen": -2.4595086574554443,
      "logps/rejected": -71.75892639160156,
      "loss": 0.1114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.448441982269287,
      "rewards/margins": 3.448441982269287,
      "rewards/rejected": 0.0,
      "step": 2851
    },
    {
      "epoch": 15.932960893854748,
      "grad_norm": 0.07249292834110609,
      "learning_rate": 2.6750731856345357e-10,
      "logits/chosen": -3.372734546661377,
      "logits/rejected": -3.533433437347412,
      "logps/chosen": -4.610701560974121,
      "logps/rejected": -51.24278259277344,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.490117073059082,
      "rewards/margins": 4.490117073059082,
      "rewards/rejected": 0.0,
      "step": 2852
    },
    {
      "epoch": 15.938547486033519,
      "grad_norm": 0.10991222857396198,
      "learning_rate": 2.2478109547391381e-10,
      "logits/chosen": -3.3430559635162354,
      "logits/rejected": -3.4304921627044678,
      "logps/chosen": -3.4207632541656494,
      "logps/rejected": -68.03253173828125,
      "loss": 0.1148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.980964183807373,
      "rewards/margins": 3.980964183807373,
      "rewards/rejected": 0.0,
      "step": 2853
    },
    {
      "epoch": 15.94413407821229,
      "grad_norm": 0.07105383557729578,
      "learning_rate": 1.8576998354502552e-10,
      "logits/chosen": -3.6207902431488037,
      "logits/rejected": -3.281219720840454,
      "logps/chosen": -0.1572730839252472,
      "logps/rejected": -58.96430969238281,
      "loss": 0.0885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.828357458114624,
      "rewards/margins": 2.828357458114624,
      "rewards/rejected": 0.0,
      "step": 2854
    },
    {
      "epoch": 15.949720670391061,
      "grad_norm": 0.06790516435770795,
      "learning_rate": 1.5047404075457794e-10,
      "logits/chosen": -3.264843702316284,
      "logits/rejected": -3.2750017642974854,
      "logps/chosen": -0.11574513465166092,
      "logps/rejected": -70.9504165649414,
      "loss": 0.078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.658302068710327,
      "rewards/margins": 2.658302068710327,
      "rewards/rejected": 0.0,
      "step": 2855
    },
    {
      "epoch": 15.955307262569832,
      "grad_norm": 0.08399368570063082,
      "learning_rate": 1.1889331955866613e-10,
      "logits/chosen": -3.4427995681762695,
      "logits/rejected": -3.408165693283081,
      "logps/chosen": -0.11211597174406052,
      "logps/rejected": -73.08062744140625,
      "loss": 0.1099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.213984727859497,
      "rewards/margins": 2.213984727859497,
      "rewards/rejected": 0.0,
      "step": 2856
    },
    {
      "epoch": 15.960893854748603,
      "grad_norm": 0.06710378743008386,
      "learning_rate": 9.102786689169086e-11,
      "logits/chosen": -3.4562289714813232,
      "logits/rejected": -3.4486474990844727,
      "logps/chosen": -0.17738944292068481,
      "logps/rejected": -57.805213928222656,
      "loss": 0.1138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.885660171508789,
      "rewards/margins": 2.885660171508789,
      "rewards/rejected": 0.0,
      "step": 2857
    },
    {
      "epoch": 15.966480446927374,
      "grad_norm": 0.1915329993128672,
      "learning_rate": 6.687772416691385e-11,
      "logits/chosen": -3.530916452407837,
      "logits/rejected": -3.372704267501831,
      "logps/chosen": -0.6016235947608948,
      "logps/rejected": -54.4246940612793,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.197883129119873,
      "rewards/margins": 3.197883129119873,
      "rewards/rejected": 0.0,
      "step": 2858
    },
    {
      "epoch": 15.972067039106145,
      "grad_norm": 0.12654279067889576,
      "learning_rate": 4.6442927275347494e-11,
      "logits/chosen": -3.517580270767212,
      "logits/rejected": -3.308044195175171,
      "logps/chosen": -0.22740349173545837,
      "logps/rejected": -41.24491500854492,
      "loss": 0.106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.685245990753174,
      "rewards/margins": 2.685245990753174,
      "rewards/rejected": 0.0,
      "step": 2859
    },
    {
      "epoch": 15.977653631284916,
      "grad_norm": 0.08062274870430663,
      "learning_rate": 2.9723506587142625e-11,
      "logits/chosen": -3.2542426586151123,
      "logits/rejected": -3.305727481842041,
      "logps/chosen": -0.5297117829322815,
      "logps/rejected": -74.84120178222656,
      "loss": 0.1051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2012722492218018,
      "rewards/margins": 2.2012722492218018,
      "rewards/rejected": 0.0,
      "step": 2860
    },
    {
      "epoch": 15.977653631284916,
      "eval_logits/chosen": -3.3399853706359863,
      "eval_logits/rejected": -3.4662749767303467,
      "eval_logps/chosen": -47.42780685424805,
      "eval_logps/rejected": -75.15325927734375,
      "eval_loss": 2.4961345195770264,
      "eval_rewards/accuracies": 0.125,
      "eval_rewards/chosen": -1.7357765436172485,
      "eval_rewards/margins": -1.7357765436172485,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 33.525,
      "eval_samples_per_second": 9.247,
      "eval_steps_per_second": 0.298,
      "step": 2860
    },
    {
      "epoch": 15.983240223463687,
      "grad_norm": 0.19641153986274212,
      "learning_rate": 1.6719486949923202e-11,
      "logits/chosen": -3.237496852874756,
      "logits/rejected": -3.5832409858703613,
      "logps/chosen": -2.699066162109375,
      "logps/rejected": -114.77009582519531,
      "loss": 0.0974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.334101676940918,
      "rewards/margins": 3.334101676940918,
      "rewards/rejected": 0.0,
      "step": 2861
    },
    {
      "epoch": 15.988826815642458,
      "grad_norm": 0.06813943490802679,
      "learning_rate": 7.430887690451638e-12,
      "logits/chosen": -3.32602596282959,
      "logits/rejected": -3.3812060356140137,
      "logps/chosen": -8.990257263183594,
      "logps/rejected": -72.118896484375,
      "loss": 0.0857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3817543983459473,
      "rewards/margins": 3.3817543983459473,
      "rewards/rejected": 0.0,
      "step": 2862
    },
    {
      "epoch": 15.994413407821229,
      "grad_norm": 0.13729832957934315,
      "learning_rate": 1.8577226126859082e-12,
      "logits/chosen": -3.598761796951294,
      "logits/rejected": -3.7483444213867188,
      "logps/chosen": -27.1510009765625,
      "logps/rejected": -54.622013092041016,
      "loss": 0.1033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.549412727355957,
      "rewards/margins": 4.549412727355957,
      "rewards/rejected": 0.0,
      "step": 2863
    },
    {
      "epoch": 16.0,
      "grad_norm": 0.06583257495503447,
      "learning_rate": 0.0,
      "logits/chosen": -3.789564609527588,
      "logits/rejected": -3.6094675064086914,
      "logps/chosen": -1.2147160768508911,
      "logps/rejected": -70.31568908691406,
      "loss": 0.0704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7421398162841797,
      "rewards/margins": 3.7421398162841797,
      "rewards/rejected": 0.0,
      "step": 2864
    },
    {
      "epoch": 16.0,
      "step": 2864,
      "total_flos": 0.0,
      "train_loss": 0.0022047922514670387,
      "train_runtime": 763.2324,
      "train_samples_per_second": 119.492,
      "train_steps_per_second": 3.752
    }
  ],
  "logging_steps": 1,
  "max_steps": 2864,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 16,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}