{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 267,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_losses": 0.6931471824645996,
      "epoch": 0.011235955056179775,
      "grad_norm": 71.71370697021484,
      "learning_rate": 0.0,
      "logits/chosen": -0.3973936140537262,
      "logits/rejected": -0.19569522142410278,
      "logps/chosen": -212.93252563476562,
      "logps/rejected": -266.0229187011719,
      "loss": 1.0274,
      "mlc_losses": 0.33424538373947144,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "dpo_losses": 0.6931471824645996,
      "epoch": 0.02247191011235955,
      "grad_norm": 67.82925415039062,
      "learning_rate": 2.222222222222222e-08,
      "logits/chosen": -0.45047780871391296,
      "logits/rejected": -0.3416946530342102,
      "logps/chosen": -222.4302978515625,
      "logps/rejected": -274.4136047363281,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "dpo_losses": 0.692604660987854,
      "epoch": 0.033707865168539325,
      "grad_norm": 80.24111938476562,
      "learning_rate": 4.444444444444444e-08,
      "logits/chosen": -0.5316526889801025,
      "logits/rejected": -0.1835283488035202,
      "logps/chosen": -226.9559326171875,
      "logps/rejected": -262.3705139160156,
      "loss": 1.0053,
      "mlc_losses": 0.31267061829566956,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.018749892711639404,
      "rewards/margins": 0.0020933635532855988,
      "rewards/rejected": -0.020843256264925003,
      "step": 3
    },
    {
      "dpo_losses": 0.675106406211853,
      "epoch": 0.0449438202247191,
      "grad_norm": 81.84447479248047,
      "learning_rate": 6.666666666666665e-08,
      "logits/chosen": -0.512816309928894,
      "logits/rejected": -0.2699843943119049,
      "logps/chosen": -207.0283966064453,
      "logps/rejected": -293.243408203125,
      "loss": 0.6751,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.020512914285063744,
      "rewards/margins": 0.03780413046479225,
      "rewards/rejected": -0.058317042887210846,
      "step": 4
    },
    {
      "dpo_losses": 0.6823002099990845,
      "epoch": 0.056179775280898875,
      "grad_norm": 64.6352310180664,
      "learning_rate": 8.888888888888888e-08,
      "logits/chosen": -0.390508770942688,
      "logits/rejected": -0.19000676274299622,
      "logps/chosen": -213.3816375732422,
      "logps/rejected": -241.50535583496094,
      "loss": 1.0233,
      "mlc_losses": 0.3409666419029236,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.008901000022888184,
      "rewards/margins": 0.023310281336307526,
      "rewards/rejected": -0.014409280382096767,
      "step": 5
    },
    {
      "dpo_losses": 0.692997395992279,
      "epoch": 0.06741573033707865,
      "grad_norm": 72.02581024169922,
      "learning_rate": 1.111111111111111e-07,
      "logits/chosen": -0.5404237508773804,
      "logits/rejected": -0.3554574251174927,
      "logps/chosen": -249.73684692382812,
      "logps/rejected": -313.73779296875,
      "loss": 0.693,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02942866086959839,
      "rewards/margins": 0.002566373674198985,
      "rewards/rejected": -0.03199503570795059,
      "step": 6
    },
    {
      "dpo_losses": 0.6970054507255554,
      "epoch": 0.07865168539325842,
      "grad_norm": 74.11085510253906,
      "learning_rate": 1.333333333333333e-07,
      "logits/chosen": -0.5076188445091248,
      "logits/rejected": -0.26660415530204773,
      "logps/chosen": -249.79197692871094,
      "logps/rejected": -259.6405029296875,
      "loss": 1.0279,
      "mlc_losses": 0.3309324383735657,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01827683486044407,
      "rewards/margins": -0.006754613481462002,
      "rewards/rejected": -0.011522223241627216,
      "step": 7
    },
    {
      "dpo_losses": 0.6961629390716553,
      "epoch": 0.0898876404494382,
      "grad_norm": 62.69937515258789,
      "learning_rate": 1.5555555555555554e-07,
      "logits/chosen": -0.4704933166503906,
      "logits/rejected": -0.24644428491592407,
      "logps/chosen": -276.5002746582031,
      "logps/rejected": -264.92156982421875,
      "loss": 0.6962,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.032434940338134766,
      "rewards/margins": -0.0037503940984606743,
      "rewards/rejected": -0.028684545308351517,
      "step": 8
    },
    {
      "dpo_losses": 0.6946243047714233,
      "epoch": 0.10112359550561797,
      "grad_norm": 74.58344268798828,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -0.5600687861442566,
      "logits/rejected": -0.27216824889183044,
      "logps/chosen": -206.4434356689453,
      "logps/rejected": -227.16131591796875,
      "loss": 1.0276,
      "mlc_losses": 0.33292850852012634,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03547070175409317,
      "rewards/margins": 0.0002887244336307049,
      "rewards/rejected": -0.03575942665338516,
      "step": 9
    },
    {
      "dpo_losses": 0.6827702522277832,
      "epoch": 0.11235955056179775,
      "grad_norm": 86.76605987548828,
      "learning_rate": 2e-07,
      "logits/chosen": -0.4563573896884918,
      "logits/rejected": -0.30447372794151306,
      "logps/chosen": -237.57620239257812,
      "logps/rejected": -296.6573181152344,
      "loss": 0.6828,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0016886240337044,
      "rewards/margins": 0.02298278734087944,
      "rewards/rejected": -0.021294167265295982,
      "step": 10
    },
    {
      "dpo_losses": 0.6844837069511414,
      "epoch": 0.12359550561797752,
      "grad_norm": 77.4293212890625,
      "learning_rate": 2.222222222222222e-07,
      "logits/chosen": -0.5707773566246033,
      "logits/rejected": -0.32518136501312256,
      "logps/chosen": -207.65211486816406,
      "logps/rejected": -288.21466064453125,
      "loss": 1.006,
      "mlc_losses": 0.3215576112270355,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.02576892264187336,
      "rewards/margins": 0.02003498189151287,
      "rewards/rejected": -0.04580390453338623,
      "step": 11
    },
    {
      "dpo_losses": 0.6744129657745361,
      "epoch": 0.1348314606741573,
      "grad_norm": 76.6510009765625,
      "learning_rate": 2.4444444444444445e-07,
      "logits/chosen": -0.4092194139957428,
      "logits/rejected": -0.2574714422225952,
      "logps/chosen": -232.82110595703125,
      "logps/rejected": -322.1761474609375,
      "loss": 0.6744,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03844718635082245,
      "rewards/margins": 0.04014239460229874,
      "rewards/rejected": -0.07858958840370178,
      "step": 12
    },
    {
      "dpo_losses": 0.6757420301437378,
      "epoch": 0.14606741573033707,
      "grad_norm": 78.10440063476562,
      "learning_rate": 2.666666666666666e-07,
      "logits/chosen": -0.40626153349876404,
      "logits/rejected": -0.047795142978429794,
      "logps/chosen": -197.4783935546875,
      "logps/rejected": -292.2260437011719,
      "loss": 1.0026,
      "mlc_losses": 0.326830118894577,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.05778537318110466,
      "rewards/margins": 0.038571882992982864,
      "rewards/rejected": -0.09635725617408752,
      "step": 13
    },
    {
      "dpo_losses": 0.6582584381103516,
      "epoch": 0.15730337078651685,
      "grad_norm": 66.40240478515625,
      "learning_rate": 2.8888888888888885e-07,
      "logits/chosen": -0.374487042427063,
      "logits/rejected": -0.26223358511924744,
      "logps/chosen": -202.6383056640625,
      "logps/rejected": -252.57720947265625,
      "loss": 0.6583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0766158178448677,
      "rewards/margins": 0.0763230100274086,
      "rewards/rejected": -0.1529388278722763,
      "step": 14
    },
    {
      "dpo_losses": 0.6589199304580688,
      "epoch": 0.16853932584269662,
      "grad_norm": 60.7664909362793,
      "learning_rate": 3.1111111111111107e-07,
      "logits/chosen": -0.3879810869693756,
      "logits/rejected": -0.3472948968410492,
      "logps/chosen": -224.2183837890625,
      "logps/rejected": -238.93350219726562,
      "loss": 0.9936,
      "mlc_losses": 0.3346329927444458,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08444397896528244,
      "rewards/margins": 0.07498157024383545,
      "rewards/rejected": -0.1594255417585373,
      "step": 15
    },
    {
      "dpo_losses": 0.6448157429695129,
      "epoch": 0.1797752808988764,
      "grad_norm": 75.97528076171875,
      "learning_rate": 3.3333333333333335e-07,
      "logits/chosen": -0.6155813932418823,
      "logits/rejected": -0.3031560778617859,
      "logps/chosen": -209.913330078125,
      "logps/rejected": -322.13616943359375,
      "loss": 0.6448,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.14568084478378296,
      "rewards/margins": 0.10836409777402878,
      "rewards/rejected": -0.25404495000839233,
      "step": 16
    },
    {
      "dpo_losses": 0.6058359146118164,
      "epoch": 0.19101123595505617,
      "grad_norm": 70.67768096923828,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -0.4716479480266571,
      "logits/rejected": -0.27703845500946045,
      "logps/chosen": -197.96463012695312,
      "logps/rejected": -279.54266357421875,
      "loss": 0.927,
      "mlc_losses": 0.32120031118392944,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.17159658670425415,
      "rewards/margins": 0.19868886470794678,
      "rewards/rejected": -0.37028539180755615,
      "step": 17
    },
    {
      "dpo_losses": 0.5746362805366516,
      "epoch": 0.20224719101123595,
      "grad_norm": 71.09721374511719,
      "learning_rate": 3.7777777777777775e-07,
      "logits/chosen": -0.45729735493659973,
      "logits/rejected": -0.18910562992095947,
      "logps/chosen": -214.71493530273438,
      "logps/rejected": -332.02886962890625,
      "loss": 0.5746,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.21361100673675537,
      "rewards/margins": 0.27720141410827637,
      "rewards/rejected": -0.49081242084503174,
      "step": 18
    },
    {
      "dpo_losses": 0.6230942606925964,
      "epoch": 0.21348314606741572,
      "grad_norm": 59.28813934326172,
      "learning_rate": 4e-07,
      "logits/chosen": -0.5592689514160156,
      "logits/rejected": -0.3385673463344574,
      "logps/chosen": -218.03929138183594,
      "logps/rejected": -274.0564880371094,
      "loss": 0.9401,
      "mlc_losses": 0.3170403838157654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25444382429122925,
      "rewards/margins": 0.1756950318813324,
      "rewards/rejected": -0.43013888597488403,
      "step": 19
    },
    {
      "dpo_losses": 0.6030579209327698,
      "epoch": 0.2247191011235955,
      "grad_norm": 61.46278762817383,
      "learning_rate": 4.222222222222222e-07,
      "logits/chosen": -0.2605047821998596,
      "logits/rejected": -0.20405693352222443,
      "logps/chosen": -277.1026916503906,
      "logps/rejected": -324.7381591796875,
      "loss": 0.6031,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3492606282234192,
      "rewards/margins": 0.2280697226524353,
      "rewards/rejected": -0.5773303508758545,
      "step": 20
    },
    {
      "dpo_losses": 0.5197358131408691,
      "epoch": 0.23595505617977527,
      "grad_norm": 57.361427307128906,
      "learning_rate": 4.444444444444444e-07,
      "logits/chosen": -0.5325208306312561,
      "logits/rejected": -0.2736327648162842,
      "logps/chosen": -204.84945678710938,
      "logps/rejected": -273.41217041015625,
      "loss": 0.8475,
      "mlc_losses": 0.32773417234420776,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.27061957120895386,
      "rewards/margins": 0.4501085877418518,
      "rewards/rejected": -0.7207280993461609,
      "step": 21
    },
    {
      "dpo_losses": 0.5812721252441406,
      "epoch": 0.24719101123595505,
      "grad_norm": 57.247581481933594,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -0.3549671173095703,
      "logits/rejected": -0.24516427516937256,
      "logps/chosen": -265.821533203125,
      "logps/rejected": -285.24810791015625,
      "loss": 0.5813,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.506575882434845,
      "rewards/margins": 0.4285789132118225,
      "rewards/rejected": -0.9351547956466675,
      "step": 22
    },
    {
      "dpo_losses": 0.43169087171554565,
      "epoch": 0.25842696629213485,
      "grad_norm": 53.13246154785156,
      "learning_rate": 4.888888888888889e-07,
      "logits/chosen": -0.47587093710899353,
      "logits/rejected": -0.19655972719192505,
      "logps/chosen": -228.7388458251953,
      "logps/rejected": -344.453857421875,
      "loss": 0.7386,
      "mlc_losses": 0.30686941742897034,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.39738669991493225,
      "rewards/margins": 0.795508623123169,
      "rewards/rejected": -1.1928951740264893,
      "step": 23
    },
    {
      "dpo_losses": 0.5544271469116211,
      "epoch": 0.2696629213483146,
      "grad_norm": 57.574771881103516,
      "learning_rate": 5.111111111111111e-07,
      "logits/chosen": -0.242996484041214,
      "logits/rejected": -0.22109737992286682,
      "logps/chosen": -299.983154296875,
      "logps/rejected": -322.911865234375,
      "loss": 0.5544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6205170750617981,
      "rewards/margins": 0.6515043377876282,
      "rewards/rejected": -1.2720214128494263,
      "step": 24
    },
    {
      "dpo_losses": 0.5010688304901123,
      "epoch": 0.2808988764044944,
      "grad_norm": 56.84252166748047,
      "learning_rate": 5.333333333333332e-07,
      "logits/chosen": -0.45927193760871887,
      "logits/rejected": -0.3314107060432434,
      "logps/chosen": -255.6609344482422,
      "logps/rejected": -392.3662414550781,
      "loss": 0.8138,
      "mlc_losses": 0.31273338198661804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5582017302513123,
      "rewards/margins": 0.7420036792755127,
      "rewards/rejected": -1.3002054691314697,
      "step": 25
    },
    {
      "dpo_losses": 0.3757786452770233,
      "epoch": 0.29213483146067415,
      "grad_norm": 39.900604248046875,
      "learning_rate": 5.555555555555555e-07,
      "logits/chosen": -0.5059583187103271,
      "logits/rejected": -0.31819427013397217,
      "logps/chosen": -170.7957763671875,
      "logps/rejected": -268.80108642578125,
      "loss": 0.3758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40587490797042847,
      "rewards/margins": 1.0467809438705444,
      "rewards/rejected": -1.4526557922363281,
      "step": 26
    },
    {
      "dpo_losses": 0.42024677991867065,
      "epoch": 0.30337078651685395,
      "grad_norm": 41.0886344909668,
      "learning_rate": 5.777777777777777e-07,
      "logits/chosen": -0.44776076078414917,
      "logits/rejected": -0.36347073316574097,
      "logps/chosen": -205.3132781982422,
      "logps/rejected": -249.7548828125,
      "loss": 0.7267,
      "mlc_losses": 0.3064589202404022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39828816056251526,
      "rewards/margins": 0.8648014068603516,
      "rewards/rejected": -1.2630894184112549,
      "step": 27
    },
    {
      "dpo_losses": 0.37718990445137024,
      "epoch": 0.3146067415730337,
      "grad_norm": 37.93653106689453,
      "learning_rate": 6e-07,
      "logits/chosen": -0.44035258889198303,
      "logits/rejected": -0.28944191336631775,
      "logps/chosen": -200.85986328125,
      "logps/rejected": -315.3087463378906,
      "loss": 0.3772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3907305598258972,
      "rewards/margins": 1.2273257970809937,
      "rewards/rejected": -1.618056297302246,
      "step": 28
    },
    {
      "dpo_losses": 0.5162763595581055,
      "epoch": 0.3258426966292135,
      "grad_norm": 48.962120056152344,
      "learning_rate": 5.999742982722021e-07,
      "logits/chosen": -0.39359354972839355,
      "logits/rejected": -0.26997244358062744,
      "logps/chosen": -218.52230834960938,
      "logps/rejected": -265.0020446777344,
      "loss": 0.8013,
      "mlc_losses": 0.2850090265274048,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4899088740348816,
      "rewards/margins": 0.7886699438095093,
      "rewards/rejected": -1.2785788774490356,
      "step": 29
    },
    {
      "dpo_losses": 0.7047361135482788,
      "epoch": 0.33707865168539325,
      "grad_norm": 70.55233764648438,
      "learning_rate": 5.998971974926671e-07,
      "logits/chosen": -0.2676395773887634,
      "logits/rejected": -0.3784557282924652,
      "logps/chosen": -297.64178466796875,
      "logps/rejected": -295.2232971191406,
      "loss": 0.7047,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.181154489517212,
      "rewards/margins": 0.23855023086071014,
      "rewards/rejected": -1.4197046756744385,
      "step": 30
    },
    {
      "dpo_losses": 0.6722465753555298,
      "epoch": 0.34831460674157305,
      "grad_norm": 65.62857055664062,
      "learning_rate": 5.997687108722169e-07,
      "logits/chosen": -0.35059821605682373,
      "logits/rejected": -0.24683937430381775,
      "logps/chosen": -222.9384307861328,
      "logps/rejected": -248.0056915283203,
      "loss": 0.9562,
      "mlc_losses": 0.28397178649902344,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8356350064277649,
      "rewards/margins": 0.5491141080856323,
      "rewards/rejected": -1.384749174118042,
      "step": 31
    },
    {
      "dpo_losses": 0.315559983253479,
      "epoch": 0.3595505617977528,
      "grad_norm": 38.70703125,
      "learning_rate": 5.995888604263721e-07,
      "logits/chosen": -0.46895095705986023,
      "logits/rejected": -0.27052780985832214,
      "logps/chosen": -220.70980834960938,
      "logps/rejected": -331.69818115234375,
      "loss": 0.3156,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.5153895616531372,
      "rewards/margins": 1.5916798114776611,
      "rewards/rejected": -2.107069492340088,
      "step": 32
    },
    {
      "dpo_losses": 0.5873298048973083,
      "epoch": 0.3707865168539326,
      "grad_norm": 75.06622314453125,
      "learning_rate": 5.99357676971581e-07,
      "logits/chosen": -0.2887728214263916,
      "logits/rejected": -0.25079360604286194,
      "logps/chosen": -271.7611389160156,
      "logps/rejected": -266.44195556640625,
      "loss": 0.8618,
      "mlc_losses": 0.27448350191116333,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.1023228168487549,
      "rewards/margins": 0.8928325176239014,
      "rewards/rejected": -1.9951553344726562,
      "step": 33
    },
    {
      "dpo_losses": 0.37381216883659363,
      "epoch": 0.38202247191011235,
      "grad_norm": 43.79627227783203,
      "learning_rate": 5.990752001199384e-07,
      "logits/chosen": -0.4255657494068146,
      "logits/rejected": -0.3663692772388458,
      "logps/chosen": -240.51339721679688,
      "logps/rejected": -329.9430847167969,
      "loss": 0.3738,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3766832947731018,
      "rewards/margins": 1.719974160194397,
      "rewards/rejected": -2.0966572761535645,
      "step": 34
    },
    {
      "dpo_losses": 0.2979634404182434,
      "epoch": 0.39325842696629215,
      "grad_norm": 37.07714080810547,
      "learning_rate": 5.987414782723985e-07,
      "logits/chosen": -0.3117474913597107,
      "logits/rejected": -0.2231053113937378,
      "logps/chosen": -241.73509216308594,
      "logps/rejected": -266.0982360839844,
      "loss": 0.5639,
      "mlc_losses": 0.2659069299697876,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4657072126865387,
      "rewards/margins": 1.594313621520996,
      "rewards/rejected": -2.060020685195923,
      "step": 35
    },
    {
      "dpo_losses": 0.3975133001804352,
      "epoch": 0.4044943820224719,
      "grad_norm": 39.59904861450195,
      "learning_rate": 5.983565686104819e-07,
      "logits/chosen": -0.3449154496192932,
      "logits/rejected": -0.21412689983844757,
      "logps/chosen": -211.71279907226562,
      "logps/rejected": -260.5697021484375,
      "loss": 0.3975,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3062106668949127,
      "rewards/margins": 1.3813904523849487,
      "rewards/rejected": -1.687601089477539,
      "step": 36
    },
    {
      "dpo_losses": 0.4449217617511749,
      "epoch": 0.4157303370786517,
      "grad_norm": 49.580989837646484,
      "learning_rate": 5.979205370864779e-07,
      "logits/chosen": -0.3591742217540741,
      "logits/rejected": -0.17171959578990936,
      "logps/chosen": -235.81483459472656,
      "logps/rejected": -299.7149658203125,
      "loss": 0.6816,
      "mlc_losses": 0.2366396188735962,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3341596722602844,
      "rewards/margins": 1.29268479347229,
      "rewards/rejected": -1.6268444061279297,
      "step": 37
    },
    {
      "dpo_losses": 0.32157617807388306,
      "epoch": 0.42696629213483145,
      "grad_norm": 31.368988037109375,
      "learning_rate": 5.974334584121432e-07,
      "logits/chosen": -0.36774033308029175,
      "logits/rejected": -0.24742820858955383,
      "logps/chosen": -201.13804626464844,
      "logps/rejected": -286.78350830078125,
      "loss": 0.3216,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.13782358169555664,
      "rewards/margins": 1.711995005607605,
      "rewards/rejected": -1.8498187065124512,
      "step": 38
    },
    {
      "dpo_losses": 0.39780598878860474,
      "epoch": 0.43820224719101125,
      "grad_norm": 52.22092819213867,
      "learning_rate": 5.968954160459011e-07,
      "logits/chosen": -0.41884854435920715,
      "logits/rejected": -0.3670758605003357,
      "logps/chosen": -237.94482421875,
      "logps/rejected": -289.7718811035156,
      "loss": 0.6304,
      "mlc_losses": 0.23256784677505493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11286990344524384,
      "rewards/margins": 1.5728089809417725,
      "rewards/rejected": -1.6856788396835327,
      "step": 39
    },
    {
      "dpo_losses": 0.35826587677001953,
      "epoch": 0.449438202247191,
      "grad_norm": 41.263267517089844,
      "learning_rate": 5.963065021785413e-07,
      "logits/chosen": -0.26799747347831726,
      "logits/rejected": -0.2562791109085083,
      "logps/chosen": -242.97096252441406,
      "logps/rejected": -277.2116394042969,
      "loss": 0.3583,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.018300659954547882,
      "rewards/margins": 1.5477335453033447,
      "rewards/rejected": -1.529433012008667,
      "step": 40
    },
    {
      "dpo_losses": 0.4649016559123993,
      "epoch": 0.4606741573033708,
      "grad_norm": 54.24028778076172,
      "learning_rate": 5.956668177174233e-07,
      "logits/chosen": -0.43532660603523254,
      "logits/rejected": -0.2562732398509979,
      "logps/chosen": -260.9028015136719,
      "logps/rejected": -282.5660705566406,
      "loss": 0.6716,
      "mlc_losses": 0.2067333459854126,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04655535891652107,
      "rewards/margins": 1.3245792388916016,
      "rewards/rejected": -1.2780237197875977,
      "step": 41
    },
    {
      "dpo_losses": 0.38900989294052124,
      "epoch": 0.47191011235955055,
      "grad_norm": 46.223350524902344,
      "learning_rate": 5.949764722691863e-07,
      "logits/chosen": -0.1549936980009079,
      "logits/rejected": -0.07813873887062073,
      "logps/chosen": -241.74185180664062,
      "logps/rejected": -275.9894714355469,
      "loss": 0.389,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.22368349134922028,
      "rewards/margins": 1.352696180343628,
      "rewards/rejected": -1.1290125846862793,
      "step": 42
    },
    {
      "dpo_losses": 0.28477975726127625,
      "epoch": 0.48314606741573035,
      "grad_norm": 41.09073257446289,
      "learning_rate": 5.942355841209691e-07,
      "logits/chosen": -0.3183077573776245,
      "logits/rejected": -0.17751125991344452,
      "logps/chosen": -196.72952270507812,
      "logps/rejected": -311.1526794433594,
      "loss": 0.4871,
      "mlc_losses": 0.20228275656700134,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3100758194923401,
      "rewards/margins": 1.8401069641113281,
      "rewards/rejected": -1.5300312042236328,
      "step": 43
    },
    {
      "dpo_losses": 0.39458924531936646,
      "epoch": 0.4943820224719101,
      "grad_norm": 66.63091278076172,
      "learning_rate": 5.934442802201417e-07,
      "logits/chosen": -0.10139335691928864,
      "logits/rejected": -0.15596897900104523,
      "logps/chosen": -274.2655944824219,
      "logps/rejected": -362.4077453613281,
      "loss": 0.3946,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.12758769094944,
      "rewards/margins": 1.6785675287246704,
      "rewards/rejected": -1.5509798526763916,
      "step": 44
    },
    {
      "dpo_losses": 0.2832614779472351,
      "epoch": 0.5056179775280899,
      "grad_norm": 37.727481842041016,
      "learning_rate": 5.926026961525537e-07,
      "logits/chosen": -0.28154438734054565,
      "logits/rejected": -0.30871301889419556,
      "logps/chosen": -250.66293334960938,
      "logps/rejected": -333.8875427246094,
      "loss": 0.464,
      "mlc_losses": 0.18073035776615143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2720063030719757,
      "rewards/margins": 1.9518296718597412,
      "rewards/rejected": -1.6798232793807983,
      "step": 45
    },
    {
      "dpo_losses": 0.40544968843460083,
      "epoch": 0.5168539325842697,
      "grad_norm": 35.690757751464844,
      "learning_rate": 5.91710976119303e-07,
      "logits/chosen": -0.3395291566848755,
      "logits/rejected": -0.24968062341213226,
      "logps/chosen": -192.4029083251953,
      "logps/rejected": -257.6268310546875,
      "loss": 0.4054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22519125044345856,
      "rewards/margins": 1.325079083442688,
      "rewards/rejected": -1.0998878479003906,
      "step": 46
    },
    {
      "dpo_losses": 0.34652435779571533,
      "epoch": 0.5280898876404494,
      "grad_norm": 38.09462356567383,
      "learning_rate": 5.907692729120263e-07,
      "logits/chosen": -0.33230119943618774,
      "logits/rejected": -0.2044130116701126,
      "logps/chosen": -175.5661163330078,
      "logps/rejected": -262.1217346191406,
      "loss": 0.5265,
      "mlc_losses": 0.17998380959033966,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5117535591125488,
      "rewards/margins": 1.8927632570266724,
      "rewards/rejected": -1.3810096979141235,
      "step": 47
    },
    {
      "dpo_losses": 0.3377050757408142,
      "epoch": 0.5393258426966292,
      "grad_norm": 47.56745910644531,
      "learning_rate": 5.897777478867205e-07,
      "logits/chosen": -0.22314497828483582,
      "logits/rejected": -0.20754240453243256,
      "logps/chosen": -231.76083374023438,
      "logps/rejected": -300.2922668457031,
      "loss": 0.3377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4299323558807373,
      "rewards/margins": 2.2686116695404053,
      "rewards/rejected": -1.838679313659668,
      "step": 48
    },
    {
      "dpo_losses": 0.3051767945289612,
      "epoch": 0.550561797752809,
      "grad_norm": 34.19914627075195,
      "learning_rate": 5.887365709360941e-07,
      "logits/chosen": -0.33198314905166626,
      "logits/rejected": -0.22209961712360382,
      "logps/chosen": -215.25344848632812,
      "logps/rejected": -303.8016357421875,
      "loss": 0.4602,
      "mlc_losses": 0.15498818457126617,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.351190447807312,
      "rewards/margins": 1.9780161380767822,
      "rewards/rejected": -1.6268256902694702,
      "step": 49
    },
    {
      "dpo_losses": 0.2870302200317383,
      "epoch": 0.5617977528089888,
      "grad_norm": 30.912633895874023,
      "learning_rate": 5.876459204604579e-07,
      "logits/chosen": -0.21440266072750092,
      "logits/rejected": -0.07751217484474182,
      "logps/chosen": -212.61447143554688,
      "logps/rejected": -282.62542724609375,
      "loss": 0.287,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.36938178539276123,
      "rewards/margins": 2.2888667583465576,
      "rewards/rejected": -1.919485092163086,
      "step": 50
    },
    {
      "dpo_losses": 0.2863060235977173,
      "epoch": 0.5730337078651685,
      "grad_norm": 42.67524337768555,
      "learning_rate": 5.86505983337156e-07,
      "logits/chosen": -0.2858361601829529,
      "logits/rejected": -0.05414776876568794,
      "logps/chosen": -216.61541748046875,
      "logps/rejected": -315.2244873046875,
      "loss": 0.4242,
      "mlc_losses": 0.13786199688911438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5145083069801331,
      "rewards/margins": 2.234452724456787,
      "rewards/rejected": -1.7199441194534302,
      "step": 51
    },
    {
      "dpo_losses": 0.6325961947441101,
      "epoch": 0.5842696629213483,
      "grad_norm": 84.55838775634766,
      "learning_rate": 5.85316954888546e-07,
      "logits/chosen": -0.1718163937330246,
      "logits/rejected": -0.22152289748191833,
      "logps/chosen": -310.93927001953125,
      "logps/rejected": -260.3511962890625,
      "loss": 0.6326,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.09869524836540222,
      "rewards/margins": 1.4532307386398315,
      "rewards/rejected": -1.5519260168075562,
      "step": 52
    },
    {
      "dpo_losses": 0.20795783400535583,
      "epoch": 0.5955056179775281,
      "grad_norm": 30.616634368896484,
      "learning_rate": 5.840790388485317e-07,
      "logits/chosen": -0.2612922787666321,
      "logits/rejected": -0.15867209434509277,
      "logps/chosen": -250.6173553466797,
      "logps/rejected": -335.6141662597656,
      "loss": 0.3262,
      "mlc_losses": 0.11820729076862335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11592104285955429,
      "rewards/margins": 2.298637866973877,
      "rewards/rejected": -2.1827168464660645,
      "step": 53
    },
    {
      "dpo_losses": 0.30787473917007446,
      "epoch": 0.6067415730337079,
      "grad_norm": 34.2471809387207,
      "learning_rate": 5.827924473276535e-07,
      "logits/chosen": -0.23087061941623688,
      "logits/rejected": -0.1823207437992096,
      "logps/chosen": -213.14266967773438,
      "logps/rejected": -277.24395751953125,
      "loss": 0.3079,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.607581615447998,
      "rewards/margins": 2.311805248260498,
      "rewards/rejected": -1.7042236328125,
      "step": 54
    },
    {
      "dpo_losses": 0.3049369156360626,
      "epoch": 0.6179775280898876,
      "grad_norm": 41.359657287597656,
      "learning_rate": 5.814574007767452e-07,
      "logits/chosen": -0.18260112404823303,
      "logits/rejected": -0.2203604280948639,
      "logps/chosen": -297.9373474121094,
      "logps/rejected": -280.0711669921875,
      "loss": 0.423,
      "mlc_losses": 0.11801837384700775,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0717427060008049,
      "rewards/margins": 1.9314234256744385,
      "rewards/rejected": -1.8596808910369873,
      "step": 55
    },
    {
      "dpo_losses": 0.24090223014354706,
      "epoch": 0.6292134831460674,
      "grad_norm": 25.504060745239258,
      "learning_rate": 5.800741279491605e-07,
      "logits/chosen": -0.38320815563201904,
      "logits/rejected": -0.2623540163040161,
      "logps/chosen": -188.7933807373047,
      "logps/rejected": -253.9022674560547,
      "loss": 0.2409,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.699580192565918,
      "rewards/margins": 2.642176628112793,
      "rewards/rejected": -1.942596197128296,
      "step": 56
    },
    {
      "dpo_losses": 0.20872285962104797,
      "epoch": 0.6404494382022472,
      "grad_norm": 30.659990310668945,
      "learning_rate": 5.786428658615773e-07,
      "logits/chosen": -0.06069768965244293,
      "logits/rejected": -0.14261652529239655,
      "logps/chosen": -270.9815368652344,
      "logps/rejected": -273.96942138671875,
      "loss": 0.3143,
      "mlc_losses": 0.10553327202796936,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3159443140029907,
      "rewards/margins": 2.776801586151123,
      "rewards/rejected": -2.4608571529388428,
      "step": 57
    },
    {
      "dpo_losses": 0.30205145478248596,
      "epoch": 0.651685393258427,
      "grad_norm": 35.40267562866211,
      "learning_rate": 5.77163859753386e-07,
      "logits/chosen": -0.32091236114501953,
      "logits/rejected": -0.17004680633544922,
      "logps/chosen": -186.09356689453125,
      "logps/rejected": -255.5754852294922,
      "loss": 0.3021,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.08357106149196625,
      "rewards/margins": 2.3176286220550537,
      "rewards/rejected": -2.234057664871216,
      "step": 58
    },
    {
      "dpo_losses": 0.28850117325782776,
      "epoch": 0.6629213483146067,
      "grad_norm": 32.45212936401367,
      "learning_rate": 5.756373630446695e-07,
      "logits/chosen": -0.4012342095375061,
      "logits/rejected": -0.22248440980911255,
      "logps/chosen": -179.804931640625,
      "logps/rejected": -257.164794921875,
      "loss": 0.3899,
      "mlc_losses": 0.10135263204574585,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12721067667007446,
      "rewards/margins": 2.2218923568725586,
      "rewards/rejected": -2.094681739807129,
      "step": 59
    },
    {
      "dpo_losses": 0.30296486616134644,
      "epoch": 0.6741573033707865,
      "grad_norm": 40.26031494140625,
      "learning_rate": 5.740636372927802e-07,
      "logits/chosen": -0.3851511776447296,
      "logits/rejected": -0.19832468032836914,
      "logps/chosen": -179.94442749023438,
      "logps/rejected": -305.33477783203125,
      "loss": 0.303,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2988460659980774,
      "rewards/margins": 2.479841947555542,
      "rewards/rejected": -2.1809957027435303,
      "step": 60
    },
    {
      "dpo_losses": 0.31522196531295776,
      "epoch": 0.6853932584269663,
      "grad_norm": 37.035579681396484,
      "learning_rate": 5.724429521475244e-07,
      "logits/chosen": -0.3988860547542572,
      "logits/rejected": -0.2204786092042923,
      "logps/chosen": -189.47927856445312,
      "logps/rejected": -270.7264709472656,
      "loss": 0.4159,
      "mlc_losses": 0.10066892206668854,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.31987324357032776,
      "rewards/margins": 2.1371607780456543,
      "rewards/rejected": -1.8172876834869385,
      "step": 61
    },
    {
      "dpo_losses": 0.2468000203371048,
      "epoch": 0.6966292134831461,
      "grad_norm": 26.50150489807129,
      "learning_rate": 5.707755853049582e-07,
      "logits/chosen": -0.37235960364341736,
      "logits/rejected": -0.21495608985424042,
      "logps/chosen": -211.15957641601562,
      "logps/rejected": -268.4115295410156,
      "loss": 0.2468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06007450446486473,
      "rewards/margins": 2.347381114959717,
      "rewards/rejected": -2.4074554443359375,
      "step": 62
    },
    {
      "dpo_losses": 0.18872404098510742,
      "epoch": 0.7078651685393258,
      "grad_norm": 26.066137313842773,
      "learning_rate": 5.690618224598064e-07,
      "logits/chosen": -0.23109470307826996,
      "logits/rejected": -0.21755929291248322,
      "logps/chosen": -223.43898010253906,
      "logps/rejected": -260.3247375488281,
      "loss": 0.2869,
      "mlc_losses": 0.09812837094068527,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07142992317676544,
      "rewards/margins": 2.891908884048462,
      "rewards/rejected": -2.963338613510132,
      "step": 63
    },
    {
      "dpo_losses": 0.2657685875892639,
      "epoch": 0.7191011235955056,
      "grad_norm": 35.579185485839844,
      "learning_rate": 5.673019572565103e-07,
      "logits/chosen": -0.3190760016441345,
      "logits/rejected": -0.21535080671310425,
      "logps/chosen": -249.2989044189453,
      "logps/rejected": -300.7616882324219,
      "loss": 0.2658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08060808479785919,
      "rewards/margins": 2.4678738117218018,
      "rewards/rejected": -2.5484819412231445,
      "step": 64
    },
    {
      "dpo_losses": 0.19597342610359192,
      "epoch": 0.7303370786516854,
      "grad_norm": 23.535085678100586,
      "learning_rate": 5.654962912389125e-07,
      "logits/chosen": -0.19642549753189087,
      "logits/rejected": -0.061087049543857574,
      "logps/chosen": -212.7875518798828,
      "logps/rejected": -293.56298828125,
      "loss": 0.2926,
      "mlc_losses": 0.09662766754627228,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12845130264759064,
      "rewards/margins": 2.992464542388916,
      "rewards/rejected": -2.864013195037842,
      "step": 65
    },
    {
      "dpo_losses": 0.31936126947402954,
      "epoch": 0.7415730337078652,
      "grad_norm": 60.769386291503906,
      "learning_rate": 5.636451337985896e-07,
      "logits/chosen": -0.44397950172424316,
      "logits/rejected": -0.30197063088417053,
      "logps/chosen": -243.04217529296875,
      "logps/rejected": -284.6426696777344,
      "loss": 0.3194,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08025036007165909,
      "rewards/margins": 3.3099417686462402,
      "rewards/rejected": -3.22969126701355,
      "step": 66
    },
    {
      "dpo_losses": 0.283762663602829,
      "epoch": 0.7528089887640449,
      "grad_norm": 37.79719161987305,
      "learning_rate": 5.617488021218391e-07,
      "logits/chosen": -0.25557592511177063,
      "logits/rejected": -0.3623576760292053,
      "logps/chosen": -267.0937805175781,
      "logps/rejected": -271.79852294921875,
      "loss": 0.3793,
      "mlc_losses": 0.0955551415681839,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09278859943151474,
      "rewards/margins": 3.0720884799957275,
      "rewards/rejected": -2.979299783706665,
      "step": 67
    },
    {
      "dpo_losses": 0.12901580333709717,
      "epoch": 0.7640449438202247,
      "grad_norm": 21.102943420410156,
      "learning_rate": 5.598076211353317e-07,
      "logits/chosen": -0.2882211208343506,
      "logits/rejected": -0.3936389684677124,
      "logps/chosen": -228.1523895263672,
      "logps/rejected": -300.36572265625,
      "loss": 0.129,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32128483057022095,
      "rewards/margins": 3.8241050243377686,
      "rewards/rejected": -3.5028200149536133,
      "step": 68
    },
    {
      "dpo_losses": 0.29635992646217346,
      "epoch": 0.7752808988764045,
      "grad_norm": 36.00950241088867,
      "learning_rate": 5.578219234504358e-07,
      "logits/chosen": -0.2854154407978058,
      "logits/rejected": -0.14007869362831116,
      "logps/chosen": -279.2039794921875,
      "logps/rejected": -306.55999755859375,
      "loss": 0.3873,
      "mlc_losses": 0.09093204140663147,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.18384435772895813,
      "rewards/margins": 2.4405882358551025,
      "rewards/rejected": -2.256743907928467,
      "step": 69
    },
    {
      "dpo_losses": 0.3256467282772064,
      "epoch": 0.7865168539325843,
      "grad_norm": 43.85177993774414,
      "learning_rate": 5.557920493062276e-07,
      "logits/chosen": -0.22562973201274872,
      "logits/rejected": -0.11599800735712051,
      "logps/chosen": -301.34808349609375,
      "logps/rejected": -364.5964660644531,
      "loss": 0.3256,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0981779396533966,
      "rewards/margins": 2.67763352394104,
      "rewards/rejected": -2.775811195373535,
      "step": 70
    },
    {
      "dpo_losses": 0.26869893074035645,
      "epoch": 0.797752808988764,
      "grad_norm": 37.53038024902344,
      "learning_rate": 5.537183465111921e-07,
      "logits/chosen": -0.3095349967479706,
      "logits/rejected": -0.12219690531492233,
      "logps/chosen": -232.06443786621094,
      "logps/rejected": -314.03155517578125,
      "loss": 0.3599,
      "mlc_losses": 0.09120401740074158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13127678632736206,
      "rewards/margins": 3.3310184478759766,
      "rewards/rejected": -3.1997413635253906,
      "step": 71
    },
    {
      "dpo_losses": 0.32701921463012695,
      "epoch": 0.8089887640449438,
      "grad_norm": 33.53452682495117,
      "learning_rate": 5.516011703836272e-07,
      "logits/chosen": -0.24728164076805115,
      "logits/rejected": -0.2513982951641083,
      "logps/chosen": -206.10000610351562,
      "logps/rejected": -274.163330078125,
      "loss": 0.327,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.08349277079105377,
      "rewards/margins": 2.4448511600494385,
      "rewards/rejected": -2.528343677520752,
      "step": 72
    },
    {
      "dpo_losses": 0.19302591681480408,
      "epoch": 0.8202247191011236,
      "grad_norm": 23.735361099243164,
      "learning_rate": 5.494408836907636e-07,
      "logits/chosen": -0.24546977877616882,
      "logits/rejected": -0.29239946603775024,
      "logps/chosen": -238.60166931152344,
      "logps/rejected": -291.6976318359375,
      "loss": 0.2826,
      "mlc_losses": 0.08959058672189713,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16881439089775085,
      "rewards/margins": 3.7578306198120117,
      "rewards/rejected": -3.5890161991119385,
      "step": 73
    },
    {
      "dpo_losses": 0.1744595468044281,
      "epoch": 0.8314606741573034,
      "grad_norm": 25.490812301635742,
      "learning_rate": 5.472378565866046e-07,
      "logits/chosen": -0.3442319631576538,
      "logits/rejected": -0.17393910884857178,
      "logps/chosen": -239.69276428222656,
      "logps/rejected": -275.39654541015625,
      "loss": 0.1745,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8161203861236572,
      "rewards/margins": 3.4448609352111816,
      "rewards/rejected": -2.6287405490875244,
      "step": 74
    },
    {
      "dpo_losses": 0.17628560960292816,
      "epoch": 0.8426966292134831,
      "grad_norm": 24.84612274169922,
      "learning_rate": 5.449924665485037e-07,
      "logits/chosen": -0.33834511041641235,
      "logits/rejected": -0.18344438076019287,
      "logps/chosen": -209.62879943847656,
      "logps/rejected": -293.17413330078125,
      "loss": 0.2656,
      "mlc_losses": 0.08929458260536194,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.47269555926322937,
      "rewards/margins": 3.615607500076294,
      "rewards/rejected": -3.142911672592163,
      "step": 75
    },
    {
      "dpo_losses": 0.21524450182914734,
      "epoch": 0.8539325842696629,
      "grad_norm": 28.742876052856445,
      "learning_rate": 5.427050983124842e-07,
      "logits/chosen": -0.2760738134384155,
      "logits/rejected": -0.19537319242954254,
      "logps/chosen": -205.47634887695312,
      "logps/rejected": -275.4461669921875,
      "loss": 0.2152,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09524285793304443,
      "rewards/margins": 3.197638511657715,
      "rewards/rejected": -3.1023952960968018,
      "step": 76
    },
    {
      "dpo_losses": 0.43001294136047363,
      "epoch": 0.8651685393258427,
      "grad_norm": 40.152950286865234,
      "learning_rate": 5.403761438073181e-07,
      "logits/chosen": -0.22040213644504547,
      "logits/rejected": -0.16624252498149872,
      "logps/chosen": -203.95123291015625,
      "logps/rejected": -247.06448364257812,
      "loss": 0.5194,
      "mlc_losses": 0.0893571749329567,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.24889257550239563,
      "rewards/margins": 2.5958309173583984,
      "rewards/rejected": -2.8447234630584717,
      "step": 77
    },
    {
      "dpo_losses": 0.38309112191200256,
      "epoch": 0.8764044943820225,
      "grad_norm": 45.54226303100586,
      "learning_rate": 5.380060020873705e-07,
      "logits/chosen": -0.26499372720718384,
      "logits/rejected": -0.23001722991466522,
      "logps/chosen": -223.71719360351562,
      "logps/rejected": -308.1044006347656,
      "loss": 0.3831,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.17075587809085846,
      "rewards/margins": 2.20479154586792,
      "rewards/rejected": -2.0340356826782227,
      "step": 78
    },
    {
      "dpo_losses": 0.27023109793663025,
      "epoch": 0.8876404494382022,
      "grad_norm": 32.474769592285156,
      "learning_rate": 5.355950792642234e-07,
      "logits/chosen": -0.29779571294784546,
      "logits/rejected": -0.1908102035522461,
      "logps/chosen": -385.2134094238281,
      "logps/rejected": -325.4622802734375,
      "loss": 0.3575,
      "mlc_losses": 0.08723724633455276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4042339026927948,
      "rewards/margins": 3.0598676204681396,
      "rewards/rejected": -2.6556341648101807,
      "step": 79
    },
    {
      "dpo_losses": 0.2620306611061096,
      "epoch": 0.898876404494382,
      "grad_norm": 31.36235809326172,
      "learning_rate": 5.331437884370913e-07,
      "logits/chosen": -0.17084719240665436,
      "logits/rejected": -0.16955561935901642,
      "logps/chosen": -232.34341430664062,
      "logps/rejected": -293.1461486816406,
      "loss": 0.262,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.30327221751213074,
      "rewards/margins": 3.2132458686828613,
      "rewards/rejected": -2.909973621368408,
      "step": 80
    },
    {
      "dpo_losses": 0.23416748642921448,
      "epoch": 0.9101123595505618,
      "grad_norm": 26.75750160217285,
      "learning_rate": 5.306525496220379e-07,
      "logits/chosen": -0.42881959676742554,
      "logits/rejected": -0.2682591676712036,
      "logps/chosen": -223.98333740234375,
      "logps/rejected": -298.98565673828125,
      "loss": 0.3208,
      "mlc_losses": 0.08658619225025177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4671562612056732,
      "rewards/margins": 3.1325180530548096,
      "rewards/rejected": -3.599674701690674,
      "step": 81
    },
    {
      "dpo_losses": 0.22752560675144196,
      "epoch": 0.9213483146067416,
      "grad_norm": 32.858821868896484,
      "learning_rate": 5.281217896800093e-07,
      "logits/chosen": -0.3147488534450531,
      "logits/rejected": -0.18721704185009003,
      "logps/chosen": -217.717529296875,
      "logps/rejected": -311.546142578125,
      "loss": 0.2275,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5945912599563599,
      "rewards/margins": 3.5704855918884277,
      "rewards/rejected": -2.9758942127227783,
      "step": 82
    },
    {
      "dpo_losses": 0.2875739634037018,
      "epoch": 0.9325842696629213,
      "grad_norm": 35.29627227783203,
      "learning_rate": 5.255519422436932e-07,
      "logits/chosen": -0.21859852969646454,
      "logits/rejected": -0.15790581703186035,
      "logps/chosen": -209.10311889648438,
      "logps/rejected": -270.3597412109375,
      "loss": 0.3738,
      "mlc_losses": 0.08621428906917572,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.035095468163490295,
      "rewards/margins": 2.3280959129333496,
      "rewards/rejected": -2.3631913661956787,
      "step": 83
    },
    {
      "dpo_losses": 0.22774004936218262,
      "epoch": 0.9438202247191011,
      "grad_norm": 29.526731491088867,
      "learning_rate": 5.229434476432182e-07,
      "logits/chosen": -0.30525314807891846,
      "logits/rejected": -0.2164076715707779,
      "logps/chosen": -228.212646484375,
      "logps/rejected": -335.55596923828125,
      "loss": 0.2277,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.4362592101097107,
      "rewards/margins": 3.3663806915283203,
      "rewards/rejected": -2.930121660232544,
      "step": 84
    },
    {
      "dpo_losses": 0.14130745828151703,
      "epoch": 0.9550561797752809,
      "grad_norm": 22.59909439086914,
      "learning_rate": 5.202967528307056e-07,
      "logits/chosen": -0.3794705271720886,
      "logits/rejected": -0.23356378078460693,
      "logps/chosen": -231.82675170898438,
      "logps/rejected": -286.4014892578125,
      "loss": 0.2272,
      "mlc_losses": 0.08587169647216797,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07730613648891449,
      "rewards/margins": 3.496046304702759,
      "rewards/rejected": -3.4187397956848145,
      "step": 85
    },
    {
      "dpo_losses": 0.15753330290317535,
      "epoch": 0.9662921348314607,
      "grad_norm": 23.591726303100586,
      "learning_rate": 5.176123113036862e-07,
      "logits/chosen": -0.15606281161308289,
      "logits/rejected": -0.15668240189552307,
      "logps/chosen": -245.2542266845703,
      "logps/rejected": -296.13433837890625,
      "loss": 0.1575,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5952562093734741,
      "rewards/margins": 4.4204607009887695,
      "rewards/rejected": -3.825204849243164,
      "step": 86
    },
    {
      "dpo_losses": 0.371351420879364,
      "epoch": 0.9775280898876404,
      "grad_norm": 38.443084716796875,
      "learning_rate": 5.148905830273963e-07,
      "logits/chosen": -0.2363731563091278,
      "logits/rejected": -0.20752808451652527,
      "logps/chosen": -257.8831481933594,
      "logps/rejected": -274.0070495605469,
      "loss": 0.4572,
      "mlc_losses": 0.08587026596069336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21402868628501892,
      "rewards/margins": 2.458051919937134,
      "rewards/rejected": -2.6720805168151855,
      "step": 87
    },
    {
      "dpo_losses": 0.17759592831134796,
      "epoch": 0.9887640449438202,
      "grad_norm": 23.675674438476562,
      "learning_rate": 5.121320343559642e-07,
      "logits/chosen": -0.3223496377468109,
      "logits/rejected": -0.308361679315567,
      "logps/chosen": -230.1756591796875,
      "logps/rejected": -261.1312561035156,
      "loss": 0.1776,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07694463431835175,
      "rewards/margins": 3.427633047103882,
      "rewards/rejected": -3.3506879806518555,
      "step": 88
    },
    {
      "dpo_losses": 0.16600888967514038,
      "epoch": 1.0,
      "grad_norm": 47.198543548583984,
      "learning_rate": 5.093371379525041e-07,
      "logits/chosen": -0.088236004114151,
      "logits/rejected": -0.1937893182039261,
      "logps/chosen": -300.04974365234375,
      "logps/rejected": -275.07000732421875,
      "loss": 0.2496,
      "mlc_losses": 0.0835433378815651,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.4509970545768738,
      "rewards/margins": 3.507065773010254,
      "rewards/rejected": -3.0560686588287354,
      "step": 89
    },
    {
      "dpo_losses": 0.09895338118076324,
      "epoch": 1.0112359550561798,
      "grad_norm": 13.258711814880371,
      "learning_rate": 5.065063727081262e-07,
      "logits/chosen": -0.4080594480037689,
      "logits/rejected": -0.2121903896331787,
      "logps/chosen": -285.6038513183594,
      "logps/rejected": -301.27618408203125,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5650959610939026,
      "rewards/margins": 3.539473056793213,
      "rewards/rejected": -2.974377155303955,
      "step": 90
    },
    {
      "dpo_losses": 0.144447922706604,
      "epoch": 1.0224719101123596,
      "grad_norm": 23.489042282104492,
      "learning_rate": 5.036402236598825e-07,
      "logits/chosen": -0.25496891140937805,
      "logits/rejected": -0.2861313819885254,
      "logps/chosen": -265.3791198730469,
      "logps/rejected": -276.1051025390625,
      "loss": 0.2264,
      "mlc_losses": 0.08196409046649933,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.03591753542423248,
      "rewards/margins": 3.852104902267456,
      "rewards/rejected": -3.8880224227905273,
      "step": 91
    },
    {
      "dpo_losses": 0.20026057958602905,
      "epoch": 1.0337078651685394,
      "grad_norm": 24.441452026367188,
      "learning_rate": 5.007391819076575e-07,
      "logits/chosen": -0.14988964796066284,
      "logits/rejected": -0.11256756633520126,
      "logps/chosen": -254.53488159179688,
      "logps/rejected": -313.51092529296875,
      "loss": 0.2003,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.41513770818710327,
      "rewards/margins": 3.498171806335449,
      "rewards/rejected": -3.0830342769622803,
      "step": 92
    },
    {
      "dpo_losses": 0.09788758307695389,
      "epoch": 1.0449438202247192,
      "grad_norm": 17.23200798034668,
      "learning_rate": 4.978037445300206e-07,
      "logits/chosen": -0.29246068000793457,
      "logits/rejected": -0.32417356967926025,
      "logps/chosen": -224.05776977539062,
      "logps/rejected": -294.0564880371094,
      "loss": 0.1786,
      "mlc_losses": 0.08068183064460754,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.1390118598937988,
      "rewards/margins": 4.216775417327881,
      "rewards/rejected": -3.077763795852661,
      "step": 93
    },
    {
      "dpo_losses": 0.2173929214477539,
      "epoch": 1.0561797752808988,
      "grad_norm": 30.523324966430664,
      "learning_rate": 4.94834414499055e-07,
      "logits/chosen": -0.26029959321022034,
      "logits/rejected": -0.15970386564731598,
      "logps/chosen": -198.63299560546875,
      "logps/rejected": -245.75155639648438,
      "loss": 0.2174,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5172857642173767,
      "rewards/margins": 3.014723539352417,
      "rewards/rejected": -2.4974377155303955,
      "step": 94
    },
    {
      "dpo_losses": 0.07335434854030609,
      "epoch": 1.0674157303370786,
      "grad_norm": 14.638993263244629,
      "learning_rate": 4.918317005941754e-07,
      "logits/chosen": -0.2018231749534607,
      "logits/rejected": -0.20004604756832123,
      "logps/chosen": -311.60003662109375,
      "logps/rejected": -328.9132385253906,
      "loss": 0.1504,
      "mlc_losses": 0.07701252400875092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4170230031013489,
      "rewards/margins": 3.9451489448547363,
      "rewards/rejected": -3.528125762939453,
      "step": 95
    },
    {
      "dpo_losses": 0.10153573751449585,
      "epoch": 1.0786516853932584,
      "grad_norm": 20.468332290649414,
      "learning_rate": 4.887961173149512e-07,
      "logits/chosen": -0.311510294675827,
      "logits/rejected": -0.17456480860710144,
      "logps/chosen": -206.75228881835938,
      "logps/rejected": -294.5263977050781,
      "loss": 0.1015,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.3995720148086548,
      "rewards/margins": 4.546015739440918,
      "rewards/rejected": -3.1464436054229736,
      "step": 96
    },
    {
      "dpo_losses": 0.09093925356864929,
      "epoch": 1.0898876404494382,
      "grad_norm": 19.572603225708008,
      "learning_rate": 4.857281847929502e-07,
      "logits/chosen": -0.24509158730506897,
      "logits/rejected": -0.19152209162712097,
      "logps/chosen": -179.67823791503906,
      "logps/rejected": -282.6198425292969,
      "loss": 0.1686,
      "mlc_losses": 0.07762990146875381,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.7365348935127258,
      "rewards/margins": 4.47177267074585,
      "rewards/rejected": -3.7352375984191895,
      "step": 97
    },
    {
      "dpo_losses": 0.19162285327911377,
      "epoch": 1.101123595505618,
      "grad_norm": 38.019775390625,
      "learning_rate": 4.826284287026162e-07,
      "logits/chosen": -0.1900499165058136,
      "logits/rejected": 0.0013899412006139755,
      "logps/chosen": -232.52049255371094,
      "logps/rejected": -331.609375,
      "loss": 0.1916,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15399914979934692,
      "rewards/margins": 4.031907081604004,
      "rewards/rejected": -3.8779079914093018,
      "step": 98
    },
    {
      "dpo_losses": 0.1716744601726532,
      "epoch": 1.1123595505617978,
      "grad_norm": 22.84613800048828,
      "learning_rate": 4.794973801711977e-07,
      "logits/chosen": -0.1854378581047058,
      "logits/rejected": -0.2181500792503357,
      "logps/chosen": -267.3836669921875,
      "logps/rejected": -289.1987609863281,
      "loss": 0.249,
      "mlc_losses": 0.07727637141942978,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19293330609798431,
      "rewards/margins": 3.2254037857055664,
      "rewards/rejected": -3.032470703125,
      "step": 99
    },
    {
      "dpo_losses": 0.13370123505592346,
      "epoch": 1.1235955056179776,
      "grad_norm": 24.03730583190918,
      "learning_rate": 4.763355756877419e-07,
      "logits/chosen": -0.36160486936569214,
      "logits/rejected": -0.17351248860359192,
      "logps/chosen": -179.75003051757812,
      "logps/rejected": -298.92706298828125,
      "loss": 0.1337,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.7193828821182251,
      "rewards/margins": 4.008037567138672,
      "rewards/rejected": -3.2886548042297363,
      "step": 100
    },
    {
      "dpo_losses": 0.13373428583145142,
      "epoch": 1.1348314606741572,
      "grad_norm": 19.22919464111328,
      "learning_rate": 4.731435570111701e-07,
      "logits/chosen": -0.2976127564907074,
      "logits/rejected": -0.17991909384727478,
      "logps/chosen": -180.16217041015625,
      "logps/rejected": -261.5129699707031,
      "loss": 0.2113,
      "mlc_losses": 0.07757552713155746,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.0447502136230469,
      "rewards/margins": 3.9734246730804443,
      "rewards/rejected": -2.9286744594573975,
      "step": 101
    },
    {
      "dpo_losses": 0.1574607938528061,
      "epoch": 1.146067415730337,
      "grad_norm": 22.214130401611328,
      "learning_rate": 4.699218710774498e-07,
      "logits/chosen": -0.2809007167816162,
      "logits/rejected": -0.39235401153564453,
      "logps/chosen": -232.3876190185547,
      "logps/rejected": -232.58714294433594,
      "loss": 0.1575,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6156913638114929,
      "rewards/margins": 3.4036855697631836,
      "rewards/rejected": -2.787994146347046,
      "step": 102
    },
    {
      "dpo_losses": 0.03843872249126434,
      "epoch": 1.1573033707865168,
      "grad_norm": 7.406707763671875,
      "learning_rate": 4.666710699058806e-07,
      "logits/chosen": -0.2721264958381653,
      "logits/rejected": -0.10290277004241943,
      "logps/chosen": -234.03775024414062,
      "logps/rejected": -331.619873046875,
      "loss": 0.1139,
      "mlc_losses": 0.07547596842050552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4835370182991028,
      "rewards/margins": 5.105040550231934,
      "rewards/rejected": -4.6215033531188965,
      "step": 103
    },
    {
      "dpo_losses": 0.09987197071313858,
      "epoch": 1.1685393258426966,
      "grad_norm": 17.5762882232666,
      "learning_rate": 4.6339171050450815e-07,
      "logits/chosen": -0.28370916843414307,
      "logits/rejected": -0.23584362864494324,
      "logps/chosen": -221.1128692626953,
      "logps/rejected": -318.09405517578125,
      "loss": 0.0999,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.8999959230422974,
      "rewards/margins": 4.440188884735107,
      "rewards/rejected": -3.5401930809020996,
      "step": 104
    },
    {
      "dpo_losses": 0.13613992929458618,
      "epoch": 1.1797752808988764,
      "grad_norm": 21.720300674438477,
      "learning_rate": 4.6008435477468346e-07,
      "logits/chosen": -0.2302273064851761,
      "logits/rejected": -0.0780046284198761,
      "logps/chosen": -252.9706268310547,
      "logps/rejected": -314.6314697265625,
      "loss": 0.2152,
      "mlc_losses": 0.07902853935956955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7986468076705933,
      "rewards/margins": 4.213955402374268,
      "rewards/rejected": -3.415308952331543,
      "step": 105
    },
    {
      "dpo_losses": 0.07160705327987671,
      "epoch": 1.1910112359550562,
      "grad_norm": 12.819025039672852,
      "learning_rate": 4.567495694147846e-07,
      "logits/chosen": -0.22845754027366638,
      "logits/rejected": -0.22594624757766724,
      "logps/chosen": -248.3838348388672,
      "logps/rejected": -275.8777160644531,
      "loss": 0.0716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8828292489051819,
      "rewards/margins": 4.654167175292969,
      "rewards/rejected": -3.77133846282959,
      "step": 106
    },
    {
      "dpo_losses": 0.13211899995803833,
      "epoch": 1.202247191011236,
      "grad_norm": 22.62706184387207,
      "learning_rate": 4.5338792582311554e-07,
      "logits/chosen": -0.3690427243709564,
      "logits/rejected": -0.23113597929477692,
      "logps/chosen": -192.12294006347656,
      "logps/rejected": -306.2420654296875,
      "loss": 0.2069,
      "mlc_losses": 0.07474146783351898,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23642683029174805,
      "rewards/margins": 4.332835674285889,
      "rewards/rejected": -4.096408843994141,
      "step": 107
    },
    {
      "dpo_losses": 0.08297644555568695,
      "epoch": 1.2134831460674158,
      "grad_norm": 14.449069023132324,
      "learning_rate": 4.5e-07,
      "logits/chosen": -0.31904008984565735,
      "logits/rejected": -0.2752552032470703,
      "logps/chosen": -208.72726440429688,
      "logps/rejected": -307.96722412109375,
      "loss": 0.083,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08797970414161682,
      "rewards/margins": 4.371471881866455,
      "rewards/rejected": -4.459451198577881,
      "step": 108
    },
    {
      "dpo_losses": 0.20213258266448975,
      "epoch": 1.2247191011235956,
      "grad_norm": 25.64740753173828,
      "learning_rate": 4.465863724490865e-07,
      "logits/chosen": -0.3261739909648895,
      "logits/rejected": -0.12754617631435394,
      "logps/chosen": -184.1927490234375,
      "logps/rejected": -293.601806640625,
      "loss": 0.2762,
      "mlc_losses": 0.07409306615591049,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3594084680080414,
      "rewards/margins": 4.463174343109131,
      "rewards/rejected": -4.822582244873047,
      "step": 109
    },
    {
      "dpo_losses": 0.19961777329444885,
      "epoch": 1.2359550561797752,
      "grad_norm": 32.54959487915039,
      "learning_rate": 4.431476280778825e-07,
      "logits/chosen": -0.301967054605484,
      "logits/rejected": -0.1478167325258255,
      "logps/chosen": -198.92391967773438,
      "logps/rejected": -339.36376953125,
      "loss": 0.1996,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4834173321723938,
      "rewards/margins": 4.5257673263549805,
      "rewards/rejected": -4.042349338531494,
      "step": 110
    },
    {
      "dpo_losses": 0.08639223873615265,
      "epoch": 1.247191011235955,
      "grad_norm": 16.69534683227539,
      "learning_rate": 4.396843560975334e-07,
      "logits/chosen": -0.2516857385635376,
      "logits/rejected": -0.10403590649366379,
      "logps/chosen": -270.62847900390625,
      "logps/rejected": -327.615478515625,
      "loss": 0.1587,
      "mlc_losses": 0.07227244228124619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33834314346313477,
      "rewards/margins": 4.70582389831543,
      "rewards/rejected": -5.044167518615723,
      "step": 111
    },
    {
      "dpo_losses": 0.05453304946422577,
      "epoch": 1.2584269662921348,
      "grad_norm": 13.081391334533691,
      "learning_rate": 4.36197149921864e-07,
      "logits/chosen": -0.4095671474933624,
      "logits/rejected": -0.20690888166427612,
      "logps/chosen": -212.62738037109375,
      "logps/rejected": -293.46185302734375,
      "loss": 0.0545,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.25598156452178955,
      "rewards/margins": 5.324401378631592,
      "rewards/rejected": -5.068419933319092,
      "step": 112
    },
    {
      "dpo_losses": 0.0838097557425499,
      "epoch": 1.2696629213483146,
      "grad_norm": 24.456132888793945,
      "learning_rate": 4.326866070657004e-07,
      "logits/chosen": -0.3211326599121094,
      "logits/rejected": -0.11328861862421036,
      "logps/chosen": -256.2494812011719,
      "logps/rejected": -333.9193115234375,
      "loss": 0.1557,
      "mlc_losses": 0.07187239825725555,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30535709857940674,
      "rewards/margins": 4.905904293060303,
      "rewards/rejected": -4.600546836853027,
      "step": 113
    },
    {
      "dpo_losses": 0.10422882437705994,
      "epoch": 1.2808988764044944,
      "grad_norm": 21.332326889038086,
      "learning_rate": 4.2915332904248856e-07,
      "logits/chosen": -0.35836607217788696,
      "logits/rejected": -0.15678803622722626,
      "logps/chosen": -197.5643768310547,
      "logps/rejected": -292.4544677734375,
      "loss": 0.1042,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5091192722320557,
      "rewards/margins": 4.762969493865967,
      "rewards/rejected": -4.25385046005249,
      "step": 114
    },
    {
      "dpo_losses": 0.13593825697898865,
      "epoch": 1.2921348314606742,
      "grad_norm": 21.371288299560547,
      "learning_rate": 4.2559792126122843e-07,
      "logits/chosen": -0.286580353975296,
      "logits/rejected": -0.24696046113967896,
      "logps/chosen": -254.81690979003906,
      "logps/rejected": -352.7168273925781,
      "loss": 0.2075,
      "mlc_losses": 0.0715218335390091,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2666030824184418,
      "rewards/margins": 4.322951793670654,
      "rewards/rejected": -4.589555263519287,
      "step": 115
    },
    {
      "dpo_losses": 0.06967614591121674,
      "epoch": 1.303370786516854,
      "grad_norm": 12.814404487609863,
      "learning_rate": 4.220209929227401e-07,
      "logits/chosen": -0.28349387645721436,
      "logits/rejected": -0.1944093406200409,
      "logps/chosen": -226.96372985839844,
      "logps/rejected": -309.6138000488281,
      "loss": 0.0697,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.10722406208515167,
      "rewards/margins": 5.007747173309326,
      "rewards/rejected": -5.11497163772583,
      "step": 116
    },
    {
      "dpo_losses": 0.054774053394794464,
      "epoch": 1.3146067415730336,
      "grad_norm": 10.546464920043945,
      "learning_rate": 4.184231569152801e-07,
      "logits/chosen": -0.2959321141242981,
      "logits/rejected": -0.2736656963825226,
      "logps/chosen": -208.66360473632812,
      "logps/rejected": -313.5944519042969,
      "loss": 0.1262,
      "mlc_losses": 0.0713852122426033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2988254725933075,
      "rewards/margins": 5.079073905944824,
      "rewards/rejected": -4.780248641967773,
      "step": 117
    },
    {
      "dpo_losses": 0.204606831073761,
      "epoch": 1.3258426966292136,
      "grad_norm": 31.066041946411133,
      "learning_rate": 4.1480502970952696e-07,
      "logits/chosen": -0.2541372776031494,
      "logits/rejected": -0.18536823987960815,
      "logps/chosen": -247.82669067382812,
      "logps/rejected": -289.5704345703125,
      "loss": 0.2046,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3525732159614563,
      "rewards/margins": 4.911125183105469,
      "rewards/rejected": -5.263698577880859,
      "step": 118
    },
    {
      "dpo_losses": 0.1353965848684311,
      "epoch": 1.3370786516853932,
      "grad_norm": 23.36940574645996,
      "learning_rate": 4.111672312529509e-07,
      "logits/chosen": -0.27548930048942566,
      "logits/rejected": -0.1714160442352295,
      "logps/chosen": -252.01446533203125,
      "logps/rejected": -321.1877136230469,
      "loss": 0.2055,
      "mlc_losses": 0.07010365277528763,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4893880784511566,
      "rewards/margins": 4.902254104614258,
      "rewards/rejected": -5.391642093658447,
      "step": 119
    },
    {
      "dpo_losses": 0.06808416545391083,
      "epoch": 1.348314606741573,
      "grad_norm": 16.102663040161133,
      "learning_rate": 4.0751038486359e-07,
      "logits/chosen": -0.41380685567855835,
      "logits/rejected": -0.19940871000289917,
      "logps/chosen": -210.97610473632812,
      "logps/rejected": -323.30108642578125,
      "loss": 0.0681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06708921492099762,
      "rewards/margins": 6.033576965332031,
      "rewards/rejected": -5.966487407684326,
      "step": 120
    },
    {
      "dpo_losses": 0.0817258358001709,
      "epoch": 1.3595505617977528,
      "grad_norm": 13.104568481445312,
      "learning_rate": 4.0383511712324786e-07,
      "logits/chosen": -0.21860259771347046,
      "logits/rejected": -0.23902952671051025,
      "logps/chosen": -211.21827697753906,
      "logps/rejected": -330.4417724609375,
      "loss": 0.1519,
      "mlc_losses": 0.07016615569591522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004346087574958801,
      "rewards/margins": 4.778613567352295,
      "rewards/rejected": -4.782959938049316,
      "step": 121
    },
    {
      "dpo_losses": 0.38346266746520996,
      "epoch": 1.3707865168539326,
      "grad_norm": 42.40986633300781,
      "learning_rate": 4.0014205777013123e-07,
      "logits/chosen": -0.34212103486061096,
      "logits/rejected": -0.2788183093070984,
      "logps/chosen": -265.91357421875,
      "logps/rejected": -360.247802734375,
      "loss": 0.3835,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.2556129693984985,
      "rewards/margins": 4.274553298950195,
      "rewards/rejected": -5.530166149139404,
      "step": 122
    },
    {
      "dpo_losses": 0.08436392992734909,
      "epoch": 1.3820224719101124,
      "grad_norm": 14.770329475402832,
      "learning_rate": 3.964318395909485e-07,
      "logits/chosen": -0.26447364687919617,
      "logits/rejected": -0.15754403173923492,
      "logps/chosen": -248.6160125732422,
      "logps/rejected": -344.4999694824219,
      "loss": 0.1535,
      "mlc_losses": 0.06910189986228943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7579646706581116,
      "rewards/margins": 4.782035827636719,
      "rewards/rejected": -5.539999961853027,
      "step": 123
    },
    {
      "dpo_losses": 0.1389930099248886,
      "epoch": 1.3932584269662922,
      "grad_norm": 26.477689743041992,
      "learning_rate": 3.927050983124842e-07,
      "logits/chosen": -0.2337348759174347,
      "logits/rejected": -0.16445569694042206,
      "logps/chosen": -253.83949279785156,
      "logps/rejected": -284.0050048828125,
      "loss": 0.139,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7211804986000061,
      "rewards/margins": 4.585134506225586,
      "rewards/rejected": -5.306314945220947,
      "step": 124
    },
    {
      "dpo_losses": 0.06674088537693024,
      "epoch": 1.404494382022472,
      "grad_norm": 13.284048080444336,
      "learning_rate": 3.8896247249267124e-07,
      "logits/chosen": -0.14707143604755402,
      "logits/rejected": -0.03705991804599762,
      "logps/chosen": -239.51309204101562,
      "logps/rejected": -337.4997863769531,
      "loss": 0.1352,
      "mlc_losses": 0.06848171353340149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4742497503757477,
      "rewards/margins": 5.685024738311768,
      "rewards/rejected": -6.159274101257324,
      "step": 125
    },
    {
      "dpo_losses": 0.07781185209751129,
      "epoch": 1.4157303370786516,
      "grad_norm": 12.782258987426758,
      "learning_rate": 3.8520460341117685e-07,
      "logits/chosen": -0.35959187150001526,
      "logits/rejected": -0.2673090100288391,
      "logps/chosen": -262.33502197265625,
      "logps/rejected": -314.52239990234375,
      "loss": 0.0778,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07514780014753342,
      "rewards/margins": 5.343595504760742,
      "rewards/rejected": -5.268447399139404,
      "step": 126
    },
    {
      "dpo_losses": 0.06309129297733307,
      "epoch": 1.4269662921348314,
      "grad_norm": 15.086901664733887,
      "learning_rate": 3.8143213495952226e-07,
      "logits/chosen": -0.37919941544532776,
      "logits/rejected": -0.33268484473228455,
      "logps/chosen": -212.0961151123047,
      "logps/rejected": -321.0561218261719,
      "loss": 0.1312,
      "mlc_losses": 0.06814850866794586,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.008664190769195557,
      "rewards/margins": 4.6114959716796875,
      "rewards/rejected": -4.602832317352295,
      "step": 127
    },
    {
      "dpo_losses": 0.26559698581695557,
      "epoch": 1.4382022471910112,
      "grad_norm": 36.47563552856445,
      "learning_rate": 3.776457135307562e-07,
      "logits/chosen": -0.2777768671512604,
      "logits/rejected": -0.24057182669639587,
      "logps/chosen": -261.5926513671875,
      "logps/rejected": -291.98675537109375,
      "loss": 0.2656,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.8753973245620728,
      "rewards/margins": 3.637479782104492,
      "rewards/rejected": -4.512876987457275,
      "step": 128
    },
    {
      "dpo_losses": 0.0983341634273529,
      "epoch": 1.449438202247191,
      "grad_norm": 14.851750373840332,
      "learning_rate": 3.7384598790869793e-07,
      "logits/chosen": -0.31900712847709656,
      "logits/rejected": -0.23766086995601654,
      "logps/chosen": -202.1024169921875,
      "logps/rejected": -335.0154113769531,
      "loss": 0.1645,
      "mlc_losses": 0.06616740673780441,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07437404245138168,
      "rewards/margins": 5.405168056488037,
      "rewards/rejected": -5.330793857574463,
      "step": 129
    },
    {
      "dpo_losses": 0.15134823322296143,
      "epoch": 1.4606741573033708,
      "grad_norm": 29.6279239654541,
      "learning_rate": 3.7003360915677166e-07,
      "logits/chosen": -0.313609778881073,
      "logits/rejected": -0.2565464973449707,
      "logps/chosen": -247.48448181152344,
      "logps/rejected": -304.74285888671875,
      "loss": 0.1513,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.15975360572338104,
      "rewards/margins": 4.305423736572266,
      "rewards/rejected": -4.465177536010742,
      "step": 130
    },
    {
      "dpo_losses": 0.050643447786569595,
      "epoch": 1.4719101123595506,
      "grad_norm": 9.25830078125,
      "learning_rate": 3.6620923050645043e-07,
      "logits/chosen": -0.2061961591243744,
      "logits/rejected": -0.19961664080619812,
      "logps/chosen": -247.33958435058594,
      "logps/rejected": -341.1588439941406,
      "loss": 0.1166,
      "mlc_losses": 0.06598802655935287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18948021531105042,
      "rewards/margins": 4.851602077484131,
      "rewards/rejected": -5.04108190536499,
      "step": 131
    },
    {
      "dpo_losses": 0.07581429183483124,
      "epoch": 1.4831460674157304,
      "grad_norm": 14.813788414001465,
      "learning_rate": 3.6237350724532774e-07,
      "logits/chosen": -0.3393017649650574,
      "logits/rejected": -0.12312749028205872,
      "logps/chosen": -211.36793518066406,
      "logps/rejected": -305.57318115234375,
      "loss": 0.0758,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2927740812301636,
      "rewards/margins": 5.743563652038574,
      "rewards/rejected": -6.036337852478027,
      "step": 132
    },
    {
      "dpo_losses": 0.08460303395986557,
      "epoch": 1.49438202247191,
      "grad_norm": 16.35590171813965,
      "learning_rate": 3.585270966048385e-07,
      "logits/chosen": -0.22016192972660065,
      "logits/rejected": -0.27495789527893066,
      "logps/chosen": -254.31793212890625,
      "logps/rejected": -303.04742431640625,
      "loss": 0.1505,
      "mlc_losses": 0.06592501699924469,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.21913376450538635,
      "rewards/margins": 4.255342960357666,
      "rewards/rejected": -4.4744768142700195,
      "step": 133
    },
    {
      "dpo_losses": 0.13376693427562714,
      "epoch": 1.50561797752809,
      "grad_norm": 26.506128311157227,
      "learning_rate": 3.546706576476443e-07,
      "logits/chosen": -0.01358929742127657,
      "logits/rejected": -0.10494533181190491,
      "logps/chosen": -268.7017517089844,
      "logps/rejected": -336.8165283203125,
      "loss": 0.1338,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7277572154998779,
      "rewards/margins": 4.31929874420166,
      "rewards/rejected": -5.047055721282959,
      "step": 134
    },
    {
      "dpo_losses": 0.14806781709194183,
      "epoch": 1.5168539325842696,
      "grad_norm": 24.633228302001953,
      "learning_rate": 3.5080485115470735e-07,
      "logits/chosen": -0.2885373532772064,
      "logits/rejected": -0.21442550420761108,
      "logps/chosen": -244.1745147705078,
      "logps/rejected": -299.1640930175781,
      "loss": 0.2142,
      "mlc_losses": 0.06617090106010437,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.26367422938346863,
      "rewards/margins": 4.991931915283203,
      "rewards/rejected": -5.255606651306152,
      "step": 135
    },
    {
      "dpo_losses": 0.07189897447824478,
      "epoch": 1.5280898876404494,
      "grad_norm": 29.12295913696289,
      "learning_rate": 3.469303395120693e-07,
      "logits/chosen": -0.3134962022304535,
      "logits/rejected": -0.23961186408996582,
      "logps/chosen": -239.83335876464844,
      "logps/rejected": -376.75323486328125,
      "loss": 0.0719,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15510514378547668,
      "rewards/margins": 5.33998966217041,
      "rewards/rejected": -5.184885025024414,
      "step": 136
    },
    {
      "dpo_losses": 0.14414015412330627,
      "epoch": 1.5393258426966292,
      "grad_norm": 28.529674530029297,
      "learning_rate": 3.4304778659735374e-07,
      "logits/chosen": -0.14794759452342987,
      "logits/rejected": -0.16174988448619843,
      "logps/chosen": -239.53713989257812,
      "logps/rejected": -301.4137878417969,
      "loss": 0.209,
      "mlc_losses": 0.06488990783691406,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3111836016178131,
      "rewards/margins": 4.536076545715332,
      "rewards/rejected": -4.847259521484375,
      "step": 137
    },
    {
      "dpo_losses": 0.09966724365949631,
      "epoch": 1.550561797752809,
      "grad_norm": 17.648448944091797,
      "learning_rate": 3.3915785766601553e-07,
      "logits/chosen": -0.26403510570526123,
      "logits/rejected": -0.04745747894048691,
      "logps/chosen": -287.3805236816406,
      "logps/rejected": -419.71282958984375,
      "loss": 0.0997,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.27149900794029236,
      "rewards/margins": 4.855522632598877,
      "rewards/rejected": -5.127021789550781,
      "step": 138
    },
    {
      "dpo_losses": 0.12355044484138489,
      "epoch": 1.5617977528089888,
      "grad_norm": 20.8966121673584,
      "learning_rate": 3.352612192373513e-07,
      "logits/chosen": -0.26808881759643555,
      "logits/rejected": -0.15623272955417633,
      "logps/chosen": -231.2021942138672,
      "logps/rejected": -315.14154052734375,
      "loss": 0.1895,
      "mlc_losses": 0.06591106951236725,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2698032259941101,
      "rewards/margins": 4.267576694488525,
      "rewards/rejected": -4.537380218505859,
      "step": 139
    },
    {
      "dpo_losses": 0.08985699713230133,
      "epoch": 1.5730337078651684,
      "grad_norm": 19.193424224853516,
      "learning_rate": 3.31358538980296e-07,
      "logits/chosen": -0.13713523745536804,
      "logits/rejected": -0.16238881647586823,
      "logps/chosen": -230.34698486328125,
      "logps/rejected": -350.83929443359375,
      "loss": 0.0899,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2129572033882141,
      "rewards/margins": 4.654059886932373,
      "rewards/rejected": -4.8670172691345215,
      "step": 140
    },
    {
      "dpo_losses": 0.07766538113355637,
      "epoch": 1.5842696629213484,
      "grad_norm": 13.890239715576172,
      "learning_rate": 3.2745048559902074e-07,
      "logits/chosen": -0.3723815381526947,
      "logits/rejected": -0.192350834608078,
      "logps/chosen": -201.23947143554688,
      "logps/rejected": -327.0518798828125,
      "loss": 0.1419,
      "mlc_losses": 0.06419488787651062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7386725544929504,
      "rewards/margins": 4.274001121520996,
      "rewards/rejected": -5.012673377990723,
      "step": 141
    },
    {
      "dpo_losses": 0.09778150171041489,
      "epoch": 1.595505617977528,
      "grad_norm": 21.024797439575195,
      "learning_rate": 3.235377287183535e-07,
      "logits/chosen": -0.30659186840057373,
      "logits/rejected": -0.3896043300628662,
      "logps/chosen": -224.7509307861328,
      "logps/rejected": -268.94329833984375,
      "loss": 0.0978,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09393900632858276,
      "rewards/margins": 4.806715965270996,
      "rewards/rejected": -4.7127766609191895,
      "step": 142
    },
    {
      "dpo_losses": 0.04299405217170715,
      "epoch": 1.606741573033708,
      "grad_norm": 8.75938892364502,
      "learning_rate": 3.1962093876904293e-07,
      "logits/chosen": -0.1999714970588684,
      "logits/rejected": -0.16594436764717102,
      "logps/chosen": -265.6045227050781,
      "logps/rejected": -345.1488342285156,
      "loss": 0.1075,
      "mlc_losses": 0.06446287781000137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.029468927532434464,
      "rewards/margins": 5.813629150390625,
      "rewards/rejected": -5.843097686767578,
      "step": 143
    },
    {
      "dpo_losses": 0.16017135977745056,
      "epoch": 1.6179775280898876,
      "grad_norm": 28.66900062561035,
      "learning_rate": 3.1570078687288317e-07,
      "logits/chosen": -0.32689863443374634,
      "logits/rejected": -0.3446465730667114,
      "logps/chosen": -211.67825317382812,
      "logps/rejected": -303.42047119140625,
      "loss": 0.1602,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.002164393663406372,
      "rewards/margins": 5.50520658493042,
      "rewards/rejected": -5.507370948791504,
      "step": 144
    },
    {
      "dpo_losses": 0.07035091519355774,
      "epoch": 1.6292134831460674,
      "grad_norm": 15.066633224487305,
      "learning_rate": 3.117779447277206e-07,
      "logits/chosen": -0.2232438027858734,
      "logits/rejected": -0.06842857599258423,
      "logps/chosen": -224.94381713867188,
      "logps/rejected": -336.5796813964844,
      "loss": 0.1349,
      "mlc_losses": 0.06457129865884781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.008255884051322937,
      "rewards/margins": 5.175896644592285,
      "rewards/rejected": -5.167640686035156,
      "step": 145
    },
    {
      "dpo_losses": 0.08871494978666306,
      "epoch": 1.6404494382022472,
      "grad_norm": 13.664708137512207,
      "learning_rate": 3.07853084492362e-07,
      "logits/chosen": -0.310141384601593,
      "logits/rejected": -0.16705322265625,
      "logps/chosen": -232.72731018066406,
      "logps/rejected": -331.0142822265625,
      "loss": 0.0887,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.18568328022956848,
      "rewards/margins": 4.524687767028809,
      "rewards/rejected": -4.710371017456055,
      "step": 146
    },
    {
      "dpo_losses": 0.06252741813659668,
      "epoch": 1.651685393258427,
      "grad_norm": 16.720922470092773,
      "learning_rate": 3.039268786714033e-07,
      "logits/chosen": -0.24479587376117706,
      "logits/rejected": -0.06710276752710342,
      "logps/chosen": -214.09555053710938,
      "logps/rejected": -332.28912353515625,
      "loss": 0.127,
      "mlc_losses": 0.0644431859254837,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.19779923558235168,
      "rewards/margins": 5.258033752441406,
      "rewards/rejected": -5.4558329582214355,
      "step": 147
    },
    {
      "dpo_losses": 0.07980279624462128,
      "epoch": 1.6629213483146068,
      "grad_norm": 21.935312271118164,
      "learning_rate": 3e-07,
      "logits/chosen": -0.345943421125412,
      "logits/rejected": -0.2532770037651062,
      "logps/chosen": -249.87075805664062,
      "logps/rejected": -341.6824645996094,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08952929079532623,
      "rewards/margins": 4.92242431640625,
      "rewards/rejected": -5.011953353881836,
      "step": 148
    },
    {
      "dpo_losses": 0.1416352391242981,
      "epoch": 1.6741573033707864,
      "grad_norm": 21.115970611572266,
      "learning_rate": 2.960731213285967e-07,
      "logits/chosen": -0.21588975191116333,
      "logits/rejected": -0.1375865936279297,
      "logps/chosen": -248.87254333496094,
      "logps/rejected": -351.1211853027344,
      "loss": 0.2058,
      "mlc_losses": 0.06417912989854813,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2874871790409088,
      "rewards/margins": 4.575304985046387,
      "rewards/rejected": -4.862792491912842,
      "step": 149
    },
    {
      "dpo_losses": 0.15487204492092133,
      "epoch": 1.6853932584269664,
      "grad_norm": 26.075698852539062,
      "learning_rate": 2.921469155076381e-07,
      "logits/chosen": -0.2917395532131195,
      "logits/rejected": -0.20508810877799988,
      "logps/chosen": -227.91390991210938,
      "logps/rejected": -298.4539794921875,
      "loss": 0.1549,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.28144291043281555,
      "rewards/margins": 4.243144512176514,
      "rewards/rejected": -4.524587631225586,
      "step": 150
    },
    {
      "dpo_losses": 0.07866492867469788,
      "epoch": 1.696629213483146,
      "grad_norm": 16.43077278137207,
      "learning_rate": 2.882220552722795e-07,
      "logits/chosen": -0.40706849098205566,
      "logits/rejected": -0.25901156663894653,
      "logps/chosen": -258.6132507324219,
      "logps/rejected": -355.8100891113281,
      "loss": 0.1435,
      "mlc_losses": 0.0648583322763443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09207556396722794,
      "rewards/margins": 5.115144729614258,
      "rewards/rejected": -5.207220554351807,
      "step": 151
    },
    {
      "dpo_losses": 0.10674957185983658,
      "epoch": 1.7078651685393258,
      "grad_norm": 20.68679428100586,
      "learning_rate": 2.8429921312711686e-07,
      "logits/chosen": -0.29668235778808594,
      "logits/rejected": -0.1871112883090973,
      "logps/chosen": -226.0266571044922,
      "logps/rejected": -322.0950012207031,
      "loss": 0.1067,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.018019504845142365,
      "rewards/margins": 4.952302932739258,
      "rewards/rejected": -4.934283256530762,
      "step": 152
    },
    {
      "dpo_losses": 0.08156603574752808,
      "epoch": 1.7191011235955056,
      "grad_norm": 16.307479858398438,
      "learning_rate": 2.8037906123095704e-07,
      "logits/chosen": -0.33899933099746704,
      "logits/rejected": -0.16112667322158813,
      "logps/chosen": -198.04815673828125,
      "logps/rejected": -329.33197021484375,
      "loss": 0.1457,
      "mlc_losses": 0.0641130656003952,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.03161903843283653,
      "rewards/margins": 4.364943981170654,
      "rewards/rejected": -4.3965630531311035,
      "step": 153
    },
    {
      "dpo_losses": 0.07816822826862335,
      "epoch": 1.7303370786516854,
      "grad_norm": 15.523741722106934,
      "learning_rate": 2.7646227128164656e-07,
      "logits/chosen": -0.48672884702682495,
      "logits/rejected": -0.27015620470046997,
      "logps/chosen": -180.87600708007812,
      "logps/rejected": -329.85736083984375,
      "loss": 0.0782,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.40953561663627625,
      "rewards/margins": 5.463593006134033,
      "rewards/rejected": -5.054057598114014,
      "step": 154
    },
    {
      "dpo_losses": 0.11694952100515366,
      "epoch": 1.7415730337078652,
      "grad_norm": 19.596181869506836,
      "learning_rate": 2.725495144009793e-07,
      "logits/chosen": -0.16166327893733978,
      "logits/rejected": -0.29291385412216187,
      "logps/chosen": -274.229736328125,
      "logps/rejected": -303.4178771972656,
      "loss": 0.1812,
      "mlc_losses": 0.06429164856672287,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.654831051826477,
      "rewards/margins": 4.505830764770508,
      "rewards/rejected": -5.160661697387695,
      "step": 155
    },
    {
      "dpo_losses": 0.07120568305253983,
      "epoch": 1.7528089887640448,
      "grad_norm": 13.468121528625488,
      "learning_rate": 2.68641461019704e-07,
      "logits/chosen": -0.4402111768722534,
      "logits/rejected": -0.21577243506908417,
      "logps/chosen": -193.80540466308594,
      "logps/rejected": -295.35418701171875,
      "loss": 0.0712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5379986763000488,
      "rewards/margins": 4.973701477050781,
      "rewards/rejected": -4.435702800750732,
      "step": 156
    },
    {
      "dpo_losses": 0.041705913841724396,
      "epoch": 1.7640449438202248,
      "grad_norm": 8.239550590515137,
      "learning_rate": 2.647387807626487e-07,
      "logits/chosen": -0.2679935097694397,
      "logits/rejected": -0.1744951605796814,
      "logps/chosen": -200.54139709472656,
      "logps/rejected": -332.63067626953125,
      "loss": 0.1051,
      "mlc_losses": 0.06344284862279892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24358133971691132,
      "rewards/margins": 5.900343894958496,
      "rewards/rejected": -5.656763076782227,
      "step": 157
    },
    {
      "dpo_losses": 0.06829548627138138,
      "epoch": 1.7752808988764044,
      "grad_norm": 10.642834663391113,
      "learning_rate": 2.608421423339846e-07,
      "logits/chosen": -0.21014130115509033,
      "logits/rejected": -0.1781398206949234,
      "logps/chosen": -240.89276123046875,
      "logps/rejected": -316.5533447265625,
      "loss": 0.0683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24101391434669495,
      "rewards/margins": 4.410951137542725,
      "rewards/rejected": -4.651965618133545,
      "step": 158
    },
    {
      "dpo_losses": 0.014926884323358536,
      "epoch": 1.7865168539325844,
      "grad_norm": 3.243173837661743,
      "learning_rate": 2.5695221340264623e-07,
      "logits/chosen": -0.3468763530254364,
      "logits/rejected": -0.26080965995788574,
      "logps/chosen": -251.13156127929688,
      "logps/rejected": -353.1765441894531,
      "loss": 0.0788,
      "mlc_losses": 0.0639205127954483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15816554427146912,
      "rewards/margins": 6.206686019897461,
      "rewards/rejected": -6.048520088195801,
      "step": 159
    },
    {
      "dpo_losses": 0.08289279043674469,
      "epoch": 1.797752808988764,
      "grad_norm": 16.09496307373047,
      "learning_rate": 2.530696604879307e-07,
      "logits/chosen": -0.22804990410804749,
      "logits/rejected": -0.22570639848709106,
      "logps/chosen": -252.3598175048828,
      "logps/rejected": -341.9347839355469,
      "loss": 0.0829,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6247970461845398,
      "rewards/margins": 4.6536478996276855,
      "rewards/rejected": -5.278445243835449,
      "step": 160
    },
    {
      "dpo_losses": 0.12551791965961456,
      "epoch": 1.8089887640449438,
      "grad_norm": 22.76595115661621,
      "learning_rate": 2.491951488452926e-07,
      "logits/chosen": -0.40357670187950134,
      "logits/rejected": -0.19471757113933563,
      "logps/chosen": -210.02505493164062,
      "logps/rejected": -298.2767639160156,
      "loss": 0.189,
      "mlc_losses": 0.06349781155586243,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4647875428199768,
      "rewards/margins": 4.426042556762695,
      "rewards/rejected": -4.890830039978027,
      "step": 161
    },
    {
      "dpo_losses": 0.10385690629482269,
      "epoch": 1.8202247191011236,
      "grad_norm": 18.70224380493164,
      "learning_rate": 2.4532934235235574e-07,
      "logits/chosen": -0.4252513647079468,
      "logits/rejected": -0.3327357769012451,
      "logps/chosen": -242.18826293945312,
      "logps/rejected": -349.037353515625,
      "loss": 0.1039,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5017452239990234,
      "rewards/margins": 5.340740203857422,
      "rewards/rejected": -5.8424859046936035,
      "step": 162
    },
    {
      "dpo_losses": 0.041663940995931625,
      "epoch": 1.8314606741573034,
      "grad_norm": 10.261404037475586,
      "learning_rate": 2.414729033951615e-07,
      "logits/chosen": -0.32731732726097107,
      "logits/rejected": -0.22145508229732513,
      "logps/chosen": -274.0537414550781,
      "logps/rejected": -338.3710021972656,
      "loss": 0.1056,
      "mlc_losses": 0.06398442387580872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7045354843139648,
      "rewards/margins": 5.074031829833984,
      "rewards/rejected": -5.778566837310791,
      "step": 163
    },
    {
      "dpo_losses": 0.14561866223812103,
      "epoch": 1.8426966292134832,
      "grad_norm": 27.830699920654297,
      "learning_rate": 2.3762649275467223e-07,
      "logits/chosen": -0.3900427222251892,
      "logits/rejected": -0.19633735716342926,
      "logps/chosen": -240.07278442382812,
      "logps/rejected": -348.85028076171875,
      "loss": 0.1456,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6589416265487671,
      "rewards/margins": 4.564722537994385,
      "rewards/rejected": -5.223663806915283,
      "step": 164
    },
    {
      "dpo_losses": 0.13260620832443237,
      "epoch": 1.8539325842696628,
      "grad_norm": 23.316669464111328,
      "learning_rate": 2.337907694935497e-07,
      "logits/chosen": -0.28910672664642334,
      "logits/rejected": -0.19311615824699402,
      "logps/chosen": -269.94500732421875,
      "logps/rejected": -389.9777526855469,
      "loss": 0.1951,
      "mlc_losses": 0.06252987682819366,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7635687589645386,
      "rewards/margins": 4.6578369140625,
      "rewards/rejected": -6.42140531539917,
      "step": 165
    },
    {
      "dpo_losses": 0.07785408943891525,
      "epoch": 1.8651685393258428,
      "grad_norm": 18.2708740234375,
      "learning_rate": 2.2996639084322847e-07,
      "logits/chosen": -0.3967912197113037,
      "logits/rejected": -0.24030345678329468,
      "logps/chosen": -208.11325073242188,
      "logps/rejected": -305.0826416015625,
      "loss": 0.0779,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5448833107948303,
      "rewards/margins": 4.877796649932861,
      "rewards/rejected": -5.422679424285889,
      "step": 166
    },
    {
      "dpo_losses": 0.05128047615289688,
      "epoch": 1.8764044943820224,
      "grad_norm": 11.277166366577148,
      "learning_rate": 2.2615401209130207e-07,
      "logits/chosen": -0.3138977289199829,
      "logits/rejected": -0.34153103828430176,
      "logps/chosen": -255.98904418945312,
      "logps/rejected": -326.63525390625,
      "loss": 0.1155,
      "mlc_losses": 0.06419802457094193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45599907636642456,
      "rewards/margins": 5.403050422668457,
      "rewards/rejected": -5.859049320220947,
      "step": 167
    },
    {
      "dpo_losses": 0.03780509904026985,
      "epoch": 1.8876404494382022,
      "grad_norm": 7.31669807434082,
      "learning_rate": 2.2235428646924374e-07,
      "logits/chosen": -0.35764533281326294,
      "logits/rejected": -0.14056682586669922,
      "logps/chosen": -210.55401611328125,
      "logps/rejected": -339.6280517578125,
      "loss": 0.0378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37859785556793213,
      "rewards/margins": 6.102359771728516,
      "rewards/rejected": -6.480957508087158,
      "step": 168
    },
    {
      "dpo_losses": 0.05484795570373535,
      "epoch": 1.898876404494382,
      "grad_norm": 12.332880020141602,
      "learning_rate": 2.1856786504047773e-07,
      "logits/chosen": -0.33242425322532654,
      "logits/rejected": -0.27086740732192993,
      "logps/chosen": -219.1328582763672,
      "logps/rejected": -325.2514343261719,
      "loss": 0.1174,
      "mlc_losses": 0.06255704164505005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7272806167602539,
      "rewards/margins": 6.219586372375488,
      "rewards/rejected": -6.946866989135742,
      "step": 169
    },
    {
      "dpo_losses": 0.16579382121562958,
      "epoch": 1.9101123595505618,
      "grad_norm": 29.029022216796875,
      "learning_rate": 2.147953965888232e-07,
      "logits/chosen": -0.4297390282154083,
      "logits/rejected": -0.3961951434612274,
      "logps/chosen": -194.8955078125,
      "logps/rejected": -328.2601318359375,
      "loss": 0.1658,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6970565915107727,
      "rewards/margins": 4.82431697845459,
      "rewards/rejected": -5.521373748779297,
      "step": 170
    },
    {
      "dpo_losses": 0.07312853634357452,
      "epoch": 1.9213483146067416,
      "grad_norm": 15.003425598144531,
      "learning_rate": 2.1103752750732873e-07,
      "logits/chosen": -0.3375908434391022,
      "logits/rejected": -0.22286397218704224,
      "logps/chosen": -209.067138671875,
      "logps/rejected": -317.974609375,
      "loss": 0.1371,
      "mlc_losses": 0.0639248937368393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19853639602661133,
      "rewards/margins": 6.132803440093994,
      "rewards/rejected": -6.331339359283447,
      "step": 171
    },
    {
      "dpo_losses": 0.12832948565483093,
      "epoch": 1.9325842696629212,
      "grad_norm": 18.92385482788086,
      "learning_rate": 2.072949016875158e-07,
      "logits/chosen": -0.2988186478614807,
      "logits/rejected": -0.32615768909454346,
      "logps/chosen": -238.4849853515625,
      "logps/rejected": -344.54779052734375,
      "loss": 0.1283,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8068071603775024,
      "rewards/margins": 5.873997211456299,
      "rewards/rejected": -6.680803298950195,
      "step": 172
    },
    {
      "dpo_losses": 0.08407251536846161,
      "epoch": 1.9438202247191012,
      "grad_norm": 16.64447784423828,
      "learning_rate": 2.0356816040905157e-07,
      "logits/chosen": -0.2760240435600281,
      "logits/rejected": -0.35169804096221924,
      "logps/chosen": -272.1250305175781,
      "logps/rejected": -324.3094482421875,
      "loss": 0.1472,
      "mlc_losses": 0.06315474957227707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.808717966079712,
      "rewards/margins": 4.2688374519348145,
      "rewards/rejected": -6.0775556564331055,
      "step": 173
    },
    {
      "dpo_losses": 0.08654741942882538,
      "epoch": 1.9550561797752808,
      "grad_norm": 19.405454635620117,
      "learning_rate": 1.9985794222986874e-07,
      "logits/chosen": -0.4762701988220215,
      "logits/rejected": -0.31571152806282043,
      "logps/chosen": -224.74395751953125,
      "logps/rejected": -396.0409240722656,
      "loss": 0.0865,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08119875192642212,
      "rewards/margins": 5.23417854309082,
      "rewards/rejected": -5.3153767585754395,
      "step": 174
    },
    {
      "dpo_losses": 0.07921688258647919,
      "epoch": 1.9662921348314608,
      "grad_norm": 16.020570755004883,
      "learning_rate": 1.9616488287675203e-07,
      "logits/chosen": -0.38738688826560974,
      "logits/rejected": -0.24907676875591278,
      "logps/chosen": -227.8881072998047,
      "logps/rejected": -338.5246887207031,
      "loss": 0.1419,
      "mlc_losses": 0.06269218772649765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6166845560073853,
      "rewards/margins": 5.253000259399414,
      "rewards/rejected": -6.869684219360352,
      "step": 175
    },
    {
      "dpo_losses": 0.06475147604942322,
      "epoch": 1.9775280898876404,
      "grad_norm": 12.440558433532715,
      "learning_rate": 1.9248961513640992e-07,
      "logits/chosen": -0.3266042470932007,
      "logits/rejected": -0.25882574915885925,
      "logps/chosen": -237.4032440185547,
      "logps/rejected": -313.2313232421875,
      "loss": 0.0648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9604462385177612,
      "rewards/margins": 5.197487831115723,
      "rewards/rejected": -6.157933712005615,
      "step": 176
    },
    {
      "dpo_losses": 0.12705591320991516,
      "epoch": 1.9887640449438202,
      "grad_norm": 22.909664154052734,
      "learning_rate": 1.888327687470491e-07,
      "logits/chosen": -0.3426825702190399,
      "logits/rejected": -0.24973973631858826,
      "logps/chosen": -234.8782501220703,
      "logps/rejected": -344.54193115234375,
      "loss": 0.1892,
      "mlc_losses": 0.06217695772647858,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0709067583084106,
      "rewards/margins": 5.893606185913086,
      "rewards/rejected": -6.964512825012207,
      "step": 177
    },
    {
      "dpo_losses": 0.1073857843875885,
      "epoch": 2.0,
      "grad_norm": 23.07292366027832,
      "learning_rate": 1.8519497029047307e-07,
      "logits/chosen": -0.3683345317840576,
      "logits/rejected": -0.2387535721063614,
      "logps/chosen": -238.7364959716797,
      "logps/rejected": -346.25872802734375,
      "loss": 0.1074,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6798160076141357,
      "rewards/margins": 5.703872203826904,
      "rewards/rejected": -6.383687973022461,
      "step": 178
    },
    {
      "dpo_losses": 0.06853682547807693,
      "epoch": 2.0112359550561796,
      "grad_norm": 11.090790748596191,
      "learning_rate": 1.8157684308471986e-07,
      "logits/chosen": -0.3379666805267334,
      "logits/rejected": -0.2796818017959595,
      "logps/chosen": -224.72470092773438,
      "logps/rejected": -324.3167724609375,
      "loss": 0.1312,
      "mlc_losses": 0.06269238144159317,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9952882528305054,
      "rewards/margins": 6.392023086547852,
      "rewards/rejected": -7.387310981750488,
      "step": 179
    },
    {
      "dpo_losses": 0.034240931272506714,
      "epoch": 2.0224719101123596,
      "grad_norm": 8.904126167297363,
      "learning_rate": 1.7797900707725997e-07,
      "logits/chosen": -0.3933045268058777,
      "logits/rejected": -0.3473502993583679,
      "logps/chosen": -286.2529296875,
      "logps/rejected": -348.6288146972656,
      "loss": 0.0342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6954650282859802,
      "rewards/margins": 5.932584762573242,
      "rewards/rejected": -6.628049850463867,
      "step": 180
    },
    {
      "dpo_losses": 0.041446104645729065,
      "epoch": 2.033707865168539,
      "grad_norm": 8.44710922241211,
      "learning_rate": 1.7440207873877165e-07,
      "logits/chosen": -0.3652418851852417,
      "logits/rejected": -0.3678028881549835,
      "logps/chosen": -253.0131072998047,
      "logps/rejected": -313.9668884277344,
      "loss": 0.1039,
      "mlc_losses": 0.06246665492653847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2076749801635742,
      "rewards/margins": 5.682599067687988,
      "rewards/rejected": -6.890274524688721,
      "step": 181
    },
    {
      "dpo_losses": 0.05633506551384926,
      "epoch": 2.044943820224719,
      "grad_norm": 11.671477317810059,
      "learning_rate": 1.708466709575114e-07,
      "logits/chosen": -0.3676392436027527,
      "logits/rejected": -0.32216766476631165,
      "logps/chosen": -248.23248291015625,
      "logps/rejected": -313.3765869140625,
      "loss": 0.0563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5407989025115967,
      "rewards/margins": 4.987557888031006,
      "rewards/rejected": -6.528356552124023,
      "step": 182
    },
    {
      "dpo_losses": 0.1289069503545761,
      "epoch": 2.056179775280899,
      "grad_norm": 24.947187423706055,
      "learning_rate": 1.6731339293429966e-07,
      "logits/chosen": -0.39845460653305054,
      "logits/rejected": -0.21997562050819397,
      "logps/chosen": -223.6796417236328,
      "logps/rejected": -319.8377685546875,
      "loss": 0.1916,
      "mlc_losses": 0.06271415203809738,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1986894607543945,
      "rewards/margins": 4.936867713928223,
      "rewards/rejected": -6.135556697845459,
      "step": 183
    },
    {
      "dpo_losses": 0.04371622949838638,
      "epoch": 2.067415730337079,
      "grad_norm": 9.09304141998291,
      "learning_rate": 1.6380285007813597e-07,
      "logits/chosen": -0.2974357008934021,
      "logits/rejected": -0.29195693135261536,
      "logps/chosen": -293.3783264160156,
      "logps/rejected": -341.58245849609375,
      "loss": 0.0437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5599305033683777,
      "rewards/margins": 5.891586780548096,
      "rewards/rejected": -6.451517581939697,
      "step": 184
    },
    {
      "dpo_losses": 0.017477022483944893,
      "epoch": 2.0786516853932584,
      "grad_norm": 3.72212553024292,
      "learning_rate": 1.6031564390246656e-07,
      "logits/chosen": -0.24825187027454376,
      "logits/rejected": -0.26398423314094543,
      "logps/chosen": -247.05274963378906,
      "logps/rejected": -346.163818359375,
      "loss": 0.0796,
      "mlc_losses": 0.062171097844839096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6356462836265564,
      "rewards/margins": 6.192551136016846,
      "rewards/rejected": -6.828197002410889,
      "step": 185
    },
    {
      "dpo_losses": 0.025800012052059174,
      "epoch": 2.0898876404494384,
      "grad_norm": 6.956277847290039,
      "learning_rate": 1.5685237192211744e-07,
      "logits/chosen": -0.31300145387649536,
      "logits/rejected": -0.2773960530757904,
      "logps/chosen": -254.7395782470703,
      "logps/rejected": -357.224609375,
      "loss": 0.0258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8892070651054382,
      "rewards/margins": 7.0441694259643555,
      "rewards/rejected": -7.933376312255859,
      "step": 186
    },
    {
      "dpo_losses": 0.045497894287109375,
      "epoch": 2.101123595505618,
      "grad_norm": 11.339599609375,
      "learning_rate": 1.5341362755091361e-07,
      "logits/chosen": -0.34829410910606384,
      "logits/rejected": -0.27513477206230164,
      "logps/chosen": -280.6045227050781,
      "logps/rejected": -329.5090026855469,
      "loss": 0.1075,
      "mlc_losses": 0.06198051944375038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.004107922315597534,
      "rewards/margins": 6.041309356689453,
      "rewards/rejected": -6.037200927734375,
      "step": 187
    },
    {
      "dpo_losses": 0.03118971921503544,
      "epoch": 2.1123595505617976,
      "grad_norm": 5.238372802734375,
      "learning_rate": 1.5000000000000007e-07,
      "logits/chosen": -0.4346851706504822,
      "logits/rejected": -0.3925075829029083,
      "logps/chosen": -204.81137084960938,
      "logps/rejected": -311.43414306640625,
      "loss": 0.0312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31050488352775574,
      "rewards/margins": 5.840826988220215,
      "rewards/rejected": -6.151331901550293,
      "step": 188
    },
    {
      "dpo_losses": 0.06403262168169022,
      "epoch": 2.1235955056179776,
      "grad_norm": 15.77838134765625,
      "learning_rate": 1.4661207417688443e-07,
      "logits/chosen": -0.40519893169403076,
      "logits/rejected": -0.2997415363788605,
      "logps/chosen": -218.44818115234375,
      "logps/rejected": -323.7392578125,
      "loss": 0.1254,
      "mlc_losses": 0.061388347297906876,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6717126369476318,
      "rewards/margins": 6.0542097091674805,
      "rewards/rejected": -6.725922107696533,
      "step": 189
    },
    {
      "dpo_losses": 0.0376722477376461,
      "epoch": 2.134831460674157,
      "grad_norm": 7.452267169952393,
      "learning_rate": 1.4325043058521534e-07,
      "logits/chosen": -0.4307040870189667,
      "logits/rejected": -0.417324423789978,
      "logps/chosen": -247.27366638183594,
      "logps/rejected": -312.93115234375,
      "loss": 0.0377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9737645387649536,
      "rewards/margins": 5.415378570556641,
      "rewards/rejected": -6.389143466949463,
      "step": 190
    },
    {
      "dpo_losses": 0.01752106472849846,
      "epoch": 2.146067415730337,
      "grad_norm": 5.059136867523193,
      "learning_rate": 1.3991564522531654e-07,
      "logits/chosen": -0.4137434959411621,
      "logits/rejected": -0.2022404670715332,
      "logps/chosen": -323.05517578125,
      "logps/rejected": -366.1656188964844,
      "loss": 0.0793,
      "mlc_losses": 0.06176237389445305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.00825590267777443,
      "rewards/margins": 6.773263931274414,
      "rewards/rejected": -6.765007972717285,
      "step": 191
    },
    {
      "dpo_losses": 0.03828878328204155,
      "epoch": 2.157303370786517,
      "grad_norm": 8.782641410827637,
      "learning_rate": 1.3660828949549187e-07,
      "logits/chosen": -0.37221458554267883,
      "logits/rejected": -0.318632036447525,
      "logps/chosen": -273.8665466308594,
      "logps/rejected": -328.75146484375,
      "loss": 0.0383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0055478811264038,
      "rewards/margins": 5.495824813842773,
      "rewards/rejected": -6.501372814178467,
      "step": 192
    },
    {
      "dpo_losses": 0.08497800678014755,
      "epoch": 2.168539325842697,
      "grad_norm": 16.277324676513672,
      "learning_rate": 1.3332893009411942e-07,
      "logits/chosen": -0.30167222023010254,
      "logits/rejected": -0.2558479309082031,
      "logps/chosen": -253.45372009277344,
      "logps/rejected": -338.7463684082031,
      "loss": 0.1458,
      "mlc_losses": 0.06081393361091614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3000555038452148,
      "rewards/margins": 4.583809852600098,
      "rewards/rejected": -5.883865833282471,
      "step": 193
    },
    {
      "dpo_losses": 0.06383566558361053,
      "epoch": 2.1797752808988764,
      "grad_norm": 11.171575546264648,
      "learning_rate": 1.3007812892255022e-07,
      "logits/chosen": -0.3781083822250366,
      "logits/rejected": -0.3593296408653259,
      "logps/chosen": -237.28485107421875,
      "logps/rejected": -314.3851013183594,
      "loss": 0.0638,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0867336988449097,
      "rewards/margins": 4.962950229644775,
      "rewards/rejected": -6.049684047698975,
      "step": 194
    },
    {
      "dpo_losses": 0.038912467658519745,
      "epoch": 2.191011235955056,
      "grad_norm": 9.153482437133789,
      "learning_rate": 1.2685644298882994e-07,
      "logits/chosen": -0.3644222021102905,
      "logits/rejected": -0.2972428798675537,
      "logps/chosen": -204.49586486816406,
      "logps/rejected": -315.7609558105469,
      "loss": 0.1004,
      "mlc_losses": 0.06150813400745392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5360009670257568,
      "rewards/margins": 6.3270111083984375,
      "rewards/rejected": -6.863011837005615,
      "step": 195
    },
    {
      "dpo_losses": 0.04646293818950653,
      "epoch": 2.202247191011236,
      "grad_norm": 8.889599800109863,
      "learning_rate": 1.2366442431225808e-07,
      "logits/chosen": -0.4420759379863739,
      "logits/rejected": -0.3031543791294098,
      "logps/chosen": -233.81893920898438,
      "logps/rejected": -360.5773010253906,
      "loss": 0.0465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7964137196540833,
      "rewards/margins": 5.345058441162109,
      "rewards/rejected": -6.141471862792969,
      "step": 196
    },
    {
      "dpo_losses": 0.0682201161980629,
      "epoch": 2.2134831460674156,
      "grad_norm": 18.0560245513916,
      "learning_rate": 1.2050261982880228e-07,
      "logits/chosen": -0.3851815164089203,
      "logits/rejected": -0.33557039499282837,
      "logps/chosen": -248.2822265625,
      "logps/rejected": -342.8601379394531,
      "loss": 0.1297,
      "mlc_losses": 0.06145855784416199,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8079060912132263,
      "rewards/margins": 5.431468486785889,
      "rewards/rejected": -6.239373683929443,
      "step": 197
    },
    {
      "dpo_losses": 0.06225328519940376,
      "epoch": 2.2247191011235956,
      "grad_norm": 10.684039115905762,
      "learning_rate": 1.173715712973838e-07,
      "logits/chosen": -0.39869123697280884,
      "logits/rejected": -0.2800598740577698,
      "logps/chosen": -205.54550170898438,
      "logps/rejected": -313.67340087890625,
      "loss": 0.0623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9008889198303223,
      "rewards/margins": 4.465407848358154,
      "rewards/rejected": -5.366297245025635,
      "step": 198
    },
    {
      "dpo_losses": 0.03898364678025246,
      "epoch": 2.235955056179775,
      "grad_norm": 8.996004104614258,
      "learning_rate": 1.1427181520704978e-07,
      "logits/chosen": -0.534652054309845,
      "logits/rejected": -0.35140612721443176,
      "logps/chosen": -205.64306640625,
      "logps/rejected": -302.67889404296875,
      "loss": 0.0998,
      "mlc_losses": 0.06085606664419174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8730618357658386,
      "rewards/margins": 5.851526260375977,
      "rewards/rejected": -6.724588394165039,
      "step": 199
    },
    {
      "dpo_losses": 0.023045441135764122,
      "epoch": 2.247191011235955,
      "grad_norm": 5.587195873260498,
      "learning_rate": 1.112038826850488e-07,
      "logits/chosen": -0.4195081889629364,
      "logits/rejected": -0.2087799608707428,
      "logps/chosen": -183.2427215576172,
      "logps/rejected": -331.56658935546875,
      "loss": 0.023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4680759310722351,
      "rewards/margins": 5.896218299865723,
      "rewards/rejected": -6.364294528961182,
      "step": 200
    },
    {
      "dpo_losses": 0.06403011083602905,
      "epoch": 2.258426966292135,
      "grad_norm": 11.759297370910645,
      "learning_rate": 1.081682994058246e-07,
      "logits/chosen": -0.32560765743255615,
      "logits/rejected": -0.4016960561275482,
      "logps/chosen": -243.51626586914062,
      "logps/rejected": -304.4052734375,
      "loss": 0.1257,
      "mlc_losses": 0.06167040020227432,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5608245134353638,
      "rewards/margins": 5.103170871734619,
      "rewards/rejected": -5.663995265960693,
      "step": 201
    },
    {
      "dpo_losses": 0.03536307439208031,
      "epoch": 2.2696629213483144,
      "grad_norm": 5.884720802307129,
      "learning_rate": 1.0516558550094493e-07,
      "logits/chosen": -0.3747088313102722,
      "logits/rejected": -0.2546820342540741,
      "logps/chosen": -204.94891357421875,
      "logps/rejected": -285.1376647949219,
      "loss": 0.0354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1354855298995972,
      "rewards/margins": 5.10093879699707,
      "rewards/rejected": -6.236424446105957,
      "step": 202
    },
    {
      "dpo_losses": 0.041959796100854874,
      "epoch": 2.2808988764044944,
      "grad_norm": 8.450247764587402,
      "learning_rate": 1.0219625546997934e-07,
      "logits/chosen": -0.4256511628627777,
      "logits/rejected": -0.255771666765213,
      "logps/chosen": -230.3624267578125,
      "logps/rejected": -354.3971862792969,
      "loss": 0.1027,
      "mlc_losses": 0.060715921223163605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3055556118488312,
      "rewards/margins": 7.132266521453857,
      "rewards/rejected": -6.8267107009887695,
      "step": 203
    },
    {
      "dpo_losses": 0.01571439951658249,
      "epoch": 2.292134831460674,
      "grad_norm": 3.252563238143921,
      "learning_rate": 9.926081809234262e-08,
      "logits/chosen": -0.38497430086135864,
      "logits/rejected": -0.29007449746131897,
      "logps/chosen": -245.62159729003906,
      "logps/rejected": -321.3457336425781,
      "loss": 0.0157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4209508001804352,
      "rewards/margins": 6.973872184753418,
      "rewards/rejected": -7.39482307434082,
      "step": 204
    },
    {
      "dpo_losses": 0.03233326971530914,
      "epoch": 2.303370786516854,
      "grad_norm": 7.153159141540527,
      "learning_rate": 9.635977634011746e-08,
      "logits/chosen": -0.3231102228164673,
      "logits/rejected": -0.2612823247909546,
      "logps/chosen": -216.70489501953125,
      "logps/rejected": -303.5150146484375,
      "loss": 0.0935,
      "mlc_losses": 0.06114454194903374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13513702154159546,
      "rewards/margins": 5.231598377227783,
      "rewards/rejected": -5.366735935211182,
      "step": 205
    },
    {
      "dpo_losses": 0.02312319353222847,
      "epoch": 2.3146067415730336,
      "grad_norm": 4.641868591308594,
      "learning_rate": 9.349362729187375e-08,
      "logits/chosen": -0.40953731536865234,
      "logits/rejected": -0.34620165824890137,
      "logps/chosen": -191.80572509765625,
      "logps/rejected": -269.9676818847656,
      "loss": 0.0231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4423351585865021,
      "rewards/margins": 5.983387470245361,
      "rewards/rejected": -6.425723075866699,
      "step": 206
    },
    {
      "dpo_losses": 0.07203122973442078,
      "epoch": 2.3258426966292136,
      "grad_norm": 21.6843318939209,
      "learning_rate": 9.066286204749601e-08,
      "logits/chosen": -0.43475210666656494,
      "logits/rejected": -0.29222047328948975,
      "logps/chosen": -308.0385437011719,
      "logps/rejected": -327.2193908691406,
      "loss": 0.1331,
      "mlc_losses": 0.06109773740172386,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.33898329734802246,
      "rewards/margins": 6.092944145202637,
      "rewards/rejected": -6.431927680969238,
      "step": 207
    },
    {
      "dpo_losses": 0.025026444345712662,
      "epoch": 2.337078651685393,
      "grad_norm": 4.325893878936768,
      "learning_rate": 8.786796564403575e-08,
      "logits/chosen": -0.4668623208999634,
      "logits/rejected": -0.25772422552108765,
      "logps/chosen": -217.90147399902344,
      "logps/rejected": -335.8299255371094,
      "loss": 0.025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2546418607234955,
      "rewards/margins": 6.085152626037598,
      "rewards/rejected": -6.339794158935547,
      "step": 208
    },
    {
      "dpo_losses": 0.023484963923692703,
      "epoch": 2.348314606741573,
      "grad_norm": 4.858598232269287,
      "learning_rate": 8.510941697260371e-08,
      "logits/chosen": -0.35015687346458435,
      "logits/rejected": -0.3599117696285248,
      "logps/chosen": -265.98504638671875,
      "logps/rejected": -323.61932373046875,
      "loss": 0.0843,
      "mlc_losses": 0.060817617923021317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40018099546432495,
      "rewards/margins": 5.717558860778809,
      "rewards/rejected": -6.117740154266357,
      "step": 209
    },
    {
      "dpo_losses": 0.029154185205698013,
      "epoch": 2.359550561797753,
      "grad_norm": 5.997616767883301,
      "learning_rate": 8.238768869631377e-08,
      "logits/chosen": -0.46983498334884644,
      "logits/rejected": -0.3556313216686249,
      "logps/chosen": -222.56298828125,
      "logps/rejected": -330.22613525390625,
      "loss": 0.0292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7831200361251831,
      "rewards/margins": 5.815491199493408,
      "rewards/rejected": -6.598611831665039,
      "step": 210
    },
    {
      "dpo_losses": 0.024889089167118073,
      "epoch": 2.370786516853933,
      "grad_norm": 4.985291004180908,
      "learning_rate": 7.97032471692944e-08,
      "logits/chosen": -0.35216307640075684,
      "logits/rejected": -0.34606871008872986,
      "logps/chosen": -285.41473388671875,
      "logps/rejected": -359.4784240722656,
      "loss": 0.0855,
      "mlc_losses": 0.060654416680336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8768834471702576,
      "rewards/margins": 5.978155136108398,
      "rewards/rejected": -6.855038642883301,
      "step": 211
    },
    {
      "dpo_losses": 0.017940472811460495,
      "epoch": 2.3820224719101124,
      "grad_norm": 3.3157570362091064,
      "learning_rate": 7.705655235678169e-08,
      "logits/chosen": -0.39679014682769775,
      "logits/rejected": -0.2861778438091278,
      "logps/chosen": -234.3523406982422,
      "logps/rejected": -352.6709289550781,
      "loss": 0.0179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5360594987869263,
      "rewards/margins": 6.303682327270508,
      "rewards/rejected": -6.8397417068481445,
      "step": 212
    },
    {
      "dpo_losses": 0.04726354405283928,
      "epoch": 2.393258426966292,
      "grad_norm": 12.318689346313477,
      "learning_rate": 7.444805775630681e-08,
      "logits/chosen": -0.4128164052963257,
      "logits/rejected": -0.38507550954818726,
      "logps/chosen": -243.1868438720703,
      "logps/rejected": -370.5032043457031,
      "loss": 0.1083,
      "mlc_losses": 0.06105491891503334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8721380233764648,
      "rewards/margins": 5.411258697509766,
      "rewards/rejected": -6.283397197723389,
      "step": 213
    },
    {
      "dpo_losses": 0.02579297125339508,
      "epoch": 2.404494382022472,
      "grad_norm": 6.672260284423828,
      "learning_rate": 7.187821031999071e-08,
      "logits/chosen": -0.4206334948539734,
      "logits/rejected": -0.21309344470500946,
      "logps/chosen": -246.14370727539062,
      "logps/rejected": -427.66973876953125,
      "loss": 0.0258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5126113295555115,
      "rewards/margins": 6.5754547119140625,
      "rewards/rejected": -7.0880656242370605,
      "step": 214
    },
    {
      "dpo_losses": 0.049350738525390625,
      "epoch": 2.4157303370786516,
      "grad_norm": 9.151542663574219,
      "learning_rate": 6.934745037796215e-08,
      "logits/chosen": -0.5113201141357422,
      "logits/rejected": -0.432513028383255,
      "logps/chosen": -213.08030700683594,
      "logps/rejected": -246.11474609375,
      "loss": 0.1113,
      "mlc_losses": 0.061911359429359436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7423152923583984,
      "rewards/margins": 5.327437400817871,
      "rewards/rejected": -6.069752216339111,
      "step": 215
    },
    {
      "dpo_losses": 0.02488410845398903,
      "epoch": 2.4269662921348316,
      "grad_norm": 5.612128734588623,
      "learning_rate": 6.685621156290873e-08,
      "logits/chosen": -0.3838833272457123,
      "logits/rejected": -0.2145812064409256,
      "logps/chosen": -222.09408569335938,
      "logps/rejected": -354.3038330078125,
      "loss": 0.0249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17079482972621918,
      "rewards/margins": 7.1066670417785645,
      "rewards/rejected": -7.277461528778076,
      "step": 216
    },
    {
      "dpo_losses": 0.031018242239952087,
      "epoch": 2.438202247191011,
      "grad_norm": 10.157346725463867,
      "learning_rate": 6.440492073577657e-08,
      "logits/chosen": -0.4499305188655853,
      "logits/rejected": -0.3485034108161926,
      "logps/chosen": -221.2950439453125,
      "logps/rejected": -406.0693359375,
      "loss": 0.0918,
      "mlc_losses": 0.060818351805210114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3240964710712433,
      "rewards/margins": 6.559330940246582,
      "rewards/rejected": -6.88342809677124,
      "step": 217
    },
    {
      "dpo_losses": 0.05845363438129425,
      "epoch": 2.449438202247191,
      "grad_norm": 9.888949394226074,
      "learning_rate": 6.199399791262948e-08,
      "logits/chosen": -0.43520307540893555,
      "logits/rejected": -0.2905881404876709,
      "logps/chosen": -198.30722045898438,
      "logps/rejected": -384.28643798828125,
      "loss": 0.0585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7044684290885925,
      "rewards/margins": 6.203132629394531,
      "rewards/rejected": -6.9076008796691895,
      "step": 218
    },
    {
      "dpo_losses": 0.02118687517940998,
      "epoch": 2.460674157303371,
      "grad_norm": 5.831445693969727,
      "learning_rate": 5.962385619268184e-08,
      "logits/chosen": -0.46266189217567444,
      "logits/rejected": -0.384269654750824,
      "logps/chosen": -229.66802978515625,
      "logps/rejected": -399.75830078125,
      "loss": 0.0821,
      "mlc_losses": 0.06094294413924217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.016039110720157623,
      "rewards/margins": 7.02631139755249,
      "rewards/rejected": -7.01027250289917,
      "step": 219
    },
    {
      "dpo_losses": 0.02679526060819626,
      "epoch": 2.4719101123595504,
      "grad_norm": 5.804264068603516,
      "learning_rate": 5.7294901687515794e-08,
      "logits/chosen": -0.43379878997802734,
      "logits/rejected": -0.2821975648403168,
      "logps/chosen": -260.45391845703125,
      "logps/rejected": -359.5623779296875,
      "loss": 0.0268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8773566484451294,
      "rewards/margins": 5.865565776824951,
      "rewards/rejected": -6.742922782897949,
      "step": 220
    },
    {
      "dpo_losses": 0.021573485806584358,
      "epoch": 2.4831460674157304,
      "grad_norm": 6.950760364532471,
      "learning_rate": 5.5007533451496326e-08,
      "logits/chosen": -0.3734597861766815,
      "logits/rejected": -0.31296810507774353,
      "logps/chosen": -209.19300842285156,
      "logps/rejected": -343.7889709472656,
      "loss": 0.0826,
      "mlc_losses": 0.06105174124240875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0874551385641098,
      "rewards/margins": 6.584853649139404,
      "rewards/rejected": -6.672308921813965,
      "step": 221
    },
    {
      "dpo_losses": 0.0800265446305275,
      "epoch": 2.49438202247191,
      "grad_norm": 12.157212257385254,
      "learning_rate": 5.2762143413395286e-08,
      "logits/chosen": -0.3760017454624176,
      "logits/rejected": -0.3083954155445099,
      "logps/chosen": -237.12100219726562,
      "logps/rejected": -289.55255126953125,
      "loss": 0.08,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23054160177707672,
      "rewards/margins": 4.491111755371094,
      "rewards/rejected": -4.721652984619141,
      "step": 222
    },
    {
      "dpo_losses": 0.018836624920368195,
      "epoch": 2.50561797752809,
      "grad_norm": 4.643320083618164,
      "learning_rate": 5.0559116309236397e-08,
      "logits/chosen": -0.32068613171577454,
      "logits/rejected": -0.2851940989494324,
      "logps/chosen": -248.5154571533203,
      "logps/rejected": -377.65350341796875,
      "loss": 0.0794,
      "mlc_losses": 0.060529015958309174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4066470861434937,
      "rewards/margins": 6.524001121520996,
      "rewards/rejected": -7.930647373199463,
      "step": 223
    },
    {
      "dpo_losses": 0.02837708592414856,
      "epoch": 2.5168539325842696,
      "grad_norm": 6.766981601715088,
      "learning_rate": 4.839882961637282e-08,
      "logits/chosen": -0.6005704402923584,
      "logits/rejected": -0.39986154437065125,
      "logps/chosen": -170.3070831298828,
      "logps/rejected": -290.5411682128906,
      "loss": 0.0284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10943970084190369,
      "rewards/margins": 6.37166690826416,
      "rewards/rejected": -6.481106281280518,
      "step": 224
    },
    {
      "dpo_losses": 0.055564481765031815,
      "epoch": 2.5280898876404496,
      "grad_norm": 12.513469696044922,
      "learning_rate": 4.628165348880804e-08,
      "logits/chosen": -0.36567360162734985,
      "logits/rejected": -0.3523275554180145,
      "logps/chosen": -274.09918212890625,
      "logps/rejected": -306.1929931640625,
      "loss": 0.1163,
      "mlc_losses": 0.06069452315568924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2191109657287598,
      "rewards/margins": 5.09650993347168,
      "rewards/rejected": -6.315619945526123,
      "step": 225
    },
    {
      "dpo_losses": 0.06124646216630936,
      "epoch": 2.539325842696629,
      "grad_norm": 12.676579475402832,
      "learning_rate": 4.4207950693772346e-08,
      "logits/chosen": -0.4110356867313385,
      "logits/rejected": -0.4072601795196533,
      "logps/chosen": -244.24569702148438,
      "logps/rejected": -293.7549133300781,
      "loss": 0.0612,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1575911045074463,
      "rewards/margins": 5.109364986419678,
      "rewards/rejected": -6.266956329345703,
      "step": 226
    },
    {
      "dpo_losses": 0.023177120834589005,
      "epoch": 2.550561797752809,
      "grad_norm": 5.388479709625244,
      "learning_rate": 4.217807654956419e-08,
      "logits/chosen": -0.385859876871109,
      "logits/rejected": -0.3254287838935852,
      "logps/chosen": -236.93997192382812,
      "logps/rejected": -348.5915222167969,
      "loss": 0.0844,
      "mlc_losses": 0.061201054602861404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6167060732841492,
      "rewards/margins": 6.426006317138672,
      "rewards/rejected": -7.042713165283203,
      "step": 227
    },
    {
      "dpo_losses": 0.024092797189950943,
      "epoch": 2.561797752808989,
      "grad_norm": 5.206618785858154,
      "learning_rate": 4.0192378864668387e-08,
      "logits/chosen": -0.40883955359458923,
      "logits/rejected": -0.2803736925125122,
      "logps/chosen": -243.30514526367188,
      "logps/rejected": -369.591552734375,
      "loss": 0.0241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9211846590042114,
      "rewards/margins": 6.478778839111328,
      "rewards/rejected": -7.39996337890625,
      "step": 228
    },
    {
      "dpo_losses": 0.02002987265586853,
      "epoch": 2.5730337078651684,
      "grad_norm": 4.653064250946045,
      "learning_rate": 3.825119787816085e-08,
      "logits/chosen": -0.4434170126914978,
      "logits/rejected": -0.2801324129104614,
      "logps/chosen": -228.64794921875,
      "logps/rejected": -395.1051940917969,
      "loss": 0.0809,
      "mlc_losses": 0.06085682660341263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3521062433719635,
      "rewards/margins": 5.745504379272461,
      "rewards/rejected": -6.0976104736328125,
      "step": 229
    },
    {
      "dpo_losses": 0.018296949565410614,
      "epoch": 2.5842696629213484,
      "grad_norm": 3.8165385723114014,
      "learning_rate": 3.635486620141042e-08,
      "logits/chosen": -0.3734816312789917,
      "logits/rejected": -0.299778550863266,
      "logps/chosen": -231.94395446777344,
      "logps/rejected": -334.51806640625,
      "loss": 0.0183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5941556692123413,
      "rewards/margins": 6.402434825897217,
      "rewards/rejected": -6.996590614318848,
      "step": 230
    },
    {
      "dpo_losses": 0.02990615926682949,
      "epoch": 2.595505617977528,
      "grad_norm": 6.177568435668945,
      "learning_rate": 3.450370876108747e-08,
      "logits/chosen": -0.35885822772979736,
      "logits/rejected": -0.26852598786354065,
      "logps/chosen": -248.57923889160156,
      "logps/rejected": -369.48699951171875,
      "loss": 0.09,
      "mlc_losses": 0.06005208194255829,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6889830827713013,
      "rewards/margins": 6.875090599060059,
      "rewards/rejected": -7.564074516296387,
      "step": 231
    },
    {
      "dpo_losses": 0.04166027531027794,
      "epoch": 2.606741573033708,
      "grad_norm": 9.053930282592773,
      "learning_rate": 3.269804274348966e-08,
      "logits/chosen": -0.4564557671546936,
      "logits/rejected": -0.2836887836456299,
      "logps/chosen": -248.748046875,
      "logps/rejected": -406.2759704589844,
      "loss": 0.0417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5762938857078552,
      "rewards/margins": 6.318434715270996,
      "rewards/rejected": -6.894728660583496,
      "step": 232
    },
    {
      "dpo_losses": 0.035194337368011475,
      "epoch": 2.6179775280898876,
      "grad_norm": 6.466554164886475,
      "learning_rate": 3.0938177540193524e-08,
      "logits/chosen": -0.5043267011642456,
      "logits/rejected": -0.40911638736724854,
      "logps/chosen": -217.04397583007812,
      "logps/rejected": -299.22442626953125,
      "loss": 0.0962,
      "mlc_losses": 0.06105510890483856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7797572612762451,
      "rewards/margins": 6.057008266448975,
      "rewards/rejected": -6.836765289306641,
      "step": 233
    },
    {
      "dpo_losses": 0.025853602215647697,
      "epoch": 2.629213483146067,
      "grad_norm": 5.916706085205078,
      "learning_rate": 2.922441469504188e-08,
      "logits/chosen": -0.4666305482387543,
      "logits/rejected": -0.4758266806602478,
      "logps/chosen": -234.84422302246094,
      "logps/rejected": -327.15179443359375,
      "loss": 0.0259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2228783667087555,
      "rewards/margins": 7.1022491455078125,
      "rewards/rejected": -7.325128078460693,
      "step": 234
    },
    {
      "dpo_losses": 0.05138521268963814,
      "epoch": 2.640449438202247,
      "grad_norm": 15.305645942687988,
      "learning_rate": 2.755704785247559e-08,
      "logits/chosen": -0.39308130741119385,
      "logits/rejected": -0.3040301203727722,
      "logps/chosen": -243.777587890625,
      "logps/rejected": -383.0447692871094,
      "loss": 0.1124,
      "mlc_losses": 0.06101509556174278,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9010135531425476,
      "rewards/margins": 6.908593654632568,
      "rewards/rejected": -7.80960750579834,
      "step": 235
    },
    {
      "dpo_losses": 0.045278966426849365,
      "epoch": 2.6516853932584272,
      "grad_norm": 7.605687618255615,
      "learning_rate": 2.5936362707219705e-08,
      "logits/chosen": -0.28915584087371826,
      "logits/rejected": -0.27242010831832886,
      "logps/chosen": -259.6003112792969,
      "logps/rejected": -290.6086120605469,
      "loss": 0.0453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8052835464477539,
      "rewards/margins": 5.695082664489746,
      "rewards/rejected": -6.5003662109375,
      "step": 236
    },
    {
      "dpo_losses": 0.05228010192513466,
      "epoch": 2.662921348314607,
      "grad_norm": 19.282941818237305,
      "learning_rate": 2.436263695533054e-08,
      "logits/chosen": -0.38602009415626526,
      "logits/rejected": -0.22363457083702087,
      "logps/chosen": -212.0613250732422,
      "logps/rejected": -356.21697998046875,
      "loss": 0.1132,
      "mlc_losses": 0.06096978485584259,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5161631107330322,
      "rewards/margins": 6.089169502258301,
      "rewards/rejected": -6.605332851409912,
      "step": 237
    },
    {
      "dpo_losses": 0.02687586471438408,
      "epoch": 2.6741573033707864,
      "grad_norm": 8.412290573120117,
      "learning_rate": 2.283614024661398e-08,
      "logits/chosen": -0.4136210083961487,
      "logits/rejected": -0.29784902930259705,
      "logps/chosen": -234.2107696533203,
      "logps/rejected": -344.21429443359375,
      "loss": 0.0269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2716310024261475,
      "rewards/margins": 6.1866841316223145,
      "rewards/rejected": -7.458315372467041,
      "step": 238
    },
    {
      "dpo_losses": 0.03061244636774063,
      "epoch": 2.6853932584269664,
      "grad_norm": 6.109188079833984,
      "learning_rate": 2.1357134138422727e-08,
      "logits/chosen": -0.37885865569114685,
      "logits/rejected": -0.26491016149520874,
      "logps/chosen": -234.90599060058594,
      "logps/rejected": -336.0806884765625,
      "loss": 0.0907,
      "mlc_losses": 0.060098301619291306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19336706399917603,
      "rewards/margins": 5.878410816192627,
      "rewards/rejected": -6.071778297424316,
      "step": 239
    },
    {
      "dpo_losses": 0.01742285117506981,
      "epoch": 2.696629213483146,
      "grad_norm": 4.2932562828063965,
      "learning_rate": 1.9925872050839508e-08,
      "logits/chosen": -0.3716609477996826,
      "logits/rejected": -0.3510451912879944,
      "logps/chosen": -299.50238037109375,
      "logps/rejected": -361.8536682128906,
      "loss": 0.0174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0121623277664185,
      "rewards/margins": 6.308050155639648,
      "rewards/rejected": -7.320213317871094,
      "step": 240
    },
    {
      "dpo_losses": 0.03240359202027321,
      "epoch": 2.7078651685393256,
      "grad_norm": 12.02367115020752,
      "learning_rate": 1.8542599223254784e-08,
      "logits/chosen": -0.3826124966144562,
      "logits/rejected": -0.34453460574150085,
      "logps/chosen": -228.5043487548828,
      "logps/rejected": -353.08251953125,
      "loss": 0.0922,
      "mlc_losses": 0.05978183075785637,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6027748584747314,
      "rewards/margins": 7.022447109222412,
      "rewards/rejected": -7.6252217292785645,
      "step": 241
    },
    {
      "dpo_losses": 0.03307972475886345,
      "epoch": 2.7191011235955056,
      "grad_norm": 6.809433937072754,
      "learning_rate": 1.720755267234647e-08,
      "logits/chosen": -0.3754655122756958,
      "logits/rejected": -0.3748812675476074,
      "logps/chosen": -279.20599365234375,
      "logps/rejected": -323.4715576171875,
      "loss": 0.0331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5544207096099854,
      "rewards/margins": 5.444005012512207,
      "rewards/rejected": -5.998425483703613,
      "step": 242
    },
    {
      "dpo_losses": 0.03047693707048893,
      "epoch": 2.7303370786516856,
      "grad_norm": 5.505368709564209,
      "learning_rate": 1.5920961151468326e-08,
      "logits/chosen": -0.3878582715988159,
      "logits/rejected": -0.2500898838043213,
      "logps/chosen": -252.25814819335938,
      "logps/rejected": -327.5415954589844,
      "loss": 0.0907,
      "mlc_losses": 0.060212209820747375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0513584613800049,
      "rewards/margins": 5.9650068283081055,
      "rewards/rejected": -7.016365051269531,
      "step": 243
    },
    {
      "dpo_losses": 0.019725942984223366,
      "epoch": 2.741573033707865,
      "grad_norm": 4.207443714141846,
      "learning_rate": 1.468304511145394e-08,
      "logits/chosen": -0.37563517689704895,
      "logits/rejected": -0.3270558714866638,
      "logps/chosen": -271.78533935546875,
      "logps/rejected": -365.51318359375,
      "loss": 0.0197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1691395044326782,
      "rewards/margins": 6.4981184005737305,
      "rewards/rejected": -7.667257308959961,
      "step": 244
    },
    {
      "dpo_losses": 0.03742143511772156,
      "epoch": 2.752808988764045,
      "grad_norm": 6.745202541351318,
      "learning_rate": 1.349401666284401e-08,
      "logits/chosen": -0.3283732533454895,
      "logits/rejected": -0.2039058804512024,
      "logps/chosen": -219.10134887695312,
      "logps/rejected": -377.8810119628906,
      "loss": 0.0983,
      "mlc_losses": 0.06084643676877022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1528424173593521,
      "rewards/margins": 6.622564315795898,
      "rewards/rejected": -6.775406837463379,
      "step": 245
    },
    {
      "dpo_losses": 0.026618365198373795,
      "epoch": 2.764044943820225,
      "grad_norm": 5.809737682342529,
      "learning_rate": 1.2354079539542084e-08,
      "logits/chosen": -0.38344067335128784,
      "logits/rejected": -0.26703229546546936,
      "logps/chosen": -247.97935485839844,
      "logps/rejected": -316.73162841796875,
      "loss": 0.0266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.34024983644485474,
      "rewards/margins": 6.45767879486084,
      "rewards/rejected": -6.797928333282471,
      "step": 246
    },
    {
      "dpo_losses": 0.030915413051843643,
      "epoch": 2.7752808988764044,
      "grad_norm": 5.3809003829956055,
      "learning_rate": 1.1263429063905849e-08,
      "logits/chosen": -0.21524623036384583,
      "logits/rejected": -0.33242306113243103,
      "logps/chosen": -255.32598876953125,
      "logps/rejected": -324.5967712402344,
      "loss": 0.0912,
      "mlc_losses": 0.06032935902476311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5099984407424927,
      "rewards/margins": 5.915024757385254,
      "rewards/rejected": -6.425023555755615,
      "step": 247
    },
    {
      "dpo_losses": 0.023845313116908073,
      "epoch": 2.7865168539325844,
      "grad_norm": 5.46895694732666,
      "learning_rate": 1.0222252113279539e-08,
      "logits/chosen": -0.44617563486099243,
      "logits/rejected": -0.3259655237197876,
      "logps/chosen": -308.4246520996094,
      "logps/rejected": -388.9064025878906,
      "loss": 0.0238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.672629952430725,
      "rewards/margins": 5.878673076629639,
      "rewards/rejected": -7.551302909851074,
      "step": 248
    },
    {
      "dpo_losses": 0.030282393097877502,
      "epoch": 2.797752808988764,
      "grad_norm": 6.575648784637451,
      "learning_rate": 9.230727087973711e-09,
      "logits/chosen": -0.3502185344696045,
      "logits/rejected": -0.19682945311069489,
      "logps/chosen": -293.44586181640625,
      "logps/rejected": -308.74908447265625,
      "loss": 0.0903,
      "mlc_losses": 0.05998140573501587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3410561084747314,
      "rewards/margins": 5.547425270080566,
      "rewards/rejected": -6.888481140136719,
      "step": 249
    },
    {
      "dpo_losses": 0.028972037136554718,
      "epoch": 2.808988764044944,
      "grad_norm": 6.113420486450195,
      "learning_rate": 8.289023880697032e-09,
      "logits/chosen": -0.43773382902145386,
      "logits/rejected": -0.40647202730178833,
      "logps/chosen": -261.4711608886719,
      "logps/rejected": -295.1462097167969,
      "loss": 0.029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9020113945007324,
      "rewards/margins": 5.585984706878662,
      "rewards/rejected": -6.487995624542236,
      "step": 250
    },
    {
      "dpo_losses": 0.018160240724682808,
      "epoch": 2.8202247191011236,
      "grad_norm": 4.361262798309326,
      "learning_rate": 7.3973038474462015e-09,
      "logits/chosen": -0.47576549649238586,
      "logits/rejected": -0.26581382751464844,
      "logps/chosen": -213.90231323242188,
      "logps/rejected": -314.9590759277344,
      "loss": 0.0787,
      "mlc_losses": 0.06055981665849686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21365657448768616,
      "rewards/margins": 6.5755839347839355,
      "rewards/rejected": -6.361927509307861,
      "step": 251
    },
    {
      "dpo_losses": 0.027872378006577492,
      "epoch": 2.831460674157303,
      "grad_norm": 5.747492790222168,
      "learning_rate": 6.555719779858293e-09,
      "logits/chosen": -0.4283604621887207,
      "logits/rejected": -0.3142379820346832,
      "logps/chosen": -232.1664276123047,
      "logps/rejected": -341.2335510253906,
      "loss": 0.0279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.234907865524292,
      "rewards/margins": 6.902548789978027,
      "rewards/rejected": -6.667640686035156,
      "step": 252
    },
    {
      "dpo_losses": 0.024066036567091942,
      "epoch": 2.842696629213483,
      "grad_norm": 4.594196796417236,
      "learning_rate": 5.76441587903087e-09,
      "logits/chosen": -0.507655143737793,
      "logits/rejected": -0.37163567543029785,
      "logps/chosen": -196.41575622558594,
      "logps/rejected": -348.9814453125,
      "loss": 0.0838,
      "mlc_losses": 0.059689901769161224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0017116069793701172,
      "rewards/margins": 7.251550674438477,
      "rewards/rejected": -7.253262519836426,
      "step": 253
    },
    {
      "dpo_losses": 0.04229250177741051,
      "epoch": 2.853932584269663,
      "grad_norm": 11.362602233886719,
      "learning_rate": 5.023527730813648e-09,
      "logits/chosen": -0.39333537220954895,
      "logits/rejected": -0.2787695527076721,
      "logps/chosen": -249.4808349609375,
      "logps/rejected": -366.49371337890625,
      "loss": 0.0423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8788717985153198,
      "rewards/margins": 6.5388994216918945,
      "rewards/rejected": -7.417771339416504,
      "step": 254
    },
    {
      "dpo_losses": 0.045197781175374985,
      "epoch": 2.865168539325843,
      "grad_norm": 8.96122932434082,
      "learning_rate": 4.333182282576675e-09,
      "logits/chosen": -0.473237544298172,
      "logits/rejected": -0.20807625353336334,
      "logps/chosen": -191.90115356445312,
      "logps/rejected": -352.6612854003906,
      "loss": 0.1052,
      "mlc_losses": 0.06004050374031067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9112832546234131,
      "rewards/margins": 5.7438249588012695,
      "rewards/rejected": -6.6551079750061035,
      "step": 255
    },
    {
      "dpo_losses": 0.05975145101547241,
      "epoch": 2.8764044943820224,
      "grad_norm": 13.739983558654785,
      "learning_rate": 3.693497821458702e-09,
      "logits/chosen": -0.4158109128475189,
      "logits/rejected": -0.24543578922748566,
      "logps/chosen": -231.5672607421875,
      "logps/rejected": -354.674560546875,
      "loss": 0.0598,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.32128942012786865,
      "rewards/margins": 6.633323669433594,
      "rewards/rejected": -6.954612731933594,
      "step": 256
    },
    {
      "dpo_losses": 0.043626464903354645,
      "epoch": 2.8876404494382024,
      "grad_norm": 15.917800903320312,
      "learning_rate": 3.1045839540989273e-09,
      "logits/chosen": -0.3700627088546753,
      "logits/rejected": -0.2980884611606598,
      "logps/chosen": -273.1658935546875,
      "logps/rejected": -325.5955505371094,
      "loss": 0.1037,
      "mlc_losses": 0.06005360186100006,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2100778818130493,
      "rewards/margins": 5.5026068687438965,
      "rewards/rejected": -6.712684631347656,
      "step": 257
    },
    {
      "dpo_losses": 0.01589292287826538,
      "epoch": 2.898876404494382,
      "grad_norm": 4.385040283203125,
      "learning_rate": 2.5665415878568854e-09,
      "logits/chosen": -0.36164867877960205,
      "logits/rejected": -0.30448365211486816,
      "logps/chosen": -224.77285766601562,
      "logps/rejected": -405.9618835449219,
      "loss": 0.0159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5998871326446533,
      "rewards/margins": 8.029801368713379,
      "rewards/rejected": -8.629688262939453,
      "step": 258
    },
    {
      "dpo_losses": 0.014980494976043701,
      "epoch": 2.9101123595505616,
      "grad_norm": 6.339247226715088,
      "learning_rate": 2.079462913522112e-09,
      "logits/chosen": -0.39144831895828247,
      "logits/rejected": -0.28923726081848145,
      "logps/chosen": -238.64752197265625,
      "logps/rejected": -417.0428466796875,
      "loss": 0.0747,
      "mlc_losses": 0.05975378677248955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1901901364326477,
      "rewards/margins": 7.471563339233398,
      "rewards/rejected": -7.661753177642822,
      "step": 259
    },
    {
      "dpo_losses": 0.05812764912843704,
      "epoch": 2.9213483146067416,
      "grad_norm": 7.9527268409729,
      "learning_rate": 1.643431389518013e-09,
      "logits/chosen": -0.4760197699069977,
      "logits/rejected": -0.36595746874809265,
      "logps/chosen": -196.29254150390625,
      "logps/rejected": -277.0702209472656,
      "loss": 0.0581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5755787491798401,
      "rewards/margins": 5.7625885009765625,
      "rewards/rejected": -6.338167190551758,
      "step": 260
    },
    {
      "dpo_losses": 0.03084026835858822,
      "epoch": 2.932584269662921,
      "grad_norm": 6.611967086791992,
      "learning_rate": 1.2585217276015025e-09,
      "logits/chosen": -0.40135857462882996,
      "logits/rejected": -0.3166777491569519,
      "logps/chosen": -209.81192016601562,
      "logps/rejected": -351.0107421875,
      "loss": 0.0906,
      "mlc_losses": 0.0597648024559021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10303108394145966,
      "rewards/margins": 6.831051826477051,
      "rewards/rejected": -6.728020668029785,
      "step": 261
    },
    {
      "dpo_losses": 0.016360506415367126,
      "epoch": 2.943820224719101,
      "grad_norm": 4.061613082885742,
      "learning_rate": 9.247998800616108e-10,
      "logits/chosen": -0.4979334771633148,
      "logits/rejected": -0.3732694089412689,
      "logps/chosen": -203.42120361328125,
      "logps/rejected": -333.8220520019531,
      "loss": 0.0164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6134711503982544,
      "rewards/margins": 6.046405792236328,
      "rewards/rejected": -6.659876823425293,
      "step": 262
    },
    {
      "dpo_losses": 0.023932956159114838,
      "epoch": 2.955056179775281,
      "grad_norm": 5.956596851348877,
      "learning_rate": 6.423230284189562e-10,
      "logits/chosen": -0.46829062700271606,
      "logits/rejected": -0.40760549902915955,
      "logps/chosen": -176.79029846191406,
      "logps/rejected": -358.0665588378906,
      "loss": 0.0849,
      "mlc_losses": 0.060922279953956604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24552202224731445,
      "rewards/margins": 7.301923751831055,
      "rewards/rejected": -7.547446250915527,
      "step": 263
    },
    {
      "dpo_losses": 0.06231420859694481,
      "epoch": 2.966292134831461,
      "grad_norm": 15.509610176086426,
      "learning_rate": 4.1113957362785e-10,
      "logits/chosen": -0.300924152135849,
      "logits/rejected": -0.2706843614578247,
      "logps/chosen": -244.69204711914062,
      "logps/rejected": -322.1877136230469,
      "loss": 0.0623,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7611143589019775,
      "rewards/margins": 5.790822505950928,
      "rewards/rejected": -6.551936626434326,
      "step": 264
    },
    {
      "dpo_losses": 0.014090299606323242,
      "epoch": 2.9775280898876404,
      "grad_norm": 3.050595998764038,
      "learning_rate": 2.312891277831297e-10,
      "logits/chosen": -0.4637715518474579,
      "logits/rejected": -0.36232277750968933,
      "logps/chosen": -210.6854705810547,
      "logps/rejected": -332.3436279296875,
      "loss": 0.0743,
      "mlc_losses": 0.06025440990924835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37845590710639954,
      "rewards/margins": 6.13828706741333,
      "rewards/rejected": -6.516743183135986,
      "step": 265
    },
    {
      "dpo_losses": 0.03357948735356331,
      "epoch": 2.98876404494382,
      "grad_norm": 7.727936744689941,
      "learning_rate": 1.0280250733282203e-10,
      "logits/chosen": -0.4643649458885193,
      "logits/rejected": -0.3479286730289459,
      "logps/chosen": -246.70823669433594,
      "logps/rejected": -341.30859375,
      "loss": 0.0336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7084284424781799,
      "rewards/margins": 6.0391106605529785,
      "rewards/rejected": -6.747539520263672,
      "step": 266
    },
    {
      "dpo_losses": 0.02088983729481697,
      "epoch": 3.0,
      "grad_norm": 5.476436614990234,
      "learning_rate": 2.570172779789992e-11,
      "logits/chosen": -0.46314606070518494,
      "logits/rejected": -0.22247427701950073,
      "logps/chosen": -215.33938598632812,
      "logps/rejected": -344.0314025878906,
      "loss": 0.0814,
      "mlc_losses": 0.06052825599908829,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2580088675022125,
      "rewards/margins": 7.045166492462158,
      "rewards/rejected": -6.78715705871582,
      "step": 267
    },
    {
      "epoch": 3.0,
      "step": 267,
      "total_flos": 8.768589976326636e+17,
      "train_loss": 0.23950207320631442,
      "train_runtime": 868.6403,
      "train_samples_per_second": 9.836,
      "train_steps_per_second": 0.307
    }
  ],
  "logging_steps": 1,
  "max_steps": 267,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 8.768589976326636e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}