{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8599068434252956,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004777260241251642,
      "grad_norm": 47.0,
      "learning_rate": 2.222222222222222e-08,
      "logits/chosen": -1.8765161037445068,
      "logits/rejected": -1.8362525701522827,
      "logps/chosen": -0.3087114691734314,
      "logps/rejected": -0.3054303526878357,
      "loss": 1.0474976301193237,
      "loss/core": 1.0474976301193237,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4050393104553223,
      "rewards/margins": 1.9889495372772217,
      "rewards/rejected": 1.416089653968811,
      "step": 5
    },
    {
      "epoch": 0.009554520482503284,
      "grad_norm": 15.125,
      "learning_rate": 5e-08,
      "logits/chosen": -1.8804371356964111,
      "logits/rejected": -1.8574333190917969,
      "logps/chosen": -0.29082149267196655,
      "logps/rejected": -0.27977004647254944,
      "loss": 1.056221842765808,
      "loss/core": 1.056221842765808,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.9182629585266113,
      "rewards/margins": 1.5795632600784302,
      "rewards/rejected": 1.3386995792388916,
      "step": 10
    },
    {
      "epoch": 0.014331780723754926,
      "grad_norm": 7.03125,
      "learning_rate": 7.777777777777778e-08,
      "logits/chosen": -1.6232883930206299,
      "logits/rejected": -1.6565492153167725,
      "logps/chosen": -0.2895023226737976,
      "logps/rejected": -0.28725099563598633,
      "loss": 1.042079210281372,
      "loss/core": 1.042079210281372,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9274177551269531,
      "rewards/margins": 1.0600453615188599,
      "rewards/rejected": 0.8673725128173828,
      "step": 15
    },
    {
      "epoch": 0.01910904096500657,
      "grad_norm": 7.65625,
      "learning_rate": 1.0555555555555555e-07,
      "logits/chosen": -1.892533302307129,
      "logits/rejected": -1.9024165868759155,
      "logps/chosen": -0.28663748502731323,
      "logps/rejected": -0.2839517891407013,
      "loss": 1.0445116758346558,
      "loss/core": 1.0445116758346558,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.6367988586425781,
      "rewards/margins": 1.0847914218902588,
      "rewards/rejected": 0.5520075559616089,
      "step": 20
    },
    {
      "epoch": 0.02388630120625821,
      "grad_norm": 17.25,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -1.929903268814087,
      "logits/rejected": -1.9212831258773804,
      "logps/chosen": -0.27033165097236633,
      "logps/rejected": -0.2790217995643616,
      "loss": 1.028629183769226,
      "loss/core": 1.028629183769226,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8104407787322998,
      "rewards/margins": 0.6213824152946472,
      "rewards/rejected": 1.1890584230422974,
      "step": 25
    },
    {
      "epoch": 0.028663561447509853,
      "grad_norm": 46.5,
      "learning_rate": 1.611111111111111e-07,
      "logits/chosen": -1.8559471368789673,
      "logits/rejected": -1.8298406600952148,
      "logps/chosen": -0.31229084730148315,
      "logps/rejected": -0.27183204889297485,
      "loss": 1.1012113094329834,
      "loss/core": 1.1012113094329834,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.737351179122925,
      "rewards/margins": 0.9300357103347778,
      "rewards/rejected": 1.8073152303695679,
      "step": 30
    },
    {
      "epoch": 0.033440821688761495,
      "grad_norm": 9.3125,
      "learning_rate": 1.8888888888888888e-07,
      "logits/chosen": -1.848472237586975,
      "logits/rejected": -1.9320148229599,
      "logps/chosen": -0.2827574610710144,
      "logps/rejected": -0.26435643434524536,
      "loss": 1.06340754032135,
      "loss/core": 1.06340754032135,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8763984441757202,
      "rewards/margins": 1.0876319408416748,
      "rewards/rejected": 0.7887665629386902,
      "step": 35
    },
    {
      "epoch": 0.03821808193001314,
      "grad_norm": 10.8125,
      "learning_rate": 2.1666666666666667e-07,
      "logits/chosen": -1.5899121761322021,
      "logits/rejected": -1.6173330545425415,
      "logps/chosen": -0.2993370592594147,
      "logps/rejected": -0.30007094144821167,
      "loss": 1.0462119579315186,
      "loss/core": 1.0462119579315186,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.3351786136627197,
      "rewards/margins": 1.363264799118042,
      "rewards/rejected": 0.9719139933586121,
      "step": 40
    },
    {
      "epoch": 0.04299534217126478,
      "grad_norm": 34.75,
      "learning_rate": 2.4444444444444445e-07,
      "logits/chosen": -1.8352829217910767,
      "logits/rejected": -1.8310245275497437,
      "logps/chosen": -0.2917270064353943,
      "logps/rejected": -0.2925964891910553,
      "loss": 1.0427584648132324,
      "loss/core": 1.0427584648132324,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.797705888748169,
      "rewards/margins": 0.967245876789093,
      "rewards/rejected": 1.8304599523544312,
      "step": 45
    },
    {
      "epoch": 0.04777260241251642,
      "grad_norm": 4.25,
      "learning_rate": 2.7222222222222216e-07,
      "logits/chosen": -1.7508865594863892,
      "logits/rejected": -1.819305419921875,
      "logps/chosen": -0.2858908176422119,
      "logps/rejected": -0.290945827960968,
      "loss": 1.0335471630096436,
      "loss/core": 1.0335471630096436,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8175321817398071,
      "rewards/margins": 0.4548220634460449,
      "rewards/rejected": 1.3627102375030518,
      "step": 50
    },
    {
      "epoch": 0.05254986265376806,
      "grad_norm": 6.375,
      "learning_rate": 3e-07,
      "logits/chosen": -1.7931705713272095,
      "logits/rejected": -1.942427396774292,
      "logps/chosen": -0.29617246985435486,
      "logps/rejected": -0.2928510010242462,
      "loss": 1.0443193912506104,
      "loss/core": 1.0443193912506104,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6830313205718994,
      "rewards/margins": 0.7598537802696228,
      "rewards/rejected": 0.9231774210929871,
      "step": 55
    },
    {
      "epoch": 0.057327122895019705,
      "grad_norm": 23.75,
      "learning_rate": 3.2777777777777776e-07,
      "logits/chosen": -1.921984314918518,
      "logits/rejected": -1.9806658029556274,
      "logps/chosen": -0.2908121645450592,
      "logps/rejected": -0.28627273440361023,
      "loss": 1.0474377870559692,
      "loss/core": 1.0474377870559692,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.452622652053833,
      "rewards/margins": 0.369498610496521,
      "rewards/rejected": 1.0831242799758911,
      "step": 60
    },
    {
      "epoch": 0.06210438313627135,
      "grad_norm": 35.75,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -1.824541449546814,
      "logits/rejected": -1.8805049657821655,
      "logps/chosen": -0.3165293037891388,
      "logps/rejected": -0.3103017210960388,
      "loss": 1.0472676753997803,
      "loss/core": 1.0472676753997803,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4452695846557617,
      "rewards/margins": 1.5458446741104126,
      "rewards/rejected": 0.8994248509407043,
      "step": 65
    },
    {
      "epoch": 0.06688164337752299,
      "grad_norm": 62.25,
      "learning_rate": 3.8333333333333335e-07,
      "logits/chosen": -1.7961727380752563,
      "logits/rejected": -1.8802435398101807,
      "logps/chosen": -0.3014804720878601,
      "logps/rejected": -0.3035959005355835,
      "loss": 1.0426177978515625,
      "loss/core": 1.0426177978515625,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.323197841644287,
      "rewards/margins": 1.3843536376953125,
      "rewards/rejected": 0.9388440847396851,
      "step": 70
    },
    {
      "epoch": 0.07165890361877464,
      "grad_norm": 7.71875,
      "learning_rate": 4.1111111111111107e-07,
      "logits/chosen": -1.7457157373428345,
      "logits/rejected": -1.7732973098754883,
      "logps/chosen": -0.29039743542671204,
      "logps/rejected": -0.29679733514785767,
      "loss": 1.031226396560669,
      "loss/core": 1.031226396560669,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5424000024795532,
      "rewards/margins": 0.5238652229309082,
      "rewards/rejected": 1.018534779548645,
      "step": 75
    },
    {
      "epoch": 0.07643616386002627,
      "grad_norm": 8.9375,
      "learning_rate": 4.3888888888888884e-07,
      "logits/chosen": -1.8642539978027344,
      "logits/rejected": -1.9320249557495117,
      "logps/chosen": -0.3098042905330658,
      "logps/rejected": -0.3067478537559509,
      "loss": 1.0435469150543213,
      "loss/core": 1.0435469150543213,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2842977046966553,
      "rewards/margins": 0.5746160745620728,
      "rewards/rejected": 0.7096816301345825,
      "step": 80
    },
    {
      "epoch": 0.08121342410127792,
      "grad_norm": 5.34375,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -1.7921772003173828,
      "logits/rejected": -1.7714446783065796,
      "logps/chosen": -0.2924823760986328,
      "logps/rejected": -0.2849774658679962,
      "loss": 1.0487070083618164,
      "loss/core": 1.0487070083618164,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.4415981769561768,
      "rewards/margins": 0.7337201833724976,
      "rewards/rejected": 1.7078783512115479,
      "step": 85
    },
    {
      "epoch": 0.08599068434252956,
      "grad_norm": 9.8125,
      "learning_rate": 4.944444444444445e-07,
      "logits/chosen": -1.7024269104003906,
      "logits/rejected": -1.8251234292984009,
      "logps/chosen": -0.30760306119918823,
      "logps/rejected": -0.30106034874916077,
      "loss": 1.0480482578277588,
      "loss/core": 1.0480482578277588,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.881970167160034,
      "rewards/margins": 2.2146878242492676,
      "rewards/rejected": 0.6672819256782532,
      "step": 90
    },
    {
      "epoch": 0.09076794458378121,
      "grad_norm": 29.875,
      "learning_rate": 4.999699149323369e-07,
      "logits/chosen": -1.787375807762146,
      "logits/rejected": -1.7270876169204712,
      "logps/chosen": -0.28113657236099243,
      "logps/rejected": -0.3039243221282959,
      "loss": 1.0137240886688232,
      "loss/core": 1.0137240886688232,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.1713666915893555,
      "rewards/margins": 0.8056703805923462,
      "rewards/rejected": 1.3656964302062988,
      "step": 95
    },
    {
      "epoch": 0.09554520482503284,
      "grad_norm": 5.6875,
      "learning_rate": 4.998477067547739e-07,
      "logits/chosen": -1.668386697769165,
      "logits/rejected": -1.639217734336853,
      "logps/chosen": -0.2826381325721741,
      "logps/rejected": -0.30979663133621216,
      "loss": 1.0171899795532227,
      "loss/core": 1.0171899795532227,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.973217725753784,
      "rewards/margins": 1.4058278799057007,
      "rewards/rejected": 1.567389726638794,
      "step": 100
    },
    {
      "epoch": 0.10032246506628449,
      "grad_norm": 7.84375,
      "learning_rate": 4.996315410727229e-07,
      "logits/chosen": -1.54190993309021,
      "logits/rejected": -1.6111608743667603,
      "logps/chosen": -0.27607348561286926,
      "logps/rejected": -0.2650708854198456,
      "loss": 1.0601215362548828,
      "loss/core": 1.0601215362548828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.1852011680603027,
      "rewards/margins": 1.8053677082061768,
      "rewards/rejected": 1.3798331022262573,
      "step": 105
    },
    {
      "epoch": 0.10509972530753613,
      "grad_norm": 4.90625,
      "learning_rate": 4.993214991772562e-07,
      "logits/chosen": -1.7653499841690063,
      "logits/rejected": -1.7720153331756592,
      "logps/chosen": -0.2952477037906647,
      "logps/rejected": -0.3086181879043579,
      "loss": 1.0259872674942017,
      "loss/core": 1.0259872674942017,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.194749355316162,
      "rewards/margins": 1.190367579460144,
      "rewards/rejected": 1.0043818950653076,
      "step": 110
    },
    {
      "epoch": 0.10987698554878778,
      "grad_norm": 78.0,
      "learning_rate": 4.989176976624511e-07,
      "logits/chosen": -1.7536379098892212,
      "logits/rejected": -1.7781728506088257,
      "logps/chosen": -0.29186949133872986,
      "logps/rejected": -0.2834053635597229,
      "loss": 1.0515763759613037,
      "loss/core": 1.0515763759613037,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.4836719036102295,
      "rewards/margins": 2.2081823348999023,
      "rewards/rejected": 1.2754894495010376,
      "step": 115
    },
    {
      "epoch": 0.11465424579003941,
      "grad_norm": 34.0,
      "learning_rate": 4.984202883815428e-07,
      "logits/chosen": -1.9431312084197998,
      "logits/rejected": -1.8143190145492554,
      "logps/chosen": -0.26071271300315857,
      "logps/rejected": -0.2756553292274475,
      "loss": 1.0258220434188843,
      "loss/core": 1.0258220434188843,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8046910762786865,
      "rewards/margins": 0.35107293725013733,
      "rewards/rejected": 1.4536181688308716,
      "step": 120
    },
    {
      "epoch": 0.11943150603129106,
      "grad_norm": 4.03125,
      "learning_rate": 4.978294583898195e-07,
      "logits/chosen": -1.7581638097763062,
      "logits/rejected": -1.8171669244766235,
      "logps/chosen": -0.2824077606201172,
      "logps/rejected": -0.27793461084365845,
      "loss": 1.0521032810211182,
      "loss/core": 1.0521032810211182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.071238040924072,
      "rewards/margins": 3.047459125518799,
      "rewards/rejected": 1.0237786769866943,
      "step": 125
    },
    {
      "epoch": 0.1242087662725427,
      "grad_norm": 91.5,
      "learning_rate": 4.971454298742779e-07,
      "logits/chosen": -1.738824486732483,
      "logits/rejected": -1.773324966430664,
      "logps/chosen": -0.29637548327445984,
      "logps/rejected": -0.292829692363739,
      "loss": 1.0435963869094849,
      "loss/core": 1.0435963869094849,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.691030979156494,
      "rewards/margins": 0.9570609331130981,
      "rewards/rejected": 1.733970284461975,
      "step": 130
    },
    {
      "epoch": 0.12898602651379434,
      "grad_norm": 11.9375,
      "learning_rate": 4.963684600700678e-07,
      "logits/chosen": -1.7135483026504517,
      "logits/rejected": -1.7729370594024658,
      "logps/chosen": -0.27682191133499146,
      "logps/rejected": -0.26342201232910156,
      "loss": 1.0573183298110962,
      "loss/core": 1.0573183298110962,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.7214828729629517,
      "rewards/margins": 0.8246873617172241,
      "rewards/rejected": 0.8967955708503723,
      "step": 135
    },
    {
      "epoch": 0.13376328675504598,
      "grad_norm": 4.9375,
      "learning_rate": 4.954988411637571e-07,
      "logits/chosen": -1.7595752477645874,
      "logits/rejected": -1.805037498474121,
      "logps/chosen": -0.2989395260810852,
      "logps/rejected": -0.29219749569892883,
      "loss": 1.0499845743179321,
      "loss/core": 1.0499845743179321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.22503924369812,
      "rewards/margins": 1.3095721006393433,
      "rewards/rejected": 0.9154669046401978,
      "step": 140
    },
    {
      "epoch": 0.13854054699629761,
      "grad_norm": 17.25,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -1.6619952917099,
      "logits/rejected": -1.6887853145599365,
      "logps/chosen": -0.3317473828792572,
      "logps/rejected": -0.31527823209762573,
      "loss": 1.0616098642349243,
      "loss/core": 1.0616098642349243,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8598458766937256,
      "rewards/margins": 0.7766186594963074,
      "rewards/rejected": 1.0832273960113525,
      "step": 145
    },
    {
      "epoch": 0.14331780723754928,
      "grad_norm": 6.21875,
      "learning_rate": 4.93482998875813e-07,
      "logits/chosen": -1.6465572118759155,
      "logits/rejected": -1.7042934894561768,
      "logps/chosen": -0.310824453830719,
      "logps/rejected": -0.3171466886997223,
      "loss": 1.0322322845458984,
      "loss/core": 1.0322322845458984,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5014384984970093,
      "rewards/margins": 0.929470419883728,
      "rewards/rejected": 0.571968138217926,
      "step": 150
    },
    {
      "epoch": 0.1480950674788009,
      "grad_norm": 14.75,
      "learning_rate": 4.923375335700223e-07,
      "logits/chosen": -1.9649051427841187,
      "logits/rejected": -1.9704341888427734,
      "logps/chosen": -0.3165343403816223,
      "logps/rejected": -0.303557425737381,
      "loss": 1.056438684463501,
      "loss/core": 1.056438684463501,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.594120740890503,
      "rewards/margins": 0.7040458917617798,
      "rewards/rejected": 0.8900747299194336,
      "step": 155
    },
    {
      "epoch": 0.15287232772005255,
      "grad_norm": 146.0,
      "learning_rate": 4.911009350287347e-07,
      "logits/chosen": -1.7473423480987549,
      "logits/rejected": -1.7553294897079468,
      "logps/chosen": -0.29741325974464417,
      "logps/rejected": -0.2803717255592346,
      "loss": 1.0651330947875977,
      "loss/core": 1.0651330947875977,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.707282543182373,
      "rewards/margins": 2.542534351348877,
      "rewards/rejected": 1.164747953414917,
      "step": 160
    },
    {
      "epoch": 0.15764958796130418,
      "grad_norm": 9.4375,
      "learning_rate": 4.897736682860885e-07,
      "logits/chosen": -1.839644193649292,
      "logits/rejected": -1.9340137243270874,
      "logps/chosen": -0.3177702724933624,
      "logps/rejected": -0.3130636215209961,
      "loss": 1.049741268157959,
      "loss/core": 1.049741268157959,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8438761234283447,
      "rewards/margins": 0.8960213661193848,
      "rewards/rejected": 0.9478548765182495,
      "step": 165
    },
    {
      "epoch": 0.16242684820255585,
      "grad_norm": 14.375,
      "learning_rate": 4.883562324728241e-07,
      "logits/chosen": -1.9273979663848877,
      "logits/rejected": -2.001987934112549,
      "logps/chosen": -0.2734295725822449,
      "logps/rejected": -0.2835696339607239,
      "loss": 1.0265181064605713,
      "loss/core": 1.0265181064605713,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.5538994073867798,
      "rewards/margins": 0.5848657488822937,
      "rewards/rejected": 0.9690335988998413,
      "step": 170
    },
    {
      "epoch": 0.16720410844380748,
      "grad_norm": 17.5,
      "learning_rate": 4.868491606285823e-07,
      "logits/chosen": -1.6086976528167725,
      "logits/rejected": -1.6097809076309204,
      "logps/chosen": -0.2869703769683838,
      "logps/rejected": -0.2960742712020874,
      "loss": 1.0377720594406128,
      "loss/core": 1.0377720594406128,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 5.263535499572754,
      "rewards/margins": 3.293330669403076,
      "rewards/rejected": 1.9702045917510986,
      "step": 175
    },
    {
      "epoch": 0.17198136868505912,
      "grad_norm": 4.375,
      "learning_rate": 4.852530195014489e-07,
      "logits/chosen": -1.8328279256820679,
      "logits/rejected": -1.9173707962036133,
      "logps/chosen": -0.28484848141670227,
      "logps/rejected": -0.2925505042076111,
      "loss": 1.0417985916137695,
      "loss/core": 1.0417985916137695,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 3.295905590057373,
      "rewards/margins": 1.7848495244979858,
      "rewards/rejected": 1.5110561847686768,
      "step": 180
    },
    {
      "epoch": 0.17675862892631075,
      "grad_norm": 6.59375,
      "learning_rate": 4.835684093348244e-07,
      "logits/chosen": -1.7860227823257446,
      "logits/rejected": -1.84549880027771,
      "logps/chosen": -0.25924015045166016,
      "logps/rejected": -0.26765456795692444,
      "loss": 1.0282881259918213,
      "loss/core": 1.0282881259918213,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9171737432479858,
      "rewards/margins": 1.1408088207244873,
      "rewards/rejected": 0.7763648629188538,
      "step": 185
    },
    {
      "epoch": 0.18153588916756241,
      "grad_norm": 5.3125,
      "learning_rate": 4.817959636416969e-07,
      "logits/chosen": -1.6373240947723389,
      "logits/rejected": -1.6969833374023438,
      "logps/chosen": -0.3073750138282776,
      "logps/rejected": -0.30846190452575684,
      "loss": 1.0419862270355225,
      "loss/core": 1.0419862270355225,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9135539531707764,
      "rewards/margins": 1.0026248693466187,
      "rewards/rejected": 0.9109293222427368,
      "step": 190
    },
    {
      "epoch": 0.18631314940881405,
      "grad_norm": 6.9375,
      "learning_rate": 4.799363489664039e-07,
      "logits/chosen": -1.808831810951233,
      "logits/rejected": -1.7658703327178955,
      "logps/chosen": -0.2862377464771271,
      "logps/rejected": -0.3009548783302307,
      "loss": 1.0329413414001465,
      "loss/core": 1.0329413414001465,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.1667561531066895,
      "rewards/margins": 0.9283055067062378,
      "rewards/rejected": 1.2384504079818726,
      "step": 195
    },
    {
      "epoch": 0.19109040965006568,
      "grad_norm": 24.875,
      "learning_rate": 4.779902646339721e-07,
      "logits/chosen": -1.722137451171875,
      "logits/rejected": -1.6766093969345093,
      "logps/chosen": -0.29227524995803833,
      "logps/rejected": -0.289249062538147,
      "loss": 1.0445283651351929,
      "loss/core": 1.0445283651351929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.8198704719543457,
      "rewards/margins": 1.2550045251846313,
      "rewards/rejected": 1.5648664236068726,
      "step": 200
    },
    {
      "epoch": 0.19586766989131732,
      "grad_norm": 18.625,
      "learning_rate": 4.759584424871301e-07,
      "logits/chosen": -1.7560946941375732,
      "logits/rejected": -1.7568002939224243,
      "logps/chosen": -0.3940495252609253,
      "logps/rejected": -0.3073170483112335,
      "loss": 1.2059910297393799,
      "loss/core": 1.2059910297393799,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.591503620147705,
      "rewards/margins": 1.2615159749984741,
      "rewards/rejected": 1.329987645149231,
      "step": 205
    },
    {
      "epoch": 0.20064493013256898,
      "grad_norm": 5.125,
      "learning_rate": 4.738416466110917e-07,
      "logits/chosen": -1.9233392477035522,
      "logits/rejected": -2.00313663482666,
      "logps/chosen": -0.25894150137901306,
      "logps/rejected": -0.2547500729560852,
      "loss": 1.0453473329544067,
      "loss/core": 1.0453473329544067,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7126867771148682,
      "rewards/margins": 0.995030403137207,
      "rewards/rejected": 0.7176561951637268,
      "step": 210
    },
    {
      "epoch": 0.20542219037382062,
      "grad_norm": 51.75,
      "learning_rate": 4.716406730462153e-07,
      "logits/chosen": -1.8582137823104858,
      "logits/rejected": -1.8355731964111328,
      "logps/chosen": -0.28673282265663147,
      "logps/rejected": -0.2773427367210388,
      "loss": 1.0536106824874878,
      "loss/core": 1.0536106824874878,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.7520334720611572,
      "rewards/margins": 1.0252889394760132,
      "rewards/rejected": 1.7267444133758545,
      "step": 215
    },
    {
      "epoch": 0.21019945061507225,
      "grad_norm": 5.46875,
      "learning_rate": 4.693563494886454e-07,
      "logits/chosen": -1.6601266860961914,
      "logits/rejected": -1.832810640335083,
      "logps/chosen": -0.2959999144077301,
      "logps/rejected": -0.28008145093917847,
      "loss": 1.061637043952942,
      "loss/core": 1.061637043952942,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.475043535232544,
      "rewards/margins": 2.701432466506958,
      "rewards/rejected": 0.7736114263534546,
      "step": 220
    },
    {
      "epoch": 0.2149767108563239,
      "grad_norm": 35.0,
      "learning_rate": 4.6698953497905016e-07,
      "logits/chosen": -1.826503038406372,
      "logits/rejected": -1.9233640432357788,
      "logps/chosen": -0.27310609817504883,
      "logps/rejected": -0.27189555764198303,
      "loss": 1.040917158126831,
      "loss/core": 1.040917158126831,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2421226501464844,
      "rewards/margins": 2.2241339683532715,
      "rewards/rejected": 1.017989158630371,
      "step": 225
    },
    {
      "epoch": 0.21975397109757555,
      "grad_norm": 124.0,
      "learning_rate": 4.645411195795709e-07,
      "logits/chosen": -1.8059947490692139,
      "logits/rejected": -1.8729541301727295,
      "logps/chosen": -0.25209683179855347,
      "logps/rejected": -0.2518470287322998,
      "loss": 1.0411286354064941,
      "loss/core": 1.0411286354064941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.6358604431152344,
      "rewards/margins": 2.28558087348938,
      "rewards/rejected": 1.3502795696258545,
      "step": 230
    },
    {
      "epoch": 0.2245312313388272,
      "grad_norm": 5.90625,
      "learning_rate": 4.6201202403910643e-07,
      "logits/chosen": -2.0312843322753906,
      "logits/rejected": -2.0044198036193848,
      "logps/chosen": -0.2595166563987732,
      "logps/rejected": -0.2711992859840393,
      "loss": 1.0278807878494263,
      "loss/core": 1.0278807878494263,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.072014331817627,
      "rewards/margins": 0.8968809247016907,
      "rewards/rejected": 1.1751338243484497,
      "step": 235
    },
    {
      "epoch": 0.22930849158007882,
      "grad_norm": 14.6875,
      "learning_rate": 4.594031994470573e-07,
      "logits/chosen": -1.8727977275848389,
      "logits/rejected": -1.8875232934951782,
      "logps/chosen": -0.27784401178359985,
      "logps/rejected": -0.29739147424697876,
      "loss": 1.0186278820037842,
      "loss/core": 1.0186278820037842,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.4038894176483154,
      "rewards/margins": 1.5082296133041382,
      "rewards/rejected": 1.8956598043441772,
      "step": 240
    },
    {
      "epoch": 0.23408575182133046,
      "grad_norm": 71.5,
      "learning_rate": 4.567156268756593e-07,
      "logits/chosen": -1.5276188850402832,
      "logits/rejected": -1.57925283908844,
      "logps/chosen": -0.2935658097267151,
      "logps/rejected": -0.3003990352153778,
      "loss": 1.0357589721679688,
      "loss/core": 1.0357589721679688,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.293689250946045,
      "rewards/margins": 1.7028945684432983,
      "rewards/rejected": 1.5907951593399048,
      "step": 245
    },
    {
      "epoch": 0.23886301206258212,
      "grad_norm": 4.125,
      "learning_rate": 4.5395031701104303e-07,
      "logits/chosen": -1.9313945770263672,
      "logits/rejected": -1.8916925191879272,
      "logps/chosen": -0.2886388301849365,
      "logps/rejected": -0.28957533836364746,
      "loss": 1.037282943725586,
      "loss/core": 1.037282943725586,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5945872068405151,
      "rewards/margins": 0.4880189895629883,
      "rewards/rejected": 1.1065683364868164,
      "step": 250
    },
    {
      "epoch": 0.24364027230383375,
      "grad_norm": 3.671875,
      "learning_rate": 4.511083097731555e-07,
      "logits/chosen": -1.8803619146347046,
      "logits/rejected": -1.8885911703109741,
      "logps/chosen": -0.30258265137672424,
      "logps/rejected": -0.30648648738861084,
      "loss": 1.0346689224243164,
      "loss/core": 1.0346689224243164,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.496037483215332,
      "rewards/margins": 0.9408340454101562,
      "rewards/rejected": 0.555203378200531,
      "step": 255
    },
    {
      "epoch": 0.2484175325450854,
      "grad_norm": 4.59375,
      "learning_rate": 4.481906739246894e-07,
      "logits/chosen": -1.963661551475525,
      "logits/rejected": -1.9137052297592163,
      "logps/chosen": -0.27444905042648315,
      "logps/rejected": -0.27626100182533264,
      "loss": 1.0373988151550293,
      "loss/core": 1.0373988151550293,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3847882747650146,
      "rewards/margins": 1.301588773727417,
      "rewards/rejected": 1.0831997394561768,
      "step": 260
    },
    {
      "epoch": 0.25319479278633705,
      "grad_norm": 5.75,
      "learning_rate": 4.451985066691648e-07,
      "logits/chosen": -2.0017013549804688,
      "logits/rejected": -2.0215392112731934,
      "logps/chosen": -0.2834641933441162,
      "logps/rejected": -0.2866167724132538,
      "loss": 1.0352154970169067,
      "loss/core": 1.0352154970169067,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.4251301288604736,
      "rewards/margins": 0.7156028151512146,
      "rewards/rejected": 0.709527313709259,
      "step": 265
    },
    {
      "epoch": 0.2579720530275887,
      "grad_norm": 12.0,
      "learning_rate": 4.421329332383158e-07,
      "logits/chosen": -1.7916221618652344,
      "logits/rejected": -1.9190409183502197,
      "logps/chosen": -0.3161014914512634,
      "logps/rejected": -0.293036550283432,
      "loss": 1.0780491828918457,
      "loss/core": 1.0780491828918457,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8342010974884033,
      "rewards/margins": 1.156964898109436,
      "rewards/rejected": 0.677236020565033,
      "step": 270
    },
    {
      "epoch": 0.2627493132688403,
      "grad_norm": 10.6875,
      "learning_rate": 4.3899510646893696e-07,
      "logits/chosen": -1.8899133205413818,
      "logits/rejected": -1.8713350296020508,
      "logps/chosen": -0.2559998631477356,
      "logps/rejected": -0.2608237862586975,
      "loss": 1.0367724895477295,
      "loss/core": 1.0367724895477295,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4705395698547363,
      "rewards/margins": 1.2723934650421143,
      "rewards/rejected": 1.1981456279754639,
      "step": 275
    },
    {
      "epoch": 0.26752657351009196,
      "grad_norm": 22.75,
      "learning_rate": 4.357862063693485e-07,
      "logits/chosen": -1.6495206356048584,
      "logits/rejected": -1.69220769405365,
      "logps/chosen": -0.3172406554222107,
      "logps/rejected": -0.32189950346946716,
      "loss": 1.0348529815673828,
      "loss/core": 1.0348529815673828,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.4244751930236816,
      "rewards/margins": 1.04317307472229,
      "rewards/rejected": 1.3813021183013916,
      "step": 280
    },
    {
      "epoch": 0.2723038337513436,
      "grad_norm": 18.625,
      "learning_rate": 4.3250743967564364e-07,
      "logits/chosen": -1.7316668033599854,
      "logits/rejected": -1.6830451488494873,
      "logps/chosen": -0.26700717210769653,
      "logps/rejected": -0.2771323621273041,
      "loss": 1.0284992456436157,
      "loss/core": 1.0284992456436157,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.8682186603546143,
      "rewards/margins": 1.2967889308929443,
      "rewards/rejected": 1.5714294910430908,
      "step": 285
    },
    {
      "epoch": 0.27708109399259523,
      "grad_norm": 10.9375,
      "learning_rate": 4.29160039397884e-07,
      "logits/chosen": -1.8204319477081299,
      "logits/rejected": -1.9146995544433594,
      "logps/chosen": -0.29401618242263794,
      "logps/rejected": -0.2924644351005554,
      "loss": 1.0413658618927002,
      "loss/core": 1.0413658618927002,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.0374536514282227,
      "rewards/margins": 1.1444836854934692,
      "rewards/rejected": 0.8929699063301086,
      "step": 290
    },
    {
      "epoch": 0.28185835423384686,
      "grad_norm": 68.0,
      "learning_rate": 4.2574526435641546e-07,
      "logits/chosen": -1.6919562816619873,
      "logits/rejected": -1.6904808282852173,
      "logps/chosen": -0.27512839436531067,
      "logps/rejected": -0.2824132442474365,
      "loss": 1.033488154411316,
      "loss/core": 1.033488154411316,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.2550134658813477,
      "rewards/margins": 0.5635195970535278,
      "rewards/rejected": 1.6914937496185303,
      "step": 295
    },
    {
      "epoch": 0.28663561447509855,
      "grad_norm": 14.5,
      "learning_rate": 4.22264398708477e-07,
      "logits/chosen": -1.6543853282928467,
      "logits/rejected": -1.6278355121612549,
      "logps/chosen": -0.2689480185508728,
      "logps/rejected": -0.28801828622817993,
      "loss": 1.0174682140350342,
      "loss/core": 1.0174682140350342,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.770953416824341,
      "rewards/margins": 1.4299379587173462,
      "rewards/rejected": 1.3410152196884155,
      "step": 300
    },
    {
      "epoch": 0.2914128747163502,
      "grad_norm": 19.0,
      "learning_rate": 4.187187514652819e-07,
      "logits/chosen": -1.756967544555664,
      "logits/rejected": -1.7581822872161865,
      "logps/chosen": -0.2890106439590454,
      "logps/rejected": -0.30179038643836975,
      "loss": 1.028267502784729,
      "loss/core": 1.028267502784729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.826168417930603,
      "rewards/margins": 0.5155857801437378,
      "rewards/rejected": 1.3105827569961548,
      "step": 305
    },
    {
      "epoch": 0.2961901349576018,
      "grad_norm": 31.5,
      "learning_rate": 4.151096559997519e-07,
      "logits/chosen": -1.8081289529800415,
      "logits/rejected": -1.7069460153579712,
      "logps/chosen": -0.2982190251350403,
      "logps/rejected": -0.3032774031162262,
      "loss": 1.0330473184585571,
      "loss/core": 1.0330473184585571,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.1015119552612305,
      "rewards/margins": 0.7455414533615112,
      "rewards/rejected": 1.3559703826904297,
      "step": 310
    },
    {
      "epoch": 0.30096739519885346,
      "grad_norm": 6.1875,
      "learning_rate": 4.114384695450905e-07,
      "logits/chosen": -1.9606285095214844,
      "logits/rejected": -1.9415584802627563,
      "logps/chosen": -0.2907028794288635,
      "logps/rejected": -0.28985944390296936,
      "loss": 1.0457550287246704,
      "loss/core": 1.0457550287246704,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.5825884342193604,
      "rewards/margins": 0.5970422029495239,
      "rewards/rejected": 0.9855462312698364,
      "step": 315
    },
    {
      "epoch": 0.3057446554401051,
      "grad_norm": 6.625,
      "learning_rate": 4.077065726843828e-07,
      "logits/chosen": -1.654712438583374,
      "logits/rejected": -1.5861515998840332,
      "logps/chosen": -0.28932780027389526,
      "logps/rejected": -0.276142954826355,
      "loss": 1.0589449405670166,
      "loss/core": 1.0589449405670166,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.8658580780029297,
      "rewards/margins": 0.9593901634216309,
      "rewards/rejected": 1.9064680337905884,
      "step": 320
    },
    {
      "epoch": 0.31052191568135673,
      "grad_norm": 8.9375,
      "learning_rate": 4.039153688314145e-07,
      "logits/chosen": -1.6774059534072876,
      "logits/rejected": -1.7040441036224365,
      "logps/chosen": -0.29178744554519653,
      "logps/rejected": -0.2866431474685669,
      "loss": 1.0456210374832153,
      "loss/core": 1.0456210374832153,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.5071741342544556,
      "rewards/margins": -0.009084177203476429,
      "rewards/rejected": 1.5162582397460938,
      "step": 325
    },
    {
      "epoch": 0.31529917592260837,
      "grad_norm": 12.6875,
      "learning_rate": 4.0006628370290613e-07,
      "logits/chosen": -1.8318054676055908,
      "logits/rejected": -1.81647527217865,
      "logps/chosen": -0.2811618745326996,
      "logps/rejected": -0.28358012437820435,
      "loss": 1.03645920753479,
      "loss/core": 1.03645920753479,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.583585023880005,
      "rewards/margins": 1.4332873821258545,
      "rewards/rejected": 1.15029776096344,
      "step": 330
    },
    {
      "epoch": 0.32007643616386,
      "grad_norm": 40.5,
      "learning_rate": 3.9616076478235826e-07,
      "logits/chosen": -1.6614739894866943,
      "logits/rejected": -1.6278941631317139,
      "logps/chosen": -0.28862547874450684,
      "logps/rejected": -0.2881496548652649,
      "loss": 1.0447041988372803,
      "loss/core": 1.0447041988372803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.6951169967651367,
      "rewards/margins": 1.3128235340118408,
      "rewards/rejected": 1.382293462753296,
      "step": 335
    },
    {
      "epoch": 0.3248536964051117,
      "grad_norm": 11.125,
      "learning_rate": 3.922002807757129e-07,
      "logits/chosen": -1.8974721431732178,
      "logits/rejected": -1.9225714206695557,
      "logps/chosen": -0.29584866762161255,
      "logps/rejected": -0.3060780167579651,
      "loss": 1.0265626907348633,
      "loss/core": 1.0265626907348633,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.664690613746643,
      "rewards/margins": 0.8293659090995789,
      "rewards/rejected": 0.8353245854377747,
      "step": 340
    },
    {
      "epoch": 0.3296309566463633,
      "grad_norm": 13.875,
      "learning_rate": 3.8818632105903315e-07,
      "logits/chosen": -1.673317551612854,
      "logits/rejected": -1.7775061130523682,
      "logps/chosen": -0.2768518328666687,
      "logps/rejected": -0.2922056317329407,
      "loss": 1.0234686136245728,
      "loss/core": 1.0234686136245728,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.881852626800537,
      "rewards/margins": 0.5690313577651978,
      "rewards/rejected": 2.31282114982605,
      "step": 345
    },
    {
      "epoch": 0.33440821688761496,
      "grad_norm": 24.5,
      "learning_rate": 3.841203951184094e-07,
      "logits/chosen": -1.9435956478118896,
      "logits/rejected": -1.8745100498199463,
      "logps/chosen": -0.28658443689346313,
      "logps/rejected": -0.2910448908805847,
      "loss": 1.0343317985534668,
      "loss/core": 1.0343317985534668,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.736780881881714,
      "rewards/margins": 2.516613483428955,
      "rewards/rejected": 1.220167636871338,
      "step": 350
    },
    {
      "epoch": 0.3391854771288666,
      "grad_norm": 4.6875,
      "learning_rate": 3.800040319823038e-07,
      "logits/chosen": -1.682003378868103,
      "logits/rejected": -1.7151581048965454,
      "logps/chosen": -0.3114383816719055,
      "logps/rejected": -0.3123466372489929,
      "loss": 1.043797492980957,
      "loss/core": 1.043797492980957,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.962998390197754,
      "rewards/margins": 1.1944884061813354,
      "rewards/rejected": 0.7685100436210632,
      "step": 355
    },
    {
      "epoch": 0.34396273737011823,
      "grad_norm": 26.875,
      "learning_rate": 3.75838779646545e-07,
      "logits/chosen": -1.719026803970337,
      "logits/rejected": -1.6863863468170166,
      "logps/chosen": -0.31425201892852783,
      "logps/rejected": -0.32097968459129333,
      "loss": 1.0321420431137085,
      "loss/core": 1.0321420431137085,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.7521181106567383,
      "rewards/margins": -0.033901017159223557,
      "rewards/rejected": 1.7860190868377686,
      "step": 360
    },
    {
      "epoch": 0.34873999761136987,
      "grad_norm": 27.75,
      "learning_rate": 3.7162620449218993e-07,
      "logits/chosen": -1.8344494104385376,
      "logits/rejected": -1.8459513187408447,
      "logps/chosen": -0.29098817706108093,
      "logps/rejected": -0.28455573320388794,
      "loss": 1.0480910539627075,
      "loss/core": 1.0480910539627075,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.697824478149414,
      "rewards/margins": 2.8228657245635986,
      "rewards/rejected": 0.8749583959579468,
      "step": 365
    },
    {
      "epoch": 0.3535172578526215,
      "grad_norm": 10.75,
      "learning_rate": 3.673678906964727e-07,
      "logits/chosen": -1.622668981552124,
      "logits/rejected": -1.6612154245376587,
      "logps/chosen": -0.2679634094238281,
      "logps/rejected": -0.2821475863456726,
      "loss": 1.0219076871871948,
      "loss/core": 1.0219076871871948,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.9509494304656982,
      "rewards/margins": 1.733298659324646,
      "rewards/rejected": 1.2176507711410522,
      "step": 370
    },
    {
      "epoch": 0.35829451809387314,
      "grad_norm": 14.25,
      "learning_rate": 3.6306543963705936e-07,
      "logits/chosen": -1.685569405555725,
      "logits/rejected": -1.6597143411636353,
      "logps/chosen": -0.30852535367012024,
      "logps/rejected": -0.2950063943862915,
      "loss": 1.0666342973709106,
      "loss/core": 1.0666342973709106,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 4.885121822357178,
      "rewards/margins": 2.607081174850464,
      "rewards/rejected": 2.278040647506714,
      "step": 375
    },
    {
      "epoch": 0.36307177833512483,
      "grad_norm": 24.375,
      "learning_rate": 3.5872046928983623e-07,
      "logits/chosen": -1.7393958568572998,
      "logits/rejected": -1.839385747909546,
      "logps/chosen": -0.2777525782585144,
      "logps/rejected": -0.28187328577041626,
      "loss": 1.037045955657959,
      "loss/core": 1.037045955657959,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.303683042526245,
      "rewards/margins": 1.0037339925765991,
      "rewards/rejected": 1.299949049949646,
      "step": 380
    },
    {
      "epoch": 0.36784903857637646,
      "grad_norm": 17.875,
      "learning_rate": 3.5433461362045447e-07,
      "logits/chosen": -1.6332000494003296,
      "logits/rejected": -1.6447935104370117,
      "logps/chosen": -0.3114931285381317,
      "logps/rejected": -0.28601643443107605,
      "loss": 1.073043704032898,
      "loss/core": 1.073043704032898,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6627743244171143,
      "rewards/margins": 0.6046234369277954,
      "rewards/rejected": 1.0581510066986084,
      "step": 385
    },
    {
      "epoch": 0.3726262988176281,
      "grad_norm": 5.6875,
      "learning_rate": 3.4990952196986305e-07,
      "logits/chosen": -1.9358383417129517,
      "logits/rejected": -1.9621446132659912,
      "logps/chosen": -0.27814939618110657,
      "logps/rejected": -0.28488942980766296,
      "loss": 1.0306986570358276,
      "loss/core": 1.0306986570358276,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.2591309547424316,
      "rewards/margins": 1.023651361465454,
      "rewards/rejected": 1.2354795932769775,
      "step": 390
    },
    {
      "epoch": 0.37740355905887973,
      "grad_norm": 6.875,
      "learning_rate": 3.4544685843405875e-07,
      "logits/chosen": -1.9065141677856445,
      "logits/rejected": -1.8645994663238525,
      "logps/chosen": -0.3078649640083313,
      "logps/rejected": -0.302230566740036,
      "loss": 1.0477346181869507,
      "loss/core": 1.0477346181869507,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9888968467712402,
      "rewards/margins": 1.2866131067276,
      "rewards/rejected": 1.7022842168807983,
      "step": 395
    },
    {
      "epoch": 0.38218081930013137,
      "grad_norm": 4.15625,
      "learning_rate": 3.4094830123828786e-07,
      "logits/chosen": -1.7724406719207764,
      "logits/rejected": -1.9576416015625,
      "logps/chosen": -0.2747136950492859,
      "logps/rejected": -0.2948540151119232,
      "loss": 1.0195709466934204,
      "loss/core": 1.0195709466934204,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.7523436546325684,
      "rewards/margins": 0.6447616815567017,
      "rewards/rejected": 2.1075825691223145,
      "step": 400
    },
    {
      "epoch": 0.386958079541383,
      "grad_norm": 42.0,
      "learning_rate": 3.3641554210593414e-07,
      "logits/chosen": -1.838844895362854,
      "logits/rejected": -1.8915517330169678,
      "logps/chosen": -0.32020050287246704,
      "logps/rejected": -0.28831055760383606,
      "loss": 1.0958722829818726,
      "loss/core": 1.0958722829818726,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.624875545501709,
      "rewards/margins": 1.3493173122406006,
      "rewards/rejected": 1.2755582332611084,
      "step": 405
    },
    {
      "epoch": 0.39173533978263464,
      "grad_norm": 5.5,
      "learning_rate": 3.3185028562233107e-07,
      "logits/chosen": -1.8786590099334717,
      "logits/rejected": -1.8944612741470337,
      "logps/chosen": -0.26763150095939636,
      "logps/rejected": -0.2702597975730896,
      "loss": 1.038240671157837,
      "loss/core": 1.038240671157837,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.777128219604492,
      "rewards/margins": 1.4700839519500732,
      "rewards/rejected": 1.3070439100265503,
      "step": 410
    },
    {
      "epoch": 0.3965126000238863,
      "grad_norm": 50.0,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -1.6480621099472046,
      "logits/rejected": -1.7010635137557983,
      "logps/chosen": -0.2993305027484894,
      "logps/rejected": -0.2903633117675781,
      "loss": 1.056257963180542,
      "loss/core": 1.056257963180542,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.3192853927612305,
      "rewards/margins": 1.6964190006256104,
      "rewards/rejected": 0.6228662729263306,
      "step": 415
    },
    {
      "epoch": 0.40128986026513797,
      "grad_norm": 8.625,
      "learning_rate": 3.226291594017137e-07,
      "logits/chosen": -1.556668996810913,
      "logits/rejected": -1.5511045455932617,
      "logps/chosen": -0.2851598262786865,
      "logps/rejected": -0.288934588432312,
      "loss": 1.0358564853668213,
      "loss/core": 1.0358564853668213,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2491157054901123,
      "rewards/margins": 1.3990309238433838,
      "rewards/rejected": 1.850084900856018,
      "step": 420
    },
    {
      "epoch": 0.4060671205063896,
      "grad_norm": 18.625,
      "learning_rate": 3.1797675735315454e-07,
      "logits/chosen": -1.6744778156280518,
      "logits/rejected": -1.7476119995117188,
      "logps/chosen": -0.2897992730140686,
      "logps/rejected": -0.2777308523654938,
      "loss": 1.0568740367889404,
      "loss/core": 1.0568740367889404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.9222044944763184,
      "rewards/margins": 0.9556252360343933,
      "rewards/rejected": 1.9665788412094116,
      "step": 425
    },
    {
      "epoch": 0.41084438074764124,
      "grad_norm": 6.375,
      "learning_rate": 3.1329879202620047e-07,
      "logits/chosen": -1.8092721700668335,
      "logits/rejected": -1.8535629510879517,
      "logps/chosen": -0.3078886866569519,
      "logps/rejected": -0.30679917335510254,
      "loss": 1.04254150390625,
      "loss/core": 1.04254150390625,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.51043438911438,
      "rewards/margins": 1.6923158168792725,
      "rewards/rejected": 0.818118691444397,
      "step": 430
    },
    {
      "epoch": 0.41562164098889287,
      "grad_norm": 15.1875,
      "learning_rate": 3.0859702261229613e-07,
      "logits/chosen": -1.84933602809906,
      "logits/rejected": -1.77957284450531,
      "logps/chosen": -0.28597092628479004,
      "logps/rejected": -0.2642970383167267,
      "loss": 1.0709216594696045,
      "loss/core": 1.0709216594696045,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.4492688179016113,
      "rewards/margins": 1.2216014862060547,
      "rewards/rejected": 1.2276675701141357,
      "step": 435
    },
    {
      "epoch": 0.4203989012301445,
      "grad_norm": 44.5,
      "learning_rate": 3.0387321725463e-07,
      "logits/chosen": -1.6569089889526367,
      "logits/rejected": -1.6702871322631836,
      "logps/chosen": -0.31207168102264404,
      "logps/rejected": -0.3157859444618225,
      "loss": 1.0342601537704468,
      "loss/core": 1.0342601537704468,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2018513679504395,
      "rewards/margins": 1.3161648511886597,
      "rewards/rejected": 0.8856865167617798,
      "step": 440
    },
    {
      "epoch": 0.42517616147139614,
      "grad_norm": 79.0,
      "learning_rate": 2.991291523832075e-07,
      "logits/chosen": -1.7703174352645874,
      "logits/rejected": -1.7698230743408203,
      "logps/chosen": -0.2817544937133789,
      "logps/rejected": -0.2641424536705017,
      "loss": 1.0657778978347778,
      "loss/core": 1.0657778978347778,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.6899399757385254,
      "rewards/margins": 1.9449743032455444,
      "rewards/rejected": 1.7449655532836914,
      "step": 445
    },
    {
      "epoch": 0.4299534217126478,
      "grad_norm": 7.625,
      "learning_rate": 2.943666120468088e-07,
      "logits/chosen": -1.723353624343872,
      "logits/rejected": -1.7594738006591797,
      "logps/chosen": -0.30141833424568176,
      "logps/rejected": -0.2993377447128296,
      "loss": 1.0418169498443604,
      "loss/core": 1.0418169498443604,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9154335260391235,
      "rewards/margins": 1.0606777667999268,
      "rewards/rejected": 0.8547555804252625,
      "step": 450
    },
    {
      "epoch": 0.4347306819538994,
      "grad_norm": 14.125,
      "learning_rate": 2.8958738724208073e-07,
      "logits/chosen": -1.8625282049179077,
      "logits/rejected": -1.9700651168823242,
      "logps/chosen": -0.26571571826934814,
      "logps/rejected": -0.2809098958969116,
      "loss": 1.0212137699127197,
      "loss/core": 1.0212137699127197,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1734156608581543,
      "rewards/margins": 0.8710582852363586,
      "rewards/rejected": 1.30235755443573,
      "step": 455
    },
    {
      "epoch": 0.4395079421951511,
      "grad_norm": 7.1875,
      "learning_rate": 2.8479327524001633e-07,
      "logits/chosen": -1.8570115566253662,
      "logits/rejected": -1.7697608470916748,
      "logps/chosen": -0.2919974625110626,
      "logps/rejected": -0.29211685061454773,
      "loss": 1.039764165878296,
      "loss/core": 1.039764165878296,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.0760490894317627,
      "rewards/margins": 1.1579499244689941,
      "rewards/rejected": 0.9180992245674133,
      "step": 460
    },
    {
      "epoch": 0.44428520243640274,
      "grad_norm": 18.375,
      "learning_rate": 2.7998607891007493e-07,
      "logits/chosen": -2.035067081451416,
      "logits/rejected": -2.0927367210388184,
      "logps/chosen": -0.3042372763156891,
      "logps/rejected": -0.3184296488761902,
      "loss": 1.069641351699829,
      "loss/core": 1.069641351699829,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.6732687950134277,
      "rewards/margins": 1.3037307262420654,
      "rewards/rejected": 1.3695380687713623,
      "step": 465
    },
    {
      "epoch": 0.4490624626776544,
      "grad_norm": 8.875,
      "learning_rate": 2.7516760604219616e-07,
      "logits/chosen": -1.872999906539917,
      "logits/rejected": -1.875903844833374,
      "logps/chosen": -0.296085923910141,
      "logps/rejected": -0.2649732828140259,
      "loss": 1.0810202360153198,
      "loss/core": 1.0810202360153198,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5348351001739502,
      "rewards/margins": 0.5164709091186523,
      "rewards/rejected": 1.0183641910552979,
      "step": 470
    },
    {
      "epoch": 0.453839722918906,
      "grad_norm": 5.25,
      "learning_rate": 2.703396686669646e-07,
      "logits/chosen": -1.8748611211776733,
      "logits/rejected": -1.8687400817871094,
      "logps/chosen": -0.3014170825481415,
      "logps/rejected": -0.2969133257865906,
      "loss": 1.044512391090393,
      "loss/core": 1.044512391090393,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.9237457513809204,
      "rewards/margins": 1.39292573928833,
      "rewards/rejected": 0.5308200120925903,
      "step": 475
    },
    {
      "epoch": 0.45861698316015764,
      "grad_norm": 44.5,
      "learning_rate": 2.6550408237417884e-07,
      "logits/chosen": -1.7492471933364868,
      "logits/rejected": -1.7116022109985352,
      "logps/chosen": -0.3013203740119934,
      "logps/rejected": -0.31782329082489014,
      "loss": 1.023486852645874,
      "loss/core": 1.023486852645874,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.455150604248047,
      "rewards/margins": 1.1551944017410278,
      "rewards/rejected": 1.2999564409255981,
      "step": 480
    },
    {
      "epoch": 0.4633942434014093,
      "grad_norm": 30.75,
      "learning_rate": 2.6066266563008265e-07,
      "logits/chosen": -1.691613793373108,
      "logits/rejected": -1.6590354442596436,
      "logps/chosen": -0.2971816658973694,
      "logps/rejected": -0.29611891508102417,
      "loss": 1.040845513343811,
      "loss/core": 1.040845513343811,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0947299003601074,
      "rewards/margins": 1.0051168203353882,
      "rewards/rejected": 1.0896131992340088,
      "step": 485
    },
    {
      "epoch": 0.4681715036426609,
      "grad_norm": 9.125,
      "learning_rate": 2.5581723909351404e-07,
      "logits/chosen": -1.6561720371246338,
      "logits/rejected": -1.6545698642730713,
      "logps/chosen": -0.3090967535972595,
      "logps/rejected": -0.30728867650032043,
      "loss": 1.0426571369171143,
      "loss/core": 1.0426571369171143,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3853068351745605,
      "rewards/margins": 1.2928699254989624,
      "rewards/rejected": 1.0924367904663086,
      "step": 490
    },
    {
      "epoch": 0.47294876388391255,
      "grad_norm": 12.1875,
      "learning_rate": 2.509696249312301e-07,
      "logits/chosen": -1.7215149402618408,
      "logits/rejected": -1.759275197982788,
      "logps/chosen": -0.28365054726600647,
      "logps/rejected": -0.2850833535194397,
      "loss": 1.0367029905319214,
      "loss/core": 1.0367029905319214,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.926408052444458,
      "rewards/margins": 0.9314010739326477,
      "rewards/rejected": 0.9950069189071655,
      "step": 495
    },
    {
      "epoch": 0.47772602412516424,
      "grad_norm": 63.25,
      "learning_rate": 2.461216461326642e-07,
      "logits/chosen": -1.6816461086273193,
      "logits/rejected": -1.7013599872589111,
      "logps/chosen": -0.2813529670238495,
      "logps/rejected": -0.28367528319358826,
      "loss": 1.0371912717819214,
      "loss/core": 1.0371912717819214,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.650829315185547,
      "rewards/margins": 1.9984947443008423,
      "rewards/rejected": 1.6523345708847046,
      "step": 500
    },
    {
      "epoch": 0.4825032843664159,
      "grad_norm": 8.1875,
      "learning_rate": 2.412751258243748e-07,
      "logits/chosen": -1.6414563655853271,
      "logits/rejected": -1.585834264755249,
      "logps/chosen": -0.3117072582244873,
      "logps/rejected": -0.3133184313774109,
      "loss": 1.0436599254608154,
      "loss/core": 1.0436599254608154,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.4120712280273438,
      "rewards/margins": 2.0417308807373047,
      "rewards/rejected": 1.3703404664993286,
      "step": 505
    },
    {
      "epoch": 0.4872805446076675,
      "grad_norm": 10.1875,
      "learning_rate": 2.3643188658444158e-07,
      "logits/chosen": -1.901231050491333,
      "logits/rejected": -1.9024569988250732,
      "logps/chosen": -0.29130491614341736,
      "logps/rejected": -0.28371134400367737,
      "loss": 1.05287504196167,
      "loss/core": 1.05287504196167,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.3334412574768066,
      "rewards/margins": 1.1235178709030151,
      "rewards/rejected": 1.209923505783081,
      "step": 510
    },
    {
      "epoch": 0.49205780484891914,
      "grad_norm": 28.375,
      "learning_rate": 2.315937497570688e-07,
      "logits/chosen": -1.5855246782302856,
      "logits/rejected": -1.6642907857894897,
      "logps/chosen": -0.2806105613708496,
      "logps/rejected": -0.2813439965248108,
      "loss": 1.0392553806304932,
      "loss/core": 1.0392553806304932,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8368934392929077,
      "rewards/margins": 0.04681912809610367,
      "rewards/rejected": 1.7900745868682861,
      "step": 515
    },
    {
      "epoch": 0.4968350650901708,
      "grad_norm": 19.875,
      "learning_rate": 2.2676253476765194e-07,
      "logits/chosen": -1.7813889980316162,
      "logits/rejected": -1.7595758438110352,
      "logps/chosen": -0.29206109046936035,
      "logps/rejected": -0.28826069831848145,
      "loss": 1.0510475635528564,
      "loss/core": 1.0510475635528564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.1681056022644043,
      "rewards/margins": 1.8101484775543213,
      "rewards/rejected": 1.357957363128662,
      "step": 520
    },
    {
      "epoch": 0.5016123253314224,
      "grad_norm": 6.0625,
      "learning_rate": 2.2194005843856633e-07,
      "logits/chosen": -1.7787355184555054,
      "logits/rejected": -1.8631689548492432,
      "logps/chosen": -0.2766266465187073,
      "logps/rejected": -0.27404943108558655,
      "loss": 1.042309045791626,
      "loss/core": 1.042309045791626,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1334118843078613,
      "rewards/margins": 1.3973788022994995,
      "rewards/rejected": 0.7360331416130066,
      "step": 525
    },
    {
      "epoch": 0.5063895855726741,
      "grad_norm": 5.28125,
      "learning_rate": 2.1712813430593434e-07,
      "logits/chosen": -1.885209321975708,
      "logits/rejected": -1.9635217189788818,
      "logps/chosen": -0.28333884477615356,
      "logps/rejected": -0.29169341921806335,
      "loss": 1.0290471315383911,
      "loss/core": 1.0290471315383911,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6382904052734375,
      "rewards/margins": 0.8100762367248535,
      "rewards/rejected": 0.8282139897346497,
      "step": 530
    },
    {
      "epoch": 0.5111668458139257,
      "grad_norm": 5.5625,
      "learning_rate": 2.123285719376292e-07,
      "logits/chosen": -1.7140337228775024,
      "logits/rejected": -1.700222373008728,
      "logps/chosen": -0.2663342356681824,
      "logps/rejected": -0.2759917378425598,
      "loss": 1.0282187461853027,
      "loss/core": 1.0282187461853027,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.3432140350341797,
      "rewards/margins": 2.151822566986084,
      "rewards/rejected": 1.1913913488388062,
      "step": 535
    },
    {
      "epoch": 0.5159441060551774,
      "grad_norm": 39.25,
      "learning_rate": 2.0754317625276982e-07,
      "logits/chosen": -1.79038405418396,
      "logits/rejected": -1.9031089544296265,
      "logps/chosen": -0.29101166129112244,
      "logps/rejected": -0.2809355556964874,
      "loss": 1.0550512075424194,
      "loss/core": 1.0550512075424194,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.919968366622925,
      "rewards/margins": 2.102735996246338,
      "rewards/rejected": 0.8172324299812317,
      "step": 540
    },
    {
      "epoch": 0.520721366296429,
      "grad_norm": 35.5,
      "learning_rate": 2.0277374684296498e-07,
      "logits/chosen": -1.715163230895996,
      "logits/rejected": -1.8029130697250366,
      "logps/chosen": -0.30172231793403625,
      "logps/rejected": -0.32220780849456787,
      "loss": 1.0138914585113525,
      "loss/core": 1.0138914585113525,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.903212070465088,
      "rewards/margins": 1.434679627418518,
      "rewards/rejected": 1.4685324430465698,
      "step": 545
    },
    {
      "epoch": 0.5254986265376806,
      "grad_norm": 5.90625,
      "learning_rate": 1.980220772955602e-07,
      "logits/chosen": -1.949233055114746,
      "logits/rejected": -2.019287347793579,
      "logps/chosen": -0.2586565315723419,
      "logps/rejected": -0.2541942596435547,
      "loss": 1.0455553531646729,
      "loss/core": 1.0455553531646729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.489222288131714,
      "rewards/margins": 0.4238598942756653,
      "rewards/rejected": 2.0653622150421143,
      "step": 550
    },
    {
      "epoch": 0.5302758867789323,
      "grad_norm": 19.125,
      "learning_rate": 1.932899545191433e-07,
      "logits/chosen": -1.9635541439056396,
      "logits/rejected": -1.9265022277832031,
      "logps/chosen": -0.290753573179245,
      "logps/rejected": -0.27892938256263733,
      "loss": 1.0557947158813477,
      "loss/core": 1.0557947158813477,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.1324610710144043,
      "rewards/margins": 0.5951575040817261,
      "rewards/rejected": 1.5373035669326782,
      "step": 555
    },
    {
      "epoch": 0.5350531470201839,
      "grad_norm": 77.0,
      "learning_rate": 1.885791580715609e-07,
      "logits/chosen": -1.6048539876937866,
      "logits/rejected": -1.6119636297225952,
      "logps/chosen": -0.28494495153427124,
      "logps/rejected": -0.286663681268692,
      "loss": 1.038069725036621,
      "loss/core": 1.038069725036621,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.65592098236084,
      "rewards/margins": 0.7274275422096252,
      "rewards/rejected": 1.9284934997558594,
      "step": 560
    },
    {
      "epoch": 0.5398304072614356,
      "grad_norm": 6.84375,
      "learning_rate": 1.8389145949069951e-07,
      "logits/chosen": -1.6353938579559326,
      "logits/rejected": -1.6788675785064697,
      "logps/chosen": -0.2736624479293823,
      "logps/rejected": -0.26380524039268494,
      "loss": 1.052668809890747,
      "loss/core": 1.052668809890747,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.4656434059143066,
      "rewards/margins": 1.5169572830200195,
      "rewards/rejected": 1.9486862421035767,
      "step": 565
    },
    {
      "epoch": 0.5446076675026872,
      "grad_norm": 45.25,
      "learning_rate": 1.792286216282824e-07,
      "logits/chosen": -1.558429479598999,
      "logits/rejected": -1.5371347665786743,
      "logps/chosen": -0.2772481441497803,
      "logps/rejected": -0.2760164737701416,
      "loss": 1.0409849882125854,
      "loss/core": 1.0409849882125854,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.2676281929016113,
      "rewards/margins": 0.6158636212348938,
      "rewards/rejected": 1.6517642736434937,
      "step": 570
    },
    {
      "epoch": 0.5493849277439389,
      "grad_norm": 3.640625,
      "learning_rate": 1.745923979869336e-07,
      "logits/chosen": -1.8342043161392212,
      "logits/rejected": -1.8309091329574585,
      "logps/chosen": -0.2796408534049988,
      "logps/rejected": -0.2815830409526825,
      "loss": 1.0364043712615967,
      "loss/core": 1.0364043712615967,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.265101194381714,
      "rewards/margins": 1.3904941082000732,
      "rewards/rejected": 0.8746069669723511,
      "step": 575
    },
    {
      "epoch": 0.5541621879851905,
      "grad_norm": 7.4375,
      "learning_rate": 1.6998453206075708e-07,
      "logits/chosen": -2.014143466949463,
      "logits/rejected": -2.0165810585021973,
      "logps/chosen": -0.2994392216205597,
      "logps/rejected": -0.30162468552589417,
      "loss": 1.037367582321167,
      "loss/core": 1.037367582321167,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.404824137687683,
      "rewards/margins": 0.4138920307159424,
      "rewards/rejected": 0.990932285785675,
      "step": 580
    },
    {
      "epoch": 0.5589394482264421,
      "grad_norm": 5.90625,
      "learning_rate": 1.6540675667967973e-07,
      "logits/chosen": -1.724384069442749,
      "logits/rejected": -1.7589279413223267,
      "logps/chosen": -0.30380764603614807,
      "logps/rejected": -0.2969801127910614,
      "loss": 1.0485934019088745,
      "loss/core": 1.0485934019088745,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.2653725147247314,
      "rewards/margins": 1.247117042541504,
      "rewards/rejected": 1.0182554721832275,
      "step": 585
    },
    {
      "epoch": 0.5637167084676937,
      "grad_norm": 11.5625,
      "learning_rate": 1.60860793357805e-07,
      "logits/chosen": -1.806414008140564,
      "logits/rejected": -1.8376471996307373,
      "logps/chosen": -0.2740582823753357,
      "logps/rejected": -0.27855148911476135,
      "loss": 1.033927083015442,
      "loss/core": 1.033927083015442,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.1178157329559326,
      "rewards/margins": 0.7777595520019531,
      "rewards/rejected": 1.3400561809539795,
      "step": 590
    },
    {
      "epoch": 0.5684939687089454,
      "grad_norm": 10.5,
      "learning_rate": 1.5634835164602196e-07,
      "logits/chosen": -1.6470069885253906,
      "logits/rejected": -1.700169563293457,
      "logps/chosen": -0.2837096154689789,
      "logps/rejected": -0.28879398107528687,
      "loss": 1.033031940460205,
      "loss/core": 1.033031940460205,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.160616636276245,
      "rewards/margins": 0.9769798517227173,
      "rewards/rejected": 1.1836367845535278,
      "step": 595
    },
    {
      "epoch": 0.5732712289501971,
      "grad_norm": 13.625,
      "learning_rate": 1.518711284891132e-07,
      "logits/chosen": -1.9625580310821533,
      "logits/rejected": -1.928464651107788,
      "logps/chosen": -0.2618710696697235,
      "logps/rejected": -0.27233943343162537,
      "loss": 1.0305383205413818,
      "loss/core": 1.0305383205413818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9427595138549805,
      "rewards/margins": 0.7403771877288818,
      "rewards/rejected": 1.2023823261260986,
      "step": 600
    },
    {
      "epoch": 0.5780484891914487,
      "grad_norm": 79.5,
      "learning_rate": 1.47430807587603e-07,
      "logits/chosen": -1.7923481464385986,
      "logits/rejected": -1.9285895824432373,
      "logps/chosen": -0.3259024918079376,
      "logps/rejected": -0.33158808946609497,
      "loss": 1.0331567525863647,
      "loss/core": 1.0331567525863647,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.143375873565674,
      "rewards/margins": 1.2113240957260132,
      "rewards/rejected": 0.9320517778396606,
      "step": 605
    },
    {
      "epoch": 0.5828257494327004,
      "grad_norm": 18.125,
      "learning_rate": 1.430290587645865e-07,
      "logits/chosen": -1.8447580337524414,
      "logits/rejected": -1.826443076133728,
      "logps/chosen": -0.2840345799922943,
      "logps/rejected": -0.29238995909690857,
      "loss": 1.0294818878173828,
      "loss/core": 1.0294818878173828,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1355230808258057,
      "rewards/margins": 0.9689822196960449,
      "rewards/rejected": 1.1665408611297607,
      "step": 610
    },
    {
      "epoch": 0.587603009673952,
      "grad_norm": 10.75,
      "learning_rate": 1.3866753733777765e-07,
      "logits/chosen": -1.8555551767349243,
      "logits/rejected": -1.9416917562484741,
      "logps/chosen": -0.27575385570526123,
      "logps/rejected": -0.3149864971637726,
      "loss": 1.0164090394973755,
      "loss/core": 1.0164090394973755,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.061546802520752,
      "rewards/margins": 1.4506876468658447,
      "rewards/rejected": 1.6108589172363281,
      "step": 615
    },
    {
      "epoch": 0.5923802699152036,
      "grad_norm": 46.25,
      "learning_rate": 1.343478834970121e-07,
      "logits/chosen": -1.8785676956176758,
      "logits/rejected": -1.8584808111190796,
      "logps/chosen": -0.2844165861606598,
      "logps/rejected": -0.29962268471717834,
      "loss": 1.0305485725402832,
      "loss/core": 1.0305485725402832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.5849804878234863,
      "rewards/margins": 1.1563302278518677,
      "rewards/rejected": 1.4286500215530396,
      "step": 620
    },
    {
      "epoch": 0.5971575301564552,
      "grad_norm": 11.375,
      "learning_rate": 1.3007172168743852e-07,
      "logits/chosen": -1.8624359369277954,
      "logits/rejected": -1.852315902709961,
      "logps/chosen": -0.2829623222351074,
      "logps/rejected": -0.30127206444740295,
      "loss": 1.0275830030441284,
      "loss/core": 1.0275830030441284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9424110651016235,
      "rewards/margins": 0.5080059170722961,
      "rewards/rejected": 1.4344053268432617,
      "step": 625
    },
    {
      "epoch": 0.6019347903977069,
      "grad_norm": 26.25,
      "learning_rate": 1.25840659998631e-07,
      "logits/chosen": -1.7425400018692017,
      "logits/rejected": -1.77998948097229,
      "logps/chosen": -0.29986807703971863,
      "logps/rejected": -0.3102700114250183,
      "loss": 1.0461167097091675,
      "loss/core": 1.0461167097091675,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.9104256629943848,
      "rewards/margins": 1.9246628284454346,
      "rewards/rejected": 0.9857630729675293,
      "step": 630
    },
    {
      "epoch": 0.6067120506389586,
      "grad_norm": 18.75,
      "learning_rate": 1.2165628955985313e-07,
      "logits/chosen": -1.8899660110473633,
      "logits/rejected": -1.788923978805542,
      "logps/chosen": -0.30364158749580383,
      "logps/rejected": -0.30772846937179565,
      "loss": 1.0387496948242188,
      "loss/core": 1.0387496948242188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.7164779901504517,
      "rewards/margins": 0.41386109590530396,
      "rewards/rejected": 1.302616834640503,
      "step": 635
    },
    {
      "epoch": 0.6114893108802102,
      "grad_norm": 11.3125,
      "learning_rate": 1.175201839416988e-07,
      "logits/chosen": -1.7163013219833374,
      "logits/rejected": -1.7982505559921265,
      "logps/chosen": -0.2873172163963318,
      "logps/rejected": -0.29000353813171387,
      "loss": 1.036372184753418,
      "loss/core": 1.036372184753418,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6618483066558838,
      "rewards/margins": 1.0304663181304932,
      "rewards/rejected": 0.6313816905021667,
      "step": 640
    },
    {
      "epoch": 0.6162665711214619,
      "grad_norm": 54.0,
      "learning_rate": 1.1343389856433658e-07,
      "logits/chosen": -1.9578911066055298,
      "logits/rejected": -1.9668989181518555,
      "logps/chosen": -0.2846335470676422,
      "logps/rejected": -0.27758553624153137,
      "loss": 1.0494635105133057,
      "loss/core": 1.0494635105133057,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0247325897216797,
      "rewards/margins": 0.21105141937732697,
      "rewards/rejected": 1.8136810064315796,
      "step": 645
    },
    {
      "epoch": 0.6210438313627135,
      "grad_norm": 70.5,
      "learning_rate": 1.0939897011258e-07,
      "logits/chosen": -1.6655654907226562,
      "logits/rejected": -1.7558271884918213,
      "logps/chosen": -0.29102975130081177,
      "logps/rejected": -0.2871955335140228,
      "loss": 1.046040654182434,
      "loss/core": 1.046040654182434,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.1432576179504395,
      "rewards/margins": 1.0412713289260864,
      "rewards/rejected": 2.1019864082336426,
      "step": 650
    },
    {
      "epoch": 0.6258210916039652,
      "grad_norm": 65.0,
      "learning_rate": 1.0541691595800336e-07,
      "logits/chosen": -1.9179801940917969,
      "logits/rejected": -1.831691026687622,
      "logps/chosen": -0.2815897464752197,
      "logps/rejected": -0.29244643449783325,
      "loss": 1.026608943939209,
      "loss/core": 1.026608943939209,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.7700217962265015,
      "rewards/margins": 0.5304393768310547,
      "rewards/rejected": 1.2395825386047363,
      "step": 655
    },
    {
      "epoch": 0.6305983518452167,
      "grad_norm": 5.0,
      "learning_rate": 1.0148923358832021e-07,
      "logits/chosen": -1.7689975500106812,
      "logits/rejected": -1.8483155965805054,
      "logps/chosen": -0.2766217589378357,
      "logps/rejected": -0.2837319076061249,
      "loss": 1.0300475358963013,
      "loss/core": 1.0300475358963013,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.812479019165039,
      "rewards/margins": 0.43489259481430054,
      "rewards/rejected": 1.3775863647460938,
      "step": 660
    },
    {
      "epoch": 0.6353756120864684,
      "grad_norm": 8.1875,
      "learning_rate": 9.761740004423926e-08,
      "logits/chosen": -1.8706638813018799,
      "logits/rejected": -1.864198088645935,
      "logps/chosen": -0.2770681381225586,
      "logps/rejected": -0.2515154182910919,
      "loss": 1.073243498802185,
      "loss/core": 1.073243498802185,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2681612968444824,
      "rewards/margins": 1.0078423023223877,
      "rewards/rejected": 1.2603189945220947,
      "step": 665
    },
    {
      "epoch": 0.64015287232772,
      "grad_norm": 5.125,
      "learning_rate": 9.380287136400999e-08,
      "logits/chosen": -1.8545506000518799,
      "logits/rejected": -1.9372055530548096,
      "logps/chosen": -0.2920622229576111,
      "logps/rejected": -0.2942867577075958,
      "loss": 1.0373417139053345,
      "loss/core": 1.0373417139053345,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.3302066326141357,
      "rewards/margins": 0.37538617849349976,
      "rewards/rejected": 0.954820454120636,
      "step": 670
    },
    {
      "epoch": 0.6449301325689717,
      "grad_norm": 19.375,
      "learning_rate": 9.004708203586628e-08,
      "logits/chosen": -2.0307302474975586,
      "logits/rejected": -1.9755264520645142,
      "logps/chosen": -0.28046339750289917,
      "logps/rejected": -0.29223278164863586,
      "loss": 1.0251643657684326,
      "loss/core": 1.0251643657684326,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.621219277381897,
      "rewards/margins": 0.32391253113746643,
      "rewards/rejected": 1.2973066568374634,
      "step": 675
    },
    {
      "epoch": 0.6497073928102234,
      "grad_norm": 11.6875,
      "learning_rate": 8.635144445857407e-08,
      "logits/chosen": -1.800567626953125,
      "logits/rejected": -1.8191674947738647,
      "logps/chosen": -0.2679905295372009,
      "logps/rejected": -0.27527937293052673,
      "loss": 1.0352510213851929,
      "loss/core": 1.0352510213851929,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.176344871520996,
      "rewards/margins": 1.9697097539901733,
      "rewards/rejected": 1.206634759902954,
      "step": 680
    },
    {
      "epoch": 0.654484653051475,
      "grad_norm": 35.25,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -1.8452231884002686,
      "logits/rejected": -1.8123219013214111,
      "logps/chosen": -0.2911657691001892,
      "logps/rejected": -0.2622499167919159,
      "loss": 1.0803478956222534,
      "loss/core": 1.0803478956222534,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.31235933303833,
      "rewards/margins": -0.02849596180021763,
      "rewards/rejected": 2.340855121612549,
      "step": 685
    },
    {
      "epoch": 0.6592619132927267,
      "grad_norm": 40.0,
      "learning_rate": 7.91461605259007e-08,
      "logits/chosen": -1.593219518661499,
      "logits/rejected": -1.6734249591827393,
      "logps/chosen": -0.31815314292907715,
      "logps/rejected": -0.32347214221954346,
      "loss": 1.0386189222335815,
      "loss/core": 1.0386189222335815,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.939263105392456,
      "rewards/margins": 0.8087685704231262,
      "rewards/rejected": 2.1304945945739746,
      "step": 690
    },
    {
      "epoch": 0.6640391735339782,
      "grad_norm": 18.625,
      "learning_rate": 7.563922378313217e-08,
      "logits/chosen": -1.778198480606079,
      "logits/rejected": -1.7811349630355835,
      "logps/chosen": -0.2683827579021454,
      "logps/rejected": -0.2815011143684387,
      "loss": 1.0239763259887695,
      "loss/core": 1.0239763259887695,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.358581781387329,
      "rewards/margins": 1.180909514427185,
      "rewards/rejected": 1.1776721477508545,
      "step": 695
    },
    {
      "epoch": 0.6688164337752299,
      "grad_norm": 29.25,
      "learning_rate": 7.219785699746572e-08,
      "logits/chosen": -1.9994821548461914,
      "logits/rejected": -1.947238564491272,
      "logps/chosen": -0.286782830953598,
      "logps/rejected": -0.27130791544914246,
      "loss": 1.0630942583084106,
      "loss/core": 1.0630942583084106,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.2841906547546387,
      "rewards/margins": 1.6503528356552124,
      "rewards/rejected": 1.633837342262268,
      "step": 700
    },
    {
      "epoch": 0.6735936940164815,
      "grad_norm": 4.53125,
      "learning_rate": 6.882335432620779e-08,
      "logits/chosen": -1.8093124628067017,
      "logits/rejected": -1.754615068435669,
      "logps/chosen": -0.3254457712173462,
      "logps/rejected": -0.31252914667129517,
      "loss": 1.0568554401397705,
      "loss/core": 1.0568554401397705,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.3327386379241943,
      "rewards/margins": 0.8064883947372437,
      "rewards/rejected": 0.5262502431869507,
      "step": 705
    },
    {
      "epoch": 0.6783709542577332,
      "grad_norm": 19.875,
      "learning_rate": 6.551698478180589e-08,
      "logits/chosen": -1.6687633991241455,
      "logits/rejected": -1.6874847412109375,
      "logps/chosen": -0.2778703570365906,
      "logps/rejected": -0.28912389278411865,
      "loss": 1.025848150253296,
      "loss/core": 1.025848150253296,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.4708938598632812,
      "rewards/margins": 1.6537853479385376,
      "rewards/rejected": 0.8171082735061646,
      "step": 710
    },
    {
      "epoch": 0.6831482144989849,
      "grad_norm": 9.0,
      "learning_rate": 6.22799917546252e-08,
      "logits/chosen": -1.8202791213989258,
      "logits/rejected": -1.9110924005508423,
      "logps/chosen": -0.2837766408920288,
      "logps/rejected": -0.28179988265037537,
      "loss": 1.0426539182662964,
      "loss/core": 1.0426539182662964,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3514044284820557,
      "rewards/margins": 1.4219096899032593,
      "rewards/rejected": 0.9294947385787964,
      "step": 715
    },
    {
      "epoch": 0.6879254747402365,
      "grad_norm": 7.40625,
      "learning_rate": 5.9113592545359944e-08,
      "logits/chosen": -1.9379427433013916,
      "logits/rejected": -1.941803216934204,
      "logps/chosen": -0.2888217568397522,
      "logps/rejected": -0.3106004297733307,
      "loss": 1.0178791284561157,
      "loss/core": 1.0178791284561157,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.548539638519287,
      "rewards/margins": 1.4435575008392334,
      "rewards/rejected": 1.1049822568893433,
      "step": 720
    },
    {
      "epoch": 0.6927027349814882,
      "grad_norm": 3.9375,
      "learning_rate": 5.601897790725643e-08,
      "logits/chosen": -1.8667323589324951,
      "logits/rejected": -1.8213609457015991,
      "logps/chosen": -0.28370243310928345,
      "logps/rejected": -0.2769761383533478,
      "loss": 1.0477391481399536,
      "loss/core": 1.0477391481399536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1843743324279785,
      "rewards/margins": 0.06037312000989914,
      "rewards/rejected": 1.124001145362854,
      "step": 725
    },
    {
      "epoch": 0.6974799952227397,
      "grad_norm": 22.125,
      "learning_rate": 5.299731159831952e-08,
      "logits/chosen": -1.7045841217041016,
      "logits/rejected": -1.7231441736221313,
      "logps/chosen": -0.3042663633823395,
      "logps/rejected": -0.3213575482368469,
      "loss": 1.0313047170639038,
      "loss/core": 1.0313047170639038,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.7923970222473145,
      "rewards/margins": 1.8490235805511475,
      "rewards/rejected": 0.9433733820915222,
      "step": 730
    },
    {
      "epoch": 0.7022572554639914,
      "grad_norm": 44.75,
      "learning_rate": 5.0049729943671013e-08,
      "logits/chosen": -1.915514349937439,
      "logits/rejected": -1.888715386390686,
      "logps/chosen": -0.2653517723083496,
      "logps/rejected": -0.27663201093673706,
      "loss": 1.0275311470031738,
      "loss/core": 1.0275311470031738,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.803576707839966,
      "rewards/margins": 1.734165906906128,
      "rewards/rejected": 1.0694106817245483,
      "step": 735
    },
    {
      "epoch": 0.707034515705243,
      "grad_norm": 30.5,
      "learning_rate": 4.7177341408223994e-08,
      "logits/chosen": -1.6809492111206055,
      "logits/rejected": -1.6404021978378296,
      "logps/chosen": -0.29158157110214233,
      "logps/rejected": -0.3046492338180542,
      "loss": 1.0248371362686157,
      "loss/core": 1.0248371362686157,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1834161281585693,
      "rewards/margins": 0.5413880348205566,
      "rewards/rejected": 1.6420282125473022,
      "step": 740
    },
    {
      "epoch": 0.7118117759464947,
      "grad_norm": 6.375,
      "learning_rate": 4.438122617983442e-08,
      "logits/chosen": -1.887942910194397,
      "logits/rejected": -1.8876336812973022,
      "logps/chosen": -0.2923053801059723,
      "logps/rejected": -0.2827454209327698,
      "loss": 1.0520001649856567,
      "loss/core": 1.0520001649856567,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3299145698547363,
      "rewards/margins": 1.1022189855575562,
      "rewards/rejected": 1.2276954650878906,
      "step": 745
    },
    {
      "epoch": 0.7165890361877463,
      "grad_norm": 36.0,
      "learning_rate": 4.1662435763087114e-08,
      "logits/chosen": -1.841260552406311,
      "logits/rejected": -1.7571378946304321,
      "logps/chosen": -0.2794218361377716,
      "logps/rejected": -0.3028506338596344,
      "loss": 1.012818694114685,
      "loss/core": 1.012818694114685,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.2234444618225098,
      "rewards/margins": 0.845853328704834,
      "rewards/rejected": 1.3775911331176758,
      "step": 750
    },
    {
      "epoch": 0.721366296428998,
      "grad_norm": 5.46875,
      "learning_rate": 3.902199258386732e-08,
      "logits/chosen": -1.8437637090682983,
      "logits/rejected": -1.8173859119415283,
      "logps/chosen": -0.27545225620269775,
      "logps/rejected": -0.2872912287712097,
      "loss": 1.0292212963104248,
      "loss/core": 1.0292212963104248,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.603247880935669,
      "rewards/margins": 1.7060693502426147,
      "rewards/rejected": 1.8971784114837646,
      "step": 755
    },
    {
      "epoch": 0.7261435566702497,
      "grad_norm": 6.96875,
      "learning_rate": 3.646088960486862e-08,
      "logits/chosen": -1.6895129680633545,
      "logits/rejected": -1.7431033849716187,
      "logps/chosen": -0.3080654442310333,
      "logps/rejected": -0.3540648818016052,
      "loss": 1.0325762033462524,
      "loss/core": 1.0325762033462524,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.53886079788208,
      "rewards/margins": 1.5185824632644653,
      "rewards/rejected": 2.020278215408325,
      "step": 760
    },
    {
      "epoch": 0.7309208169115012,
      "grad_norm": 8.4375,
      "learning_rate": 3.398008995217988e-08,
      "logits/chosen": -1.964421033859253,
      "logits/rejected": -1.953046441078186,
      "logps/chosen": -0.2779175639152527,
      "logps/rejected": -0.26899924874305725,
      "loss": 1.0525156259536743,
      "loss/core": 1.0525156259536743,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.6339603662490845,
      "rewards/margins": 0.7983501553535461,
      "rewards/rejected": 0.8356102705001831,
      "step": 765
    },
    {
      "epoch": 0.7356980771527529,
      "grad_norm": 108.0,
      "learning_rate": 3.1580526553093315e-08,
      "logits/chosen": -1.877881407737732,
      "logits/rejected": -1.9419214725494385,
      "logps/chosen": -0.28730475902557373,
      "logps/rejected": -0.2802114188671112,
      "loss": 1.0498204231262207,
      "loss/core": 1.0498204231262207,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2819581031799316,
      "rewards/margins": 1.5368332862854004,
      "rewards/rejected": 0.7451248168945312,
      "step": 770
    },
    {
      "epoch": 0.7404753373940045,
      "grad_norm": 5.125,
      "learning_rate": 2.9263101785268252e-08,
      "logits/chosen": -1.890252709388733,
      "logits/rejected": -1.786828637123108,
      "logps/chosen": -0.27267634868621826,
      "logps/rejected": -0.2676258087158203,
      "loss": 1.0456647872924805,
      "loss/core": 1.0456647872924805,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8578557968139648,
      "rewards/margins": 0.4590323567390442,
      "rewards/rejected": 1.3988234996795654,
      "step": 775
    },
    {
      "epoch": 0.7452525976352562,
      "grad_norm": 268.0,
      "learning_rate": 2.7028687137384264e-08,
      "logits/chosen": -1.9074313640594482,
      "logits/rejected": -1.8822380304336548,
      "logps/chosen": -0.2915792465209961,
      "logps/rejected": -0.293667197227478,
      "loss": 1.037200927734375,
      "loss/core": 1.037200927734375,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.0063960552215576,
      "rewards/margins": 1.0704381465911865,
      "rewards/rejected": 1.935957670211792,
      "step": 780
    },
    {
      "epoch": 0.7500298578765078,
      "grad_norm": 9.6875,
      "learning_rate": 2.4878122881409447e-08,
      "logits/chosen": -1.8461768627166748,
      "logits/rejected": -1.8658325672149658,
      "logps/chosen": -0.2820442020893097,
      "logps/rejected": -0.2832142412662506,
      "loss": 1.0385828018188477,
      "loss/core": 1.0385828018188477,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.9002225399017334,
      "rewards/margins": 1.1294481754302979,
      "rewards/rejected": 0.7707743644714355,
      "step": 785
    },
    {
      "epoch": 0.7548071181177595,
      "grad_norm": 36.25,
      "learning_rate": 2.2812217756608937e-08,
      "logits/chosen": -1.825455904006958,
      "logits/rejected": -1.8060061931610107,
      "logps/chosen": -0.2775079011917114,
      "logps/rejected": -0.27117595076560974,
      "loss": 1.0483758449554443,
      "loss/core": 1.0483758449554443,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.307039260864258,
      "rewards/margins": 2.0355687141418457,
      "rewards/rejected": 1.271470546722412,
      "step": 790
    },
    {
      "epoch": 0.7595843783590112,
      "grad_norm": 61.5,
      "learning_rate": 2.0831748665410763e-08,
      "logits/chosen": -1.770493745803833,
      "logits/rejected": -1.7957115173339844,
      "logps/chosen": -0.27306801080703735,
      "logps/rejected": -0.28165358304977417,
      "loss": 1.0311884880065918,
      "loss/core": 1.0311884880065918,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.751323699951172,
      "rewards/margins": 2.812521457672119,
      "rewards/rejected": 0.9388027191162109,
      "step": 795
    },
    {
      "epoch": 0.7643616386002627,
      "grad_norm": 10.5,
      "learning_rate": 1.8937460381244967e-08,
      "logits/chosen": -2.075711727142334,
      "logits/rejected": -2.082383394241333,
      "logps/chosen": -0.2964426577091217,
      "logps/rejected": -0.30977433919906616,
      "loss": 1.0251314640045166,
      "loss/core": 1.0251314640045166,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.5174245834350586,
      "rewards/margins": 1.0785468816757202,
      "rewards/rejected": 1.4388777017593384,
      "step": 800
    },
    {
      "epoch": 0.7691388988415144,
      "grad_norm": 7.65625,
      "learning_rate": 1.713006526846439e-08,
      "logits/chosen": -1.8648141622543335,
      "logits/rejected": -1.8252522945404053,
      "logps/chosen": -0.30121514201164246,
      "logps/rejected": -0.3006502687931061,
      "loss": 1.0409377813339233,
      "loss/core": 1.0409377813339233,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9633139371871948,
      "rewards/margins": 0.3292653262615204,
      "rewards/rejected": 0.634048581123352,
      "step": 805
    },
    {
      "epoch": 0.773916159082766,
      "grad_norm": 28.125,
      "learning_rate": 1.541024301445404e-08,
      "logits/chosen": -1.6336872577667236,
      "logits/rejected": -1.643832802772522,
      "logps/chosen": -0.26690006256103516,
      "logps/rejected": -0.26295873522758484,
      "loss": 1.0444200038909912,
      "loss/core": 1.0444200038909912,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8206228017807007,
      "rewards/margins": 0.1702648401260376,
      "rewards/rejected": 1.6503578424453735,
      "step": 810
    },
    {
      "epoch": 0.7786934193240177,
      "grad_norm": 5.84375,
      "learning_rate": 1.3778640374027983e-08,
      "logits/chosen": -1.751861333847046,
      "logits/rejected": -1.9031435251235962,
      "logps/chosen": -0.3171994984149933,
      "logps/rejected": -0.3077738881111145,
      "loss": 1.0605608224868774,
      "loss/core": 1.0605608224868774,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.4510459899902344,
      "rewards/margins": 1.3974026441574097,
      "rewards/rejected": 1.0536432266235352,
      "step": 815
    },
    {
      "epoch": 0.7834706795652693,
      "grad_norm": 19.0,
      "learning_rate": 1.2235870926211616e-08,
      "logits/chosen": -1.6588127613067627,
      "logits/rejected": -1.6749852895736694,
      "logps/chosen": -0.27939319610595703,
      "logps/rejected": -0.3374519348144531,
      "loss": 1.0130990743637085,
      "loss/core": 1.0130990743637085,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.6268069744110107,
      "rewards/margins": 0.6854991912841797,
      "rewards/rejected": 1.9413076639175415,
      "step": 820
    },
    {
      "epoch": 0.788247939806521,
      "grad_norm": 13.375,
      "learning_rate": 1.0782514843499652e-08,
      "logits/chosen": -1.7541849613189697,
      "logits/rejected": -1.7715661525726318,
      "logps/chosen": -0.29441070556640625,
      "logps/rejected": -0.27596911787986755,
      "loss": 1.0648963451385498,
      "loss/core": 1.0648963451385498,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.5321407318115234,
      "rewards/margins": 1.8472797870635986,
      "rewards/rejected": 1.6848608255386353,
      "step": 825
    },
    {
      "epoch": 0.7930252000477725,
      "grad_norm": 78.5,
      "learning_rate": 9.419118673676924e-09,
      "logits/chosen": -1.563860535621643,
      "logits/rejected": -1.5621966123580933,
      "logps/chosen": -0.3032872676849365,
      "logps/rejected": -0.3065972328186035,
      "loss": 1.0367621183395386,
      "loss/core": 1.0367621183395386,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.807102680206299,
      "rewards/margins": 1.6108064651489258,
      "rewards/rejected": 1.1962958574295044,
      "step": 830
    },
    {
      "epoch": 0.7978024602890242,
      "grad_norm": 6.03125,
      "learning_rate": 8.14619513428405e-09,
      "logits/chosen": -1.904526948928833,
      "logits/rejected": -1.9740642309188843,
      "logps/chosen": -0.29926085472106934,
      "logps/rejected": -0.2885436713695526,
      "loss": 1.054502248764038,
      "loss/core": 1.054502248764038,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6957647800445557,
      "rewards/margins": 0.6506364941596985,
      "rewards/rejected": 1.045128345489502,
      "step": 835
    },
    {
      "epoch": 0.8025797205302759,
      "grad_norm": 9.9375,
      "learning_rate": 6.96422291980539e-09,
      "logits/chosen": -1.8588794469833374,
      "logits/rejected": -1.90227472782135,
      "logps/chosen": -0.2894556224346161,
      "logps/rejected": -0.3027735650539398,
      "loss": 1.02797532081604,
      "loss/core": 1.02797532081604,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.031310796737671,
      "rewards/margins": 0.43263769149780273,
      "rewards/rejected": 1.5986733436584473,
      "step": 840
    },
    {
      "epoch": 0.8073569807715275,
      "grad_norm": 17.25,
      "learning_rate": 5.8736465216517594e-09,
      "logits/chosen": -1.9031187295913696,
      "logits/rejected": -1.9344947338104248,
      "logps/chosen": -0.2742392420768738,
      "logps/rejected": -0.2774720788002014,
      "loss": 1.0499933958053589,
      "loss/core": 1.0499933958053589,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.593908786773682,
      "rewards/margins": 2.4118456840515137,
      "rewards/rejected": 2.1820626258850098,
      "step": 845
    },
    {
      "epoch": 0.8121342410127792,
      "grad_norm": 4.09375,
      "learning_rate": 4.874876061005173e-09,
      "logits/chosen": -1.8513599634170532,
      "logits/rejected": -1.8989410400390625,
      "logps/chosen": -0.2543167173862457,
      "logps/rejected": -0.25180160999298096,
      "loss": 1.0425068140029907,
      "loss/core": 1.0425068140029907,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7452361583709717,
      "rewards/margins": 1.0755412578582764,
      "rewards/rejected": 0.6696950197219849,
      "step": 850
    },
    {
      "epoch": 0.8169115012540308,
      "grad_norm": 25.875,
      "learning_rate": 3.968287134589188e-09,
      "logits/chosen": -1.7955166101455688,
      "logits/rejected": -1.8299312591552734,
      "logps/chosen": -0.2676030695438385,
      "logps/rejected": -0.2964601516723633,
      "loss": 1.004606008529663,
      "loss/core": 1.004606008529663,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.3115034103393555,
      "rewards/margins": 0.41639384627342224,
      "rewards/rejected": 1.8951095342636108,
      "step": 855
    },
    {
      "epoch": 0.8216887614952825,
      "grad_norm": 5.375,
      "learning_rate": 3.154220673422192e-09,
      "logits/chosen": -1.8242244720458984,
      "logits/rejected": -1.871952772140503,
      "logps/chosen": -0.29963549971580505,
      "logps/rejected": -0.3072584271430969,
      "loss": 1.0296491384506226,
      "loss/core": 1.0296491384506226,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9225000143051147,
      "rewards/margins": 0.8448880910873413,
      "rewards/rejected": 1.0776121616363525,
      "step": 860
    },
    {
      "epoch": 0.826466021736534,
      "grad_norm": 7.28125,
      "learning_rate": 2.4329828146074096e-09,
      "logits/chosen": -1.9124653339385986,
      "logits/rejected": -2.0408012866973877,
      "logps/chosen": -0.29619625210762024,
      "logps/rejected": -0.2827211022377014,
      "loss": 1.0616817474365234,
      "loss/core": 1.0616817474365234,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.6343491077423096,
      "rewards/margins": 1.424903392791748,
      "rewards/rejected": 1.2094457149505615,
      "step": 865
    },
    {
      "epoch": 0.8312432819777857,
      "grad_norm": 19.5,
      "learning_rate": 1.8048447862070714e-09,
      "logits/chosen": -1.5945602655410767,
      "logits/rejected": -1.6320714950561523,
      "logps/chosen": -0.2955728769302368,
      "logps/rejected": -0.2850645184516907,
      "loss": 1.061391830444336,
      "loss/core": 1.061391830444336,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.463152885437012,
      "rewards/margins": 2.405036449432373,
      "rewards/rejected": 2.0581166744232178,
      "step": 870
    },
    {
      "epoch": 0.8360205422190374,
      "grad_norm": 4.5,
      "learning_rate": 1.2700428052447033e-09,
      "logits/chosen": -1.7374908924102783,
      "logits/rejected": -1.7853012084960938,
      "logps/chosen": -0.3029218316078186,
      "logps/rejected": -0.3070649802684784,
      "loss": 1.0413432121276855,
      "loss/core": 1.0413432121276855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9537794589996338,
      "rewards/margins": 0.5955628156661987,
      "rewards/rejected": 1.358216643333435,
      "step": 875
    },
    {
      "epoch": 0.840797802460289,
      "grad_norm": 3.453125,
      "learning_rate": 8.287779888734858e-10,
      "logits/chosen": -1.7615150213241577,
      "logits/rejected": -1.707476019859314,
      "logps/chosen": -0.28448665142059326,
      "logps/rejected": -0.3107626140117645,
      "loss": 1.0223973989486694,
      "loss/core": 1.0223973989486694,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.715723752975464,
      "rewards/margins": 2.2797560691833496,
      "rewards/rejected": 1.4359674453735352,
      "step": 880
    },
    {
      "epoch": 0.8455750627015407,
      "grad_norm": 18.0,
      "learning_rate": 4.812162787445062e-10,
      "logits/chosen": -1.8650376796722412,
      "logits/rejected": -1.8496296405792236,
      "logps/chosen": -0.2981775403022766,
      "logps/rejected": -0.30601340532302856,
      "loss": 1.029161810874939,
      "loss/core": 1.029161810874939,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5521029233932495,
      "rewards/margins": 0.7568989992141724,
      "rewards/rejected": 0.7952039241790771,
      "step": 885
    },
    {
      "epoch": 0.8503523229427923,
      "grad_norm": 11.4375,
      "learning_rate": 2.2748837860270265e-10,
      "logits/chosen": -1.8946914672851562,
      "logits/rejected": -1.9093029499053955,
      "logps/chosen": -0.2697398066520691,
      "logps/rejected": -0.2905231714248657,
      "loss": 1.019055962562561,
      "loss/core": 1.019055962562561,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8530833721160889,
      "rewards/margins": 0.24989423155784607,
      "rewards/rejected": 1.6031888723373413,
      "step": 890
    },
    {
      "epoch": 0.855129583184044,
      "grad_norm": 65.0,
      "learning_rate": 6.768970513457151e-11,
      "logits/chosen": -1.7693462371826172,
      "logits/rejected": -1.7002862691879272,
      "logps/chosen": -0.2899450659751892,
      "logps/rejected": -0.2910805940628052,
      "loss": 1.038063645362854,
      "loss/core": 1.038063645362854,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.1362929344177246,
      "rewards/margins": 1.1503578424453735,
      "rewards/rejected": 0.9859352111816406,
      "step": 895
    },
    {
      "epoch": 0.8599068434252956,
      "grad_norm": 34.0,
      "learning_rate": 1.8803520859811406e-12,
      "logits/chosen": -1.7765915393829346,
      "logits/rejected": -1.7241567373275757,
      "logps/chosen": -0.2680931091308594,
      "logps/rejected": -0.2755570411682129,
      "loss": 1.0350791215896606,
      "loss/core": 1.0350791215896606,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9140121936798096,
      "rewards/margins": 1.5131105184555054,
      "rewards/rejected": 1.4009015560150146,
      "step": 900
    },
    {
      "epoch": 0.8599068434252956,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 1.0422146828969319,
      "train_runtime": 8158.2235,
      "train_samples_per_second": 1.765,
      "train_steps_per_second": 0.11
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}