{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8599068434252956,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004777260241251642,
      "grad_norm": 24.0,
      "learning_rate": 2.222222222222222e-08,
      "logits/chosen": -1.997597098350525,
      "logits/rejected": -2.0135111808776855,
      "logps/chosen": -0.2795145511627197,
      "logps/rejected": -0.28208082914352417,
      "loss": 0.38181036710739136,
      "loss/core": 0.38181036710739136,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.418922185897827,
      "rewards/margins": 1.4596340656280518,
      "rewards/rejected": 0.9592880010604858,
      "step": 5
    },
    {
      "epoch": 0.009554520482503284,
      "grad_norm": 190.0,
      "learning_rate": 5e-08,
      "logits/chosen": -2.3454456329345703,
      "logits/rejected": -2.392616033554077,
      "logps/chosen": -0.2907744348049164,
      "logps/rejected": -0.28635361790657043,
      "loss": 1.0456370115280151,
      "loss/core": 1.0456370115280151,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6759283542633057,
      "rewards/margins": 0.041599858552217484,
      "rewards/rejected": 1.6343282461166382,
      "step": 10
    },
    {
      "epoch": 0.014331780723754926,
      "grad_norm": 81.5,
      "learning_rate": 7.777777777777778e-08,
      "logits/chosen": -2.2278778553009033,
      "logits/rejected": -2.1911208629608154,
      "logps/chosen": -0.2769435942173004,
      "logps/rejected": -0.28479596972465515,
      "loss": 0.3574354350566864,
      "loss/core": 0.3574354350566864,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.138868570327759,
      "rewards/margins": 1.00614595413208,
      "rewards/rejected": 1.1327224969863892,
      "step": 15
    },
    {
      "epoch": 0.01910904096500657,
      "grad_norm": 21.75,
      "learning_rate": 1.0555555555555555e-07,
      "logits/chosen": -2.1759560108184814,
      "logits/rejected": -2.218474864959717,
      "logps/chosen": -0.300750195980072,
      "logps/rejected": -0.2876606285572052,
      "loss": 0.4318433701992035,
      "loss/core": 0.4318433701992035,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.921849012374878,
      "rewards/margins": 1.0531971454620361,
      "rewards/rejected": 0.8686518669128418,
      "step": 20
    },
    {
      "epoch": 0.02388630120625821,
      "grad_norm": 25.5,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -2.0754263401031494,
      "logits/rejected": -2.136157512664795,
      "logps/chosen": -0.27347683906555176,
      "logps/rejected": -0.2596595585346222,
      "loss": 1.702324628829956,
      "loss/core": 1.702324628829956,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.820397138595581,
      "rewards/margins": 2.1924726963043213,
      "rewards/rejected": 1.6279243230819702,
      "step": 25
    },
    {
      "epoch": 0.028663561447509853,
      "grad_norm": 274.0,
      "learning_rate": 1.611111111111111e-07,
      "logits/chosen": -2.3495566844940186,
      "logits/rejected": -2.331704616546631,
      "logps/chosen": -0.30199751257896423,
      "logps/rejected": -0.3243906497955322,
      "loss": 0.6514055132865906,
      "loss/core": 0.6514055132865906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.259319543838501,
      "rewards/margins": 1.2338563203811646,
      "rewards/rejected": 1.0254631042480469,
      "step": 30
    },
    {
      "epoch": 0.033440821688761495,
      "grad_norm": 2.265625,
      "learning_rate": 1.8888888888888888e-07,
      "logits/chosen": -2.204205274581909,
      "logits/rejected": -2.1856656074523926,
      "logps/chosen": -0.2655085027217865,
      "logps/rejected": -0.26616171002388,
      "loss": 0.814064621925354,
      "loss/core": 0.814064621925354,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.6806728839874268,
      "rewards/margins": 1.5299947261810303,
      "rewards/rejected": 1.1506781578063965,
      "step": 35
    },
    {
      "epoch": 0.03821808193001314,
      "grad_norm": 80.0,
      "learning_rate": 2.1666666666666667e-07,
      "logits/chosen": -2.0009946823120117,
      "logits/rejected": -2.060720682144165,
      "logps/chosen": -0.2918882966041565,
      "logps/rejected": -0.28243204951286316,
      "loss": 1.9676904678344727,
      "loss/core": 1.9676904678344727,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.485463619232178,
      "rewards/margins": 2.8883402347564697,
      "rewards/rejected": 1.5971235036849976,
      "step": 40
    },
    {
      "epoch": 0.04299534217126478,
      "grad_norm": 8.125,
      "learning_rate": 2.4444444444444445e-07,
      "logits/chosen": -2.197145938873291,
      "logits/rejected": -2.359992504119873,
      "logps/chosen": -0.28900036215782166,
      "logps/rejected": -0.28796130418777466,
      "loss": 0.2774832248687744,
      "loss/core": 0.2774832248687744,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0935540199279785,
      "rewards/margins": 1.2470275163650513,
      "rewards/rejected": 0.8465266227722168,
      "step": 45
    },
    {
      "epoch": 0.04777260241251642,
      "grad_norm": 77.0,
      "learning_rate": 2.7222222222222216e-07,
      "logits/chosen": -2.166928768157959,
      "logits/rejected": -2.1634888648986816,
      "logps/chosen": -0.28184059262275696,
      "logps/rejected": -0.28433579206466675,
      "loss": 0.7104421257972717,
      "loss/core": 0.7104421257972717,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.8780951499938965,
      "rewards/margins": 1.4217535257339478,
      "rewards/rejected": 1.4563415050506592,
      "step": 50
    },
    {
      "epoch": 0.05254986265376806,
      "grad_norm": 0.87109375,
      "learning_rate": 3e-07,
      "logits/chosen": -2.197507858276367,
      "logits/rejected": -2.185612916946411,
      "logps/chosen": -0.2892429828643799,
      "logps/rejected": -0.2890060245990753,
      "loss": 1.496902346611023,
      "loss/core": 1.496902346611023,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.2559685707092285,
      "rewards/margins": 0.21959257125854492,
      "rewards/rejected": 2.0363757610321045,
      "step": 55
    },
    {
      "epoch": 0.057327122895019705,
      "grad_norm": 520.0,
      "learning_rate": 3.2777777777777776e-07,
      "logits/chosen": -2.09147572517395,
      "logits/rejected": -2.1203830242156982,
      "logps/chosen": -0.2743294835090637,
      "logps/rejected": -0.28498584032058716,
      "loss": 0.337458074092865,
      "loss/core": 0.337458074092865,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.1575465202331543,
      "rewards/margins": 0.6136163473129272,
      "rewards/rejected": 1.543930172920227,
      "step": 60
    },
    {
      "epoch": 0.06210438313627135,
      "grad_norm": 7.1875,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -1.9565308094024658,
      "logits/rejected": -2.059638023376465,
      "logps/chosen": -0.3096534311771393,
      "logps/rejected": -0.30584970116615295,
      "loss": 0.9800235629081726,
      "loss/core": 0.9800235629081726,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.9653208255767822,
      "rewards/margins": 1.7889091968536377,
      "rewards/rejected": 1.176411747932434,
      "step": 65
    },
    {
      "epoch": 0.06688164337752299,
      "grad_norm": 54.25,
      "learning_rate": 3.8333333333333335e-07,
      "logits/chosen": -1.9700387716293335,
      "logits/rejected": -1.955226182937622,
      "logps/chosen": -0.3858351707458496,
      "logps/rejected": -0.2979809641838074,
      "loss": 1.823669195175171,
      "loss/core": 1.823669195175171,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.578740358352661,
      "rewards/margins": 1.4053298234939575,
      "rewards/rejected": 2.1734108924865723,
      "step": 70
    },
    {
      "epoch": 0.07165890361877464,
      "grad_norm": 6.875,
      "learning_rate": 4.1111111111111107e-07,
      "logits/chosen": -2.0864429473876953,
      "logits/rejected": -2.051452398300171,
      "logps/chosen": -0.3145933449268341,
      "logps/rejected": -0.2891804873943329,
      "loss": 0.7372748255729675,
      "loss/core": 0.7372748255729675,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.610942840576172,
      "rewards/margins": 1.0181684494018555,
      "rewards/rejected": 1.5927742719650269,
      "step": 75
    },
    {
      "epoch": 0.07643616386002627,
      "grad_norm": 4.15625,
      "learning_rate": 4.3888888888888884e-07,
      "logits/chosen": -2.066887855529785,
      "logits/rejected": -2.186957836151123,
      "logps/chosen": -0.26004475355148315,
      "logps/rejected": -0.27356016635894775,
      "loss": 0.9191620945930481,
      "loss/core": 0.9191620945930481,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.2465484142303467,
      "rewards/margins": 1.9899933338165283,
      "rewards/rejected": 1.2565548419952393,
      "step": 80
    },
    {
      "epoch": 0.08121342410127792,
      "grad_norm": 5.75,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -2.1885199546813965,
      "logits/rejected": -2.2368600368499756,
      "logps/chosen": -0.30883267521858215,
      "logps/rejected": -0.2775411307811737,
      "loss": 1.6861064434051514,
      "loss/core": 1.6861064434051514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.484980821609497,
      "rewards/margins": 2.152528762817383,
      "rewards/rejected": 1.3324521780014038,
      "step": 85
    },
    {
      "epoch": 0.08599068434252956,
      "grad_norm": 0.98828125,
      "learning_rate": 4.944444444444445e-07,
      "logits/chosen": -2.4196181297302246,
      "logits/rejected": -2.4065661430358887,
      "logps/chosen": -0.281704306602478,
      "logps/rejected": -0.2961405813694,
      "loss": 0.30765506625175476,
      "loss/core": 0.30765506625175476,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7503442764282227,
      "rewards/margins": 0.9065144658088684,
      "rewards/rejected": 0.8438299894332886,
      "step": 90
    },
    {
      "epoch": 0.09076794458378121,
      "grad_norm": 62.5,
      "learning_rate": 4.999699149323369e-07,
      "logits/chosen": -2.0543789863586426,
      "logits/rejected": -2.11065673828125,
      "logps/chosen": -0.26749688386917114,
      "logps/rejected": -0.2789869010448456,
      "loss": 0.906001091003418,
      "loss/core": 0.906001091003418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.2087581157684326,
      "rewards/margins": 1.1049243211746216,
      "rewards/rejected": 2.1038336753845215,
      "step": 95
    },
    {
      "epoch": 0.09554520482503284,
      "grad_norm": 10.625,
      "learning_rate": 4.998477067547739e-07,
      "logits/chosen": -2.231830596923828,
      "logits/rejected": -2.2774977684020996,
      "logps/chosen": -0.25898346304893494,
      "logps/rejected": -0.24633264541625977,
      "loss": 0.3568304777145386,
      "loss/core": 0.3568304777145386,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.444751024246216,
      "rewards/margins": 1.1550977230072021,
      "rewards/rejected": 1.2896530628204346,
      "step": 100
    },
    {
      "epoch": 0.10032246506628449,
      "grad_norm": 298.0,
      "learning_rate": 4.996315410727229e-07,
      "logits/chosen": -2.3348228931427,
      "logits/rejected": -2.4207358360290527,
      "logps/chosen": -0.2856779098510742,
      "logps/rejected": -0.279996782541275,
      "loss": 0.3823498785495758,
      "loss/core": 0.3823498785495758,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.9824638366699219,
      "rewards/margins": 1.1864570379257202,
      "rewards/rejected": 0.7960068583488464,
      "step": 105
    },
    {
      "epoch": 0.10509972530753613,
      "grad_norm": 11.875,
      "learning_rate": 4.993214991772562e-07,
      "logits/chosen": -1.9041261672973633,
      "logits/rejected": -1.8569024801254272,
      "logps/chosen": -0.3389981985092163,
      "logps/rejected": -0.29700320959091187,
      "loss": 1.7510207891464233,
      "loss/core": 1.7510207891464233,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.465295314788818,
      "rewards/margins": 2.569420576095581,
      "rewards/rejected": 1.8958752155303955,
      "step": 110
    },
    {
      "epoch": 0.10987698554878778,
      "grad_norm": 9.875,
      "learning_rate": 4.989176976624511e-07,
      "logits/chosen": -2.0836009979248047,
      "logits/rejected": -2.027132749557495,
      "logps/chosen": -0.2996644973754883,
      "logps/rejected": -0.30102577805519104,
      "loss": 0.4198575019836426,
      "loss/core": 0.4198575019836426,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7827256917953491,
      "rewards/margins": 0.3579304814338684,
      "rewards/rejected": 1.424795389175415,
      "step": 115
    },
    {
      "epoch": 0.11465424579003941,
      "grad_norm": 184.0,
      "learning_rate": 4.984202883815428e-07,
      "logits/chosen": -2.0339629650115967,
      "logits/rejected": -2.0718021392822266,
      "logps/chosen": -0.30192404985427856,
      "logps/rejected": -0.29958024621009827,
      "loss": 0.16679374873638153,
      "loss/core": 0.16679374873638153,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8123009204864502,
      "rewards/margins": 0.7246686220169067,
      "rewards/rejected": 1.0876322984695435,
      "step": 120
    },
    {
      "epoch": 0.11943150603129106,
      "grad_norm": 5.5625,
      "learning_rate": 4.978294583898195e-07,
      "logits/chosen": -2.104386806488037,
      "logits/rejected": -2.1359241008758545,
      "logps/chosen": -0.2988458275794983,
      "logps/rejected": -0.3063589632511139,
      "loss": 0.2692359387874603,
      "loss/core": 0.2692359387874603,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.067023992538452,
      "rewards/margins": 1.047250747680664,
      "rewards/rejected": 1.019773244857788,
      "step": 125
    },
    {
      "epoch": 0.1242087662725427,
      "grad_norm": 10.6875,
      "learning_rate": 4.971454298742779e-07,
      "logits/chosen": -1.994943380355835,
      "logits/rejected": -2.0879642963409424,
      "logps/chosen": -0.2800437808036804,
      "logps/rejected": -0.279203325510025,
      "loss": 0.14723244309425354,
      "loss/core": 0.14723244309425354,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.806762456893921,
      "rewards/margins": 0.9239923357963562,
      "rewards/rejected": 0.8827699422836304,
      "step": 130
    },
    {
      "epoch": 0.12898602651379434,
      "grad_norm": 21.75,
      "learning_rate": 4.963684600700678e-07,
      "logits/chosen": -2.0894951820373535,
      "logits/rejected": -2.16270112991333,
      "logps/chosen": -0.3019745349884033,
      "logps/rejected": -0.30207985639572144,
      "loss": 0.8815671801567078,
      "loss/core": 0.8815671801567078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.3753859996795654,
      "rewards/margins": 2.099947214126587,
      "rewards/rejected": 1.2754385471343994,
      "step": 135
    },
    {
      "epoch": 0.13376328675504598,
      "grad_norm": 11.1875,
      "learning_rate": 4.954988411637571e-07,
      "logits/chosen": -2.44862699508667,
      "logits/rejected": -2.508138656616211,
      "logps/chosen": -0.28244370222091675,
      "logps/rejected": -0.2900107800960541,
      "loss": 0.1148979440331459,
      "loss/core": 0.1148979440331459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6288248300552368,
      "rewards/margins": 0.9194561243057251,
      "rewards/rejected": 0.7093685865402222,
      "step": 140
    },
    {
      "epoch": 0.13854054699629761,
      "grad_norm": 7.9375,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -1.7729772329330444,
      "logits/rejected": -1.803558111190796,
      "logps/chosen": -0.2981434464454651,
      "logps/rejected": -0.30744919180870056,
      "loss": 2.0826029777526855,
      "loss/core": 2.0826029777526855,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.636133909225464,
      "rewards/margins": 1.9079220294952393,
      "rewards/rejected": 1.7282119989395142,
      "step": 145
    },
    {
      "epoch": 0.14331780723754928,
      "grad_norm": 111.0,
      "learning_rate": 4.93482998875813e-07,
      "logits/chosen": -2.2263197898864746,
      "logits/rejected": -2.274482488632202,
      "logps/chosen": -0.31686437129974365,
      "logps/rejected": -0.3086201250553131,
      "loss": 1.6554409265518188,
      "loss/core": 1.6554409265518188,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.644029140472412,
      "rewards/margins": 2.1906416416168213,
      "rewards/rejected": 1.4533871412277222,
      "step": 150
    },
    {
      "epoch": 0.1480950674788009,
      "grad_norm": 4.875,
      "learning_rate": 4.923375335700223e-07,
      "logits/chosen": -2.094820022583008,
      "logits/rejected": -2.17297101020813,
      "logps/chosen": -0.2831822335720062,
      "logps/rejected": -0.2986108958721161,
      "loss": 0.7216386198997498,
      "loss/core": 0.7216386198997498,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.8636791706085205,
      "rewards/margins": 2.1000559329986572,
      "rewards/rejected": 0.7636229395866394,
      "step": 155
    },
    {
      "epoch": 0.15287232772005255,
      "grad_norm": 272.0,
      "learning_rate": 4.911009350287347e-07,
      "logits/chosen": -2.0500402450561523,
      "logits/rejected": -2.0241525173187256,
      "logps/chosen": -0.3109878599643707,
      "logps/rejected": -0.3158418536186218,
      "loss": 1.3895409107208252,
      "loss/core": 1.3895409107208252,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.1455612182617188,
      "rewards/margins": 1.7495533227920532,
      "rewards/rejected": 1.396007776260376,
      "step": 160
    },
    {
      "epoch": 0.15764958796130418,
      "grad_norm": 5.09375,
      "learning_rate": 4.897736682860885e-07,
      "logits/chosen": -2.216140031814575,
      "logits/rejected": -2.146721363067627,
      "logps/chosen": -0.31132304668426514,
      "logps/rejected": -0.30651506781578064,
      "loss": 0.1788981854915619,
      "loss/core": 0.1788981854915619,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.6000747680664062,
      "rewards/margins": 0.8192144632339478,
      "rewards/rejected": 0.7808603048324585,
      "step": 165
    },
    {
      "epoch": 0.16242684820255585,
      "grad_norm": 402.0,
      "learning_rate": 4.883562324728241e-07,
      "logits/chosen": -2.18127179145813,
      "logits/rejected": -2.126049518585205,
      "logps/chosen": -0.2751903235912323,
      "logps/rejected": -0.261197030544281,
      "loss": 1.8691749572753906,
      "loss/core": 1.8691749572753906,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.59234881401062,
      "rewards/margins": -0.16324080526828766,
      "rewards/rejected": 2.755589485168457,
      "step": 170
    },
    {
      "epoch": 0.16720410844380748,
      "grad_norm": 460.0,
      "learning_rate": 4.868491606285823e-07,
      "logits/chosen": -2.1704468727111816,
      "logits/rejected": -2.2109594345092773,
      "logps/chosen": -0.2882474958896637,
      "logps/rejected": -0.3017005920410156,
      "loss": 1.1066371202468872,
      "loss/core": 1.1066371202468872,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.569884777069092,
      "rewards/margins": 1.3101545572280884,
      "rewards/rejected": 1.259730339050293,
      "step": 175
    },
    {
      "epoch": 0.17198136868505912,
      "grad_norm": 640.0,
      "learning_rate": 4.852530195014489e-07,
      "logits/chosen": -2.193326711654663,
      "logits/rejected": -2.104137897491455,
      "logps/chosen": -0.2730237543582916,
      "logps/rejected": -0.26204752922058105,
      "loss": 1.1543080806732178,
      "loss/core": 1.1543080806732178,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.2240982055664062,
      "rewards/margins": 1.7351115942001343,
      "rewards/rejected": 1.4889863729476929,
      "step": 180
    },
    {
      "epoch": 0.17675862892631075,
      "grad_norm": 3.40625,
      "learning_rate": 4.835684093348244e-07,
      "logits/chosen": -2.1960487365722656,
      "logits/rejected": -2.154365062713623,
      "logps/chosen": -0.2888965308666229,
      "logps/rejected": -0.29239413142204285,
      "loss": 0.06165605038404465,
      "loss/core": 0.06165605038404465,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.0407021045684814,
      "rewards/margins": 0.41684430837631226,
      "rewards/rejected": 0.6238579154014587,
      "step": 185
    },
    {
      "epoch": 0.18153588916756241,
      "grad_norm": 28.875,
      "learning_rate": 4.817959636416969e-07,
      "logits/chosen": -2.2871928215026855,
      "logits/rejected": -2.264662981033325,
      "logps/chosen": -0.28313952684402466,
      "logps/rejected": -0.28482240438461304,
      "loss": 1.1405375003814697,
      "loss/core": 1.1405375003814697,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.842782497406006,
      "rewards/margins": 1.609337568283081,
      "rewards/rejected": 1.2334448099136353,
      "step": 190
    },
    {
      "epoch": 0.18631314940881405,
      "grad_norm": 23.5,
      "learning_rate": 4.799363489664039e-07,
      "logits/chosen": -1.9776523113250732,
      "logits/rejected": -1.999280571937561,
      "logps/chosen": -0.289995938539505,
      "logps/rejected": -0.2919548749923706,
      "loss": 1.1198909282684326,
      "loss/core": 1.1198909282684326,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.678499937057495,
      "rewards/margins": 1.7345187664031982,
      "rewards/rejected": 0.9439811706542969,
      "step": 195
    },
    {
      "epoch": 0.19109040965006568,
      "grad_norm": 20.625,
      "learning_rate": 4.779902646339721e-07,
      "logits/chosen": -1.88909113407135,
      "logits/rejected": -1.9297901391983032,
      "logps/chosen": -0.32461681962013245,
      "logps/rejected": -0.3156924545764923,
      "loss": 0.4924043118953705,
      "loss/core": 0.4924043118953705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.604013681411743,
      "rewards/margins": 1.374180555343628,
      "rewards/rejected": 1.2298332452774048,
      "step": 200
    },
    {
      "epoch": 0.19586766989131732,
      "grad_norm": 9.875,
      "learning_rate": 4.759584424871301e-07,
      "logits/chosen": -2.110816478729248,
      "logits/rejected": -2.1626930236816406,
      "logps/chosen": -0.25076472759246826,
      "logps/rejected": -0.2683349549770355,
      "loss": 1.2883168458938599,
      "loss/core": 1.2883168458938599,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.282876491546631,
      "rewards/margins": 1.168548345565796,
      "rewards/rejected": 2.114327907562256,
      "step": 205
    },
    {
      "epoch": 0.20064493013256898,
      "grad_norm": 27.875,
      "learning_rate": 4.738416466110917e-07,
      "logits/chosen": -2.1067051887512207,
      "logits/rejected": -2.1317334175109863,
      "logps/chosen": -0.3131873607635498,
      "logps/rejected": -0.32463163137435913,
      "loss": 1.1303998231887817,
      "loss/core": 1.1303998231887817,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.8242244720458984,
      "rewards/margins": 1.3681774139404297,
      "rewards/rejected": 1.4560470581054688,
      "step": 210
    },
    {
      "epoch": 0.20542219037382062,
      "grad_norm": 6.875,
      "learning_rate": 4.716406730462153e-07,
      "logits/chosen": -2.079124927520752,
      "logits/rejected": -2.069756031036377,
      "logps/chosen": -0.2739131450653076,
      "logps/rejected": -0.2875369191169739,
      "loss": 1.5732624530792236,
      "loss/core": 1.5732624530792236,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.207057237625122,
      "rewards/margins": 1.5373919010162354,
      "rewards/rejected": 1.6696653366088867,
      "step": 215
    },
    {
      "epoch": 0.21019945061507225,
      "grad_norm": 30.375,
      "learning_rate": 4.693563494886454e-07,
      "logits/chosen": -2.0912394523620605,
      "logits/rejected": -2.0069117546081543,
      "logps/chosen": -0.3027765452861786,
      "logps/rejected": -0.30113840103149414,
      "loss": 0.21921542286872864,
      "loss/core": 0.21921542286872864,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8828872442245483,
      "rewards/margins": 0.8069303631782532,
      "rewards/rejected": 1.0759570598602295,
      "step": 220
    },
    {
      "epoch": 0.2149767108563239,
      "grad_norm": 8.25,
      "learning_rate": 4.6698953497905016e-07,
      "logits/chosen": -2.1752381324768066,
      "logits/rejected": -2.1640679836273193,
      "logps/chosen": -0.2903914749622345,
      "logps/rejected": -0.2762240171432495,
      "loss": 0.8437272310256958,
      "loss/core": 0.8437272310256958,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.6241302490234375,
      "rewards/margins": 2.222364902496338,
      "rewards/rejected": 1.4017655849456787,
      "step": 225
    },
    {
      "epoch": 0.21975397109757555,
      "grad_norm": 54.0,
      "learning_rate": 4.645411195795709e-07,
      "logits/chosen": -2.266545057296753,
      "logits/rejected": -2.2732253074645996,
      "logps/chosen": -0.2827457785606384,
      "logps/rejected": -0.31045886874198914,
      "loss": 0.1818530261516571,
      "loss/core": 0.1818530261516571,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8441836833953857,
      "rewards/margins": 1.116914987564087,
      "rewards/rejected": 0.7272688150405884,
      "step": 230
    },
    {
      "epoch": 0.2245312313388272,
      "grad_norm": 21.0,
      "learning_rate": 4.6201202403910643e-07,
      "logits/chosen": -2.0017685890197754,
      "logits/rejected": -2.0373404026031494,
      "logps/chosen": -0.29275330901145935,
      "logps/rejected": -0.2876564562320709,
      "loss": 0.45062655210494995,
      "loss/core": 0.45062655210494995,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.9036191701889038,
      "rewards/margins": 0.5858906507492065,
      "rewards/rejected": 1.3177284002304077,
      "step": 235
    },
    {
      "epoch": 0.22930849158007882,
      "grad_norm": 11.125,
      "learning_rate": 4.594031994470573e-07,
      "logits/chosen": -2.016340732574463,
      "logits/rejected": -2.035101890563965,
      "logps/chosen": -0.29441291093826294,
      "logps/rejected": -0.29516950249671936,
      "loss": 0.14202021062374115,
      "loss/core": 0.14202021062374115,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7591701745986938,
      "rewards/margins": 0.7430254220962524,
      "rewards/rejected": 1.0161445140838623,
      "step": 240
    },
    {
      "epoch": 0.23408575182133046,
      "grad_norm": 300.0,
      "learning_rate": 4.567156268756593e-07,
      "logits/chosen": -2.0561702251434326,
      "logits/rejected": -2.043266773223877,
      "logps/chosen": -0.2777637839317322,
      "logps/rejected": -0.2947208285331726,
      "loss": 1.193971037864685,
      "loss/core": 1.193971037864685,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.6019701957702637,
      "rewards/margins": 2.486217975616455,
      "rewards/rejected": 1.1157519817352295,
      "step": 245
    },
    {
      "epoch": 0.23886301206258212,
      "grad_norm": 21.5,
      "learning_rate": 4.5395031701104303e-07,
      "logits/chosen": -1.8958946466445923,
      "logits/rejected": -1.9753758907318115,
      "logps/chosen": -0.30067577958106995,
      "logps/rejected": -0.295326292514801,
      "loss": 0.28905177116394043,
      "loss/core": 0.28905177116394043,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.3306000232696533,
      "rewards/margins": 1.2696471214294434,
      "rewards/rejected": 1.0609527826309204,
      "step": 250
    },
    {
      "epoch": 0.24364027230383375,
      "grad_norm": 12.875,
      "learning_rate": 4.511083097731555e-07,
      "logits/chosen": -2.2028048038482666,
      "logits/rejected": -2.1979146003723145,
      "logps/chosen": -0.2854974865913391,
      "logps/rejected": -0.2871183156967163,
      "loss": 0.6217617392539978,
      "loss/core": 0.6217617392539978,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4457859992980957,
      "rewards/margins": 0.8392683863639832,
      "rewards/rejected": 1.6065175533294678,
      "step": 255
    },
    {
      "epoch": 0.2484175325450854,
      "grad_norm": 0.78515625,
      "learning_rate": 4.481906739246894e-07,
      "logits/chosen": -2.1431796550750732,
      "logits/rejected": -2.214958906173706,
      "logps/chosen": -0.28957638144493103,
      "logps/rejected": -0.27189141511917114,
      "loss": 0.5298699736595154,
      "loss/core": 0.5298699736595154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3812990188598633,
      "rewards/margins": 0.8128549456596375,
      "rewards/rejected": 1.568444013595581,
      "step": 260
    },
    {
      "epoch": 0.25319479278633705,
      "grad_norm": 1.1796875,
      "learning_rate": 4.451985066691648e-07,
      "logits/chosen": -2.025351047515869,
      "logits/rejected": -2.0318732261657715,
      "logps/chosen": -0.3007538318634033,
      "logps/rejected": -0.305298388004303,
      "loss": 0.405862033367157,
      "loss/core": 0.405862033367157,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.2176848649978638,
      "rewards/margins": 0.080999456346035,
      "rewards/rejected": 1.1366853713989258,
      "step": 265
    },
    {
      "epoch": 0.2579720530275887,
      "grad_norm": 12.4375,
      "learning_rate": 4.421329332383158e-07,
      "logits/chosen": -1.928206205368042,
      "logits/rejected": -2.016925096511841,
      "logps/chosen": -0.2899303734302521,
      "logps/rejected": -0.29195359349250793,
      "loss": 1.1587004661560059,
      "loss/core": 1.1587004661560059,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.2669448852539062,
      "rewards/margins": 1.1529427766799927,
      "rewards/rejected": 2.114002227783203,
      "step": 270
    },
    {
      "epoch": 0.2627493132688403,
      "grad_norm": 68.0,
      "learning_rate": 4.3899510646893696e-07,
      "logits/chosen": -2.078123092651367,
      "logits/rejected": -2.032747745513916,
      "logps/chosen": -0.2979772388935089,
      "logps/rejected": -0.3040463328361511,
      "loss": 0.45543140172958374,
      "loss/core": 0.45543140172958374,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.310035467147827,
      "rewards/margins": 1.1866633892059326,
      "rewards/rejected": 1.123372197151184,
      "step": 275
    },
    {
      "epoch": 0.26752657351009196,
      "grad_norm": 4.03125,
      "learning_rate": 4.357862063693485e-07,
      "logits/chosen": -2.0485012531280518,
      "logits/rejected": -1.9654176235198975,
      "logps/chosen": -0.28849276900291443,
      "logps/rejected": -0.3105738162994385,
      "loss": 0.21645864844322205,
      "loss/core": 0.21645864844322205,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0423591136932373,
      "rewards/margins": 1.3129526376724243,
      "rewards/rejected": 0.7294065952301025,
      "step": 280
    },
    {
      "epoch": 0.2723038337513436,
      "grad_norm": 244.0,
      "learning_rate": 4.3250743967564364e-07,
      "logits/chosen": -2.072444438934326,
      "logits/rejected": -2.069185733795166,
      "logps/chosen": -0.29705238342285156,
      "logps/rejected": -0.29413706064224243,
      "loss": 0.5292507410049438,
      "loss/core": 0.5292507410049438,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6063647270202637,
      "rewards/margins": 0.9912127256393433,
      "rewards/rejected": 1.6151517629623413,
      "step": 285
    },
    {
      "epoch": 0.27708109399259523,
      "grad_norm": 668.0,
      "learning_rate": 4.29160039397884e-07,
      "logits/chosen": -2.148799419403076,
      "logits/rejected": -2.1607887744903564,
      "logps/chosen": -0.2877974808216095,
      "logps/rejected": -0.2921397387981415,
      "loss": 0.3775253891944885,
      "loss/core": 0.3775253891944885,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.4691474437713623,
      "rewards/margins": 0.05996546894311905,
      "rewards/rejected": 1.4091819524765015,
      "step": 290
    },
    {
      "epoch": 0.28185835423384686,
      "grad_norm": 8.125,
      "learning_rate": 4.2574526435641546e-07,
      "logits/chosen": -2.5482234954833984,
      "logits/rejected": -2.5131990909576416,
      "logps/chosen": -0.24539580941200256,
      "logps/rejected": -0.2573974132537842,
      "loss": 0.05928546190261841,
      "loss/core": 0.05928546190261841,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.3013224601745605,
      "rewards/margins": 0.635219931602478,
      "rewards/rejected": 0.6661025285720825,
      "step": 295
    },
    {
      "epoch": 0.28663561447509855,
      "grad_norm": 142.0,
      "learning_rate": 4.22264398708477e-07,
      "logits/chosen": -1.9641075134277344,
      "logits/rejected": -1.9324430227279663,
      "logps/chosen": -0.3132562041282654,
      "logps/rejected": -0.3241512179374695,
      "loss": 0.9757488369941711,
      "loss/core": 0.9757488369941711,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.715285539627075,
      "rewards/margins": 2.727508306503296,
      "rewards/rejected": 0.9877773523330688,
      "step": 300
    },
    {
      "epoch": 0.2914128747163502,
      "grad_norm": 12.0,
      "learning_rate": 4.187187514652819e-07,
      "logits/chosen": -2.102832794189453,
      "logits/rejected": -2.1649816036224365,
      "logps/chosen": -0.269456148147583,
      "logps/rejected": -0.26115578413009644,
      "loss": 0.22697356343269348,
      "loss/core": 0.22697356343269348,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1590380668640137,
      "rewards/margins": 1.2657159566879272,
      "rewards/rejected": 0.8933222889900208,
      "step": 305
    },
    {
      "epoch": 0.2961901349576018,
      "grad_norm": 132.0,
      "learning_rate": 4.151096559997519e-07,
      "logits/chosen": -2.111849308013916,
      "logits/rejected": -2.0225253105163574,
      "logps/chosen": -0.2739498019218445,
      "logps/rejected": -0.28690317273139954,
      "loss": 0.9037171602249146,
      "loss/core": 0.9037171602249146,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.6031503677368164,
      "rewards/margins": 0.6503016352653503,
      "rewards/rejected": 1.9528486728668213,
      "step": 310
    },
    {
      "epoch": 0.30096739519885346,
      "grad_norm": 5.84375,
      "learning_rate": 4.114384695450905e-07,
      "logits/chosen": -1.9431352615356445,
      "logits/rejected": -2.0480642318725586,
      "logps/chosen": -0.2702833116054535,
      "logps/rejected": -0.27680981159210205,
      "loss": 0.45981135964393616,
      "loss/core": 0.45981135964393616,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.901813268661499,
      "rewards/margins": 1.5673255920410156,
      "rewards/rejected": 1.3344876766204834,
      "step": 315
    },
    {
      "epoch": 0.3057446554401051,
      "grad_norm": 3.578125,
      "learning_rate": 4.077065726843828e-07,
      "logits/chosen": -1.9837052822113037,
      "logits/rejected": -2.000582456588745,
      "logps/chosen": -0.2918964922428131,
      "logps/rejected": -0.29292041063308716,
      "loss": 0.19344636797904968,
      "loss/core": 0.19344636797904968,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.5632867813110352,
      "rewards/margins": 0.3376685380935669,
      "rewards/rejected": 1.2256181240081787,
      "step": 320
    },
    {
      "epoch": 0.31052191568135673,
      "grad_norm": 556.0,
      "learning_rate": 4.039153688314145e-07,
      "logits/chosen": -2.2357137203216553,
      "logits/rejected": -2.1825029850006104,
      "logps/chosen": -0.28656476736068726,
      "logps/rejected": -0.28232595324516296,
      "loss": 1.8139476776123047,
      "loss/core": 1.8139476776123047,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.6962344646453857,
      "rewards/margins": 2.1483452320098877,
      "rewards/rejected": 1.5478893518447876,
      "step": 325
    },
    {
      "epoch": 0.31529917592260837,
      "grad_norm": 2.640625,
      "learning_rate": 4.0006628370290613e-07,
      "logits/chosen": -2.2076706886291504,
      "logits/rejected": -2.196662664413452,
      "logps/chosen": -0.282164603471756,
      "logps/rejected": -0.27122125029563904,
      "loss": 0.2774432897567749,
      "loss/core": 0.2774432897567749,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.995111107826233,
      "rewards/margins": 0.8708856701850891,
      "rewards/rejected": 1.12422513961792,
      "step": 330
    },
    {
      "epoch": 0.32007643616386,
      "grad_norm": 344.0,
      "learning_rate": 3.9616076478235826e-07,
      "logits/chosen": -2.0309836864471436,
      "logits/rejected": -2.014965057373047,
      "logps/chosen": -0.30268004536628723,
      "logps/rejected": -0.32926321029663086,
      "loss": 0.4523457884788513,
      "loss/core": 0.4523457884788513,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.116103410720825,
      "rewards/margins": 1.5347257852554321,
      "rewards/rejected": 0.5813775658607483,
      "step": 335
    },
    {
      "epoch": 0.3248536964051117,
      "grad_norm": 132.0,
      "learning_rate": 3.922002807757129e-07,
      "logits/chosen": -2.0748965740203857,
      "logits/rejected": -2.196530342102051,
      "logps/chosen": -0.28026294708251953,
      "logps/rejected": -0.2833474576473236,
      "loss": 0.31017398834228516,
      "loss/core": 0.31017398834228516,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.0382297039031982,
      "rewards/margins": 1.1847279071807861,
      "rewards/rejected": 0.8535016775131226,
      "step": 340
    },
    {
      "epoch": 0.3296309566463633,
      "grad_norm": 8.625,
      "learning_rate": 3.8818632105903315e-07,
      "logits/chosen": -2.299494981765747,
      "logits/rejected": -2.3579187393188477,
      "logps/chosen": -0.28255495429039,
      "logps/rejected": -0.2946614921092987,
      "loss": 0.1961742490530014,
      "loss/core": 0.1961742490530014,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.639917016029358,
      "rewards/margins": 0.8168700933456421,
      "rewards/rejected": 0.823046863079071,
      "step": 345
    },
    {
      "epoch": 0.33440821688761496,
      "grad_norm": 7.625,
      "learning_rate": 3.841203951184094e-07,
      "logits/chosen": -2.0392203330993652,
      "logits/rejected": -2.0187597274780273,
      "logps/chosen": -0.261064350605011,
      "logps/rejected": -0.25779426097869873,
      "loss": 0.20222067832946777,
      "loss/core": 0.20222067832946777,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.9913139343261719,
      "rewards/margins": 1.0575062036514282,
      "rewards/rejected": 0.9338076710700989,
      "step": 350
    },
    {
      "epoch": 0.3391854771288666,
      "grad_norm": 9.625,
      "learning_rate": 3.800040319823038e-07,
      "logits/chosen": -2.088989019393921,
      "logits/rejected": -2.2161617279052734,
      "logps/chosen": -0.2810487449169159,
      "logps/rejected": -0.2965686023235321,
      "loss": 0.40663355588912964,
      "loss/core": 0.40663355588912964,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8474032878875732,
      "rewards/margins": 0.5359752774238586,
      "rewards/rejected": 1.3114280700683594,
      "step": 355
    },
    {
      "epoch": 0.34396273737011823,
      "grad_norm": 149.0,
      "learning_rate": 3.75838779646545e-07,
      "logits/chosen": -2.110039234161377,
      "logits/rejected": -2.1716244220733643,
      "logps/chosen": -0.2803703844547272,
      "logps/rejected": -0.2838951647281647,
      "loss": 0.621938169002533,
      "loss/core": 0.621938169002533,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.6312050819396973,
      "rewards/margins": 1.3762609958648682,
      "rewards/rejected": 1.2549439668655396,
      "step": 360
    },
    {
      "epoch": 0.34873999761136987,
      "grad_norm": 474.0,
      "learning_rate": 3.7162620449218993e-07,
      "logits/chosen": -2.048468589782715,
      "logits/rejected": -2.1561145782470703,
      "logps/chosen": -0.287244588136673,
      "logps/rejected": -0.28558677434921265,
      "loss": 0.9522292017936707,
      "loss/core": 0.9522292017936707,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.8516860008239746,
      "rewards/margins": 2.0592613220214844,
      "rewards/rejected": 0.7924247980117798,
      "step": 365
    },
    {
      "epoch": 0.3535172578526215,
      "grad_norm": 21.25,
      "learning_rate": 3.673678906964727e-07,
      "logits/chosen": -2.246171474456787,
      "logits/rejected": -2.291537046432495,
      "logps/chosen": -0.28519970178604126,
      "logps/rejected": -0.2871125638484955,
      "loss": 0.17473874986171722,
      "loss/core": 0.17473874986171722,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.6407058238983154,
      "rewards/margins": 0.5970925688743591,
      "rewards/rejected": 1.0436131954193115,
      "step": 370
    },
    {
      "epoch": 0.35829451809387314,
      "grad_norm": 7.9375,
      "learning_rate": 3.6306543963705936e-07,
      "logits/chosen": -2.027927875518799,
      "logits/rejected": -2.0061910152435303,
      "logps/chosen": -0.2594534456729889,
      "logps/rejected": -0.2606300711631775,
      "loss": 0.8009087443351746,
      "loss/core": 0.8009087443351746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.0067553520202637,
      "rewards/margins": 0.8116471171379089,
      "rewards/rejected": 2.195107936859131,
      "step": 375
    },
    {
      "epoch": 0.36307177833512483,
      "grad_norm": 13.1875,
      "learning_rate": 3.5872046928983623e-07,
      "logits/chosen": -2.099975109100342,
      "logits/rejected": -2.1087899208068848,
      "logps/chosen": -0.2818046808242798,
      "logps/rejected": -0.29659533500671387,
      "loss": 0.7302302122116089,
      "loss/core": 0.7302302122116089,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.033167839050293,
      "rewards/margins": 0.5039272904396057,
      "rewards/rejected": 1.529240608215332,
      "step": 380
    },
    {
      "epoch": 0.36784903857637646,
      "grad_norm": 29.375,
      "learning_rate": 3.5433461362045447e-07,
      "logits/chosen": -2.0900938510894775,
      "logits/rejected": -2.1108756065368652,
      "logps/chosen": -0.2719820439815521,
      "logps/rejected": -0.2751936912536621,
      "loss": 0.3768286406993866,
      "loss/core": 0.3768286406993866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.160437822341919,
      "rewards/margins": 0.6413952112197876,
      "rewards/rejected": 1.5190424919128418,
      "step": 385
    },
    {
      "epoch": 0.3726262988176281,
      "grad_norm": 1.953125,
      "learning_rate": 3.4990952196986305e-07,
      "logits/chosen": -1.8510644435882568,
      "logits/rejected": -1.9279950857162476,
      "logps/chosen": -0.3081986606121063,
      "logps/rejected": -0.30626800656318665,
      "loss": 0.1976659595966339,
      "loss/core": 0.1976659595966339,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.7517478466033936,
      "rewards/margins": 0.8936412930488586,
      "rewards/rejected": 0.8581066131591797,
      "step": 390
    },
    {
      "epoch": 0.37740355905887973,
      "grad_norm": 19.75,
      "learning_rate": 3.4544685843405875e-07,
      "logits/chosen": -2.1983537673950195,
      "logits/rejected": -2.3012776374816895,
      "logps/chosen": -0.2957797050476074,
      "logps/rejected": -0.2903483510017395,
      "loss": 0.1597086489200592,
      "loss/core": 0.1597086489200592,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.6743053197860718,
      "rewards/margins": 0.8747150301933289,
      "rewards/rejected": 0.7995903491973877,
      "step": 395
    },
    {
      "epoch": 0.38218081930013137,
      "grad_norm": 50.5,
      "learning_rate": 3.4094830123828786e-07,
      "logits/chosen": -2.0785083770751953,
      "logits/rejected": -2.1138789653778076,
      "logps/chosen": -0.2923468351364136,
      "logps/rejected": -0.29322344064712524,
      "loss": 1.0465948581695557,
      "loss/core": 1.0465948581695557,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9298787117004395,
      "rewards/margins": 2.0894033908843994,
      "rewards/rejected": 0.8404749631881714,
      "step": 400
    },
    {
      "epoch": 0.386958079541383,
      "grad_norm": 5.90625,
      "learning_rate": 3.3641554210593414e-07,
      "logits/chosen": -2.1093640327453613,
      "logits/rejected": -2.1642727851867676,
      "logps/chosen": -0.31529492139816284,
      "logps/rejected": -0.30353349447250366,
      "loss": 0.6650071740150452,
      "loss/core": 0.6650071740150452,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3120980262756348,
      "rewards/margins": 1.074296236038208,
      "rewards/rejected": 1.2378015518188477,
      "step": 405
    },
    {
      "epoch": 0.39173533978263464,
      "grad_norm": 20.125,
      "learning_rate": 3.3185028562233107e-07,
      "logits/chosen": -2.227759838104248,
      "logits/rejected": -2.2588248252868652,
      "logps/chosen": -0.277637243270874,
      "logps/rejected": -0.2886066436767578,
      "loss": 0.27694252133369446,
      "loss/core": 0.27694252133369446,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.081727981567383,
      "rewards/margins": 0.8987264633178711,
      "rewards/rejected": 1.1830016374588013,
      "step": 410
    },
    {
      "epoch": 0.3965126000238863,
      "grad_norm": 12.625,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.0978333950042725,
      "logits/rejected": -2.175910234451294,
      "logps/chosen": -0.3309980034828186,
      "logps/rejected": -0.32166236639022827,
      "loss": 0.05876865237951279,
      "loss/core": 0.05876865237951279,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.988106906414032,
      "rewards/margins": 0.6192494630813599,
      "rewards/rejected": 0.36885741353034973,
      "step": 415
    },
    {
      "epoch": 0.40128986026513797,
      "grad_norm": 88.5,
      "learning_rate": 3.226291594017137e-07,
      "logits/chosen": -2.326503038406372,
      "logits/rejected": -2.2930359840393066,
      "logps/chosen": -0.28376084566116333,
      "logps/rejected": -0.3000229001045227,
      "loss": 0.3358566462993622,
      "loss/core": 0.3358566462993622,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0413100719451904,
      "rewards/margins": 1.3324414491653442,
      "rewards/rejected": 0.7088686227798462,
      "step": 420
    },
    {
      "epoch": 0.4060671205063896,
      "grad_norm": 196.0,
      "learning_rate": 3.1797675735315454e-07,
      "logits/chosen": -1.9907684326171875,
      "logits/rejected": -2.05031156539917,
      "logps/chosen": -0.2916226387023926,
      "logps/rejected": -0.2697221636772156,
      "loss": 0.768527626991272,
      "loss/core": 0.768527626991272,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2350106239318848,
      "rewards/margins": 0.6351408958435059,
      "rewards/rejected": 1.5998692512512207,
      "step": 425
    },
    {
      "epoch": 0.41084438074764124,
      "grad_norm": 732.0,
      "learning_rate": 3.1329879202620047e-07,
      "logits/chosen": -2.0472471714019775,
      "logits/rejected": -2.0714943408966064,
      "logps/chosen": -0.28120359778404236,
      "logps/rejected": -0.2685726583003998,
      "loss": 1.8000812530517578,
      "loss/core": 1.8000812530517578,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 4.386286735534668,
      "rewards/margins": 2.987924814224243,
      "rewards/rejected": 1.3983619213104248,
      "step": 430
    },
    {
      "epoch": 0.41562164098889287,
      "grad_norm": 8.8125,
      "learning_rate": 3.0859702261229613e-07,
      "logits/chosen": -2.324946165084839,
      "logits/rejected": -2.3079473972320557,
      "logps/chosen": -0.27902793884277344,
      "logps/rejected": -0.29347580671310425,
      "loss": 0.2374797761440277,
      "loss/core": 0.2374797761440277,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.617841362953186,
      "rewards/margins": 0.6150254011154175,
      "rewards/rejected": 1.0028159618377686,
      "step": 435
    },
    {
      "epoch": 0.4203989012301445,
      "grad_norm": 484.0,
      "learning_rate": 3.0387321725463e-07,
      "logits/chosen": -2.0346179008483887,
      "logits/rejected": -2.105949878692627,
      "logps/chosen": -0.3050372004508972,
      "logps/rejected": -0.304879367351532,
      "loss": 0.9780192375183105,
      "loss/core": 0.9780192375183105,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.988939046859741,
      "rewards/margins": 1.6511949300765991,
      "rewards/rejected": 1.3377439975738525,
      "step": 440
    },
    {
      "epoch": 0.42517616147139614,
      "grad_norm": 41.0,
      "learning_rate": 2.991291523832075e-07,
      "logits/chosen": -1.9136848449707031,
      "logits/rejected": -1.8891462087631226,
      "logps/chosen": -0.29649606347084045,
      "logps/rejected": -0.2987465262413025,
      "loss": 0.5171946287155151,
      "loss/core": 0.5171946287155151,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.9516830444335938,
      "rewards/margins": 1.4048423767089844,
      "rewards/rejected": 1.5468409061431885,
      "step": 445
    },
    {
      "epoch": 0.4299534217126478,
      "grad_norm": 418.0,
      "learning_rate": 2.943666120468088e-07,
      "logits/chosen": -1.9518722295761108,
      "logits/rejected": -1.9711889028549194,
      "logps/chosen": -0.3000376522541046,
      "logps/rejected": -0.29597753286361694,
      "loss": 0.8665231466293335,
      "loss/core": 0.8665231466293335,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.440600872039795,
      "rewards/margins": 1.8995399475097656,
      "rewards/rejected": 1.5410608053207397,
      "step": 450
    },
    {
      "epoch": 0.4347306819538994,
      "grad_norm": 183.0,
      "learning_rate": 2.8958738724208073e-07,
      "logits/chosen": -1.9178097248077393,
      "logits/rejected": -2.1076836585998535,
      "logps/chosen": -0.3262283205986023,
      "logps/rejected": -0.3135696053504944,
      "loss": 1.426609754562378,
      "loss/core": 1.426609754562378,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.4305222034454346,
      "rewards/margins": 1.9480667114257812,
      "rewards/rejected": 1.4824554920196533,
      "step": 455
    },
    {
      "epoch": 0.4395079421951511,
      "grad_norm": 30.75,
      "learning_rate": 2.8479327524001633e-07,
      "logits/chosen": -2.036893606185913,
      "logits/rejected": -2.0251917839050293,
      "logps/chosen": -0.2996273636817932,
      "logps/rejected": -0.30763858556747437,
      "loss": 1.1361322402954102,
      "loss/core": 1.1361322402954102,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.628493547439575,
      "rewards/margins": 1.7698605060577393,
      "rewards/rejected": 1.858633041381836,
      "step": 460
    },
    {
      "epoch": 0.44428520243640274,
      "grad_norm": 10.3125,
      "learning_rate": 2.7998607891007493e-07,
      "logits/chosen": -1.8706251382827759,
      "logits/rejected": -1.9697706699371338,
      "logps/chosen": -0.2975318431854248,
      "logps/rejected": -0.2920708954334259,
      "loss": 0.23389527201652527,
      "loss/core": 0.23389527201652527,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.297299861907959,
      "rewards/margins": 1.371745228767395,
      "rewards/rejected": 0.9255548715591431,
      "step": 465
    },
    {
      "epoch": 0.4490624626776544,
      "grad_norm": 1048.0,
      "learning_rate": 2.7516760604219616e-07,
      "logits/chosen": -2.105297803878784,
      "logits/rejected": -2.173494815826416,
      "logps/chosen": -0.28127214312553406,
      "logps/rejected": -0.2611326277256012,
      "loss": 1.1431621313095093,
      "loss/core": 1.1431621313095093,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.987865447998047,
      "rewards/margins": 1.8296188116073608,
      "rewards/rejected": 1.1582467555999756,
      "step": 470
    },
    {
      "epoch": 0.453839722918906,
      "grad_norm": 90.5,
      "learning_rate": 2.703396686669646e-07,
      "logits/chosen": -2.0695085525512695,
      "logits/rejected": -2.0106379985809326,
      "logps/chosen": -0.2961866855621338,
      "logps/rejected": -0.32303735613822937,
      "loss": 0.46794575452804565,
      "loss/core": 0.46794575452804565,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.823664665222168,
      "rewards/margins": 0.37910977005958557,
      "rewards/rejected": 1.4445549249649048,
      "step": 475
    },
    {
      "epoch": 0.45861698316015764,
      "grad_norm": 6.0,
      "learning_rate": 2.6550408237417884e-07,
      "logits/chosen": -2.2378296852111816,
      "logits/rejected": -2.2934558391571045,
      "logps/chosen": -0.2757987380027771,
      "logps/rejected": -0.2735208570957184,
      "loss": 0.5451094508171082,
      "loss/core": 0.5451094508171082,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.354505777359009,
      "rewards/margins": 1.340338945388794,
      "rewards/rejected": 1.0141668319702148,
      "step": 480
    },
    {
      "epoch": 0.4633942434014093,
      "grad_norm": 13.25,
      "learning_rate": 2.6066266563008265e-07,
      "logits/chosen": -2.183929920196533,
      "logits/rejected": -2.191530704498291,
      "logps/chosen": -0.2924697995185852,
      "logps/rejected": -0.29630663990974426,
      "loss": 0.15576469898223877,
      "loss/core": 0.15576469898223877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.7340059280395508,
      "rewards/margins": 0.8523953557014465,
      "rewards/rejected": 0.8816105127334595,
      "step": 485
    },
    {
      "epoch": 0.4681715036426609,
      "grad_norm": 20.0,
      "learning_rate": 2.5581723909351404e-07,
      "logits/chosen": -2.13179087638855,
      "logits/rejected": -2.1726982593536377,
      "logps/chosen": -0.3071516156196594,
      "logps/rejected": -0.29533272981643677,
      "loss": 0.29640907049179077,
      "loss/core": 0.29640907049179077,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.0775198936462402,
      "rewards/margins": 1.033736228942871,
      "rewards/rejected": 1.0437836647033691,
      "step": 490
    },
    {
      "epoch": 0.47294876388391255,
      "grad_norm": 26.125,
      "learning_rate": 2.509696249312301e-07,
      "logits/chosen": -2.2027294635772705,
      "logits/rejected": -2.2974157333374023,
      "logps/chosen": -0.2999098598957062,
      "logps/rejected": -0.2914625406265259,
      "loss": 0.7030426263809204,
      "loss/core": 0.7030426263809204,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.590477228164673,
      "rewards/margins": 1.704890251159668,
      "rewards/rejected": 0.8855869174003601,
      "step": 495
    },
    {
      "epoch": 0.47772602412516424,
      "grad_norm": 133.0,
      "learning_rate": 2.461216461326642e-07,
      "logits/chosen": -1.9994663000106812,
      "logits/rejected": -1.9483753442764282,
      "logps/chosen": -0.28346696496009827,
      "logps/rejected": -0.30574333667755127,
      "loss": 2.2656641006469727,
      "loss/core": 2.2656641006469727,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.139856815338135,
      "rewards/margins": 2.1449146270751953,
      "rewards/rejected": 1.994942307472229,
      "step": 500
    },
    {
      "epoch": 0.4825032843664159,
      "grad_norm": 348.0,
      "learning_rate": 2.412751258243748e-07,
      "logits/chosen": -2.1276748180389404,
      "logits/rejected": -2.1600797176361084,
      "logps/chosen": -0.2883078157901764,
      "logps/rejected": -0.29086965322494507,
      "loss": 1.4473992586135864,
      "loss/core": 1.4473992586135864,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.5834662914276123,
      "rewards/margins": 1.8612127304077148,
      "rewards/rejected": 1.7222535610198975,
      "step": 505
    },
    {
      "epoch": 0.4872805446076675,
      "grad_norm": 0.2890625,
      "learning_rate": 2.3643188658444158e-07,
      "logits/chosen": -2.127025604248047,
      "logits/rejected": -2.259706735610962,
      "logps/chosen": -0.31721481680870056,
      "logps/rejected": -0.29182881116867065,
      "loss": 0.41495251655578613,
      "loss/core": 0.41495251655578613,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9977633953094482,
      "rewards/margins": 1.2025401592254639,
      "rewards/rejected": 0.7952233552932739,
      "step": 510
    },
    {
      "epoch": 0.49205780484891914,
      "grad_norm": 35.75,
      "learning_rate": 2.315937497570688e-07,
      "logits/chosen": -2.010084629058838,
      "logits/rejected": -2.0125226974487305,
      "logps/chosen": -0.2837277054786682,
      "logps/rejected": -0.3070703148841858,
      "loss": 0.43133026361465454,
      "loss/core": 0.43133026361465454,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.5341956615448,
      "rewards/margins": 0.8866618275642395,
      "rewards/rejected": 1.6475337743759155,
      "step": 515
    },
    {
      "epoch": 0.4968350650901708,
      "grad_norm": 40.5,
      "learning_rate": 2.2676253476765194e-07,
      "logits/chosen": -2.039220094680786,
      "logits/rejected": -2.2076566219329834,
      "logps/chosen": -0.32395556569099426,
      "logps/rejected": -0.33448874950408936,
      "loss": 0.802078366279602,
      "loss/core": 0.802078366279602,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.054922580718994,
      "rewards/margins": 2.0955018997192383,
      "rewards/rejected": 0.9594209790229797,
      "step": 520
    },
    {
      "epoch": 0.5016123253314224,
      "grad_norm": 1288.0,
      "learning_rate": 2.2194005843856633e-07,
      "logits/chosen": -2.1711580753326416,
      "logits/rejected": -2.1403403282165527,
      "logps/chosen": -0.3041841983795166,
      "logps/rejected": -0.3024327754974365,
      "loss": 1.989890456199646,
      "loss/core": 1.989890456199646,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 4.269538402557373,
      "rewards/margins": 2.8283638954162598,
      "rewards/rejected": 1.441174864768982,
      "step": 525
    },
    {
      "epoch": 0.5063895855726741,
      "grad_norm": 0.408203125,
      "learning_rate": 2.1712813430593434e-07,
      "logits/chosen": -2.016860246658325,
      "logits/rejected": -1.9806495904922485,
      "logps/chosen": -0.2869611382484436,
      "logps/rejected": -0.2894926369190216,
      "loss": 0.7920265793800354,
      "loss/core": 0.7920265793800354,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.041126251220703,
      "rewards/margins": 1.5607634782791138,
      "rewards/rejected": 1.4803630113601685,
      "step": 530
    },
    {
      "epoch": 0.5111668458139257,
      "grad_norm": 644.0,
      "learning_rate": 2.123285719376292e-07,
      "logits/chosen": -2.1647980213165283,
      "logits/rejected": -2.086789846420288,
      "logps/chosen": -0.2868487536907196,
      "logps/rejected": -0.28134408593177795,
      "loss": 0.9000055193901062,
      "loss/core": 0.9000055193901062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.0810141563415527,
      "rewards/margins": 2.0247347354888916,
      "rewards/rejected": 1.056279182434082,
      "step": 535
    },
    {
      "epoch": 0.5159441060551774,
      "grad_norm": 1.453125,
      "learning_rate": 2.0754317625276982e-07,
      "logits/chosen": -2.085092067718506,
      "logits/rejected": -2.064635992050171,
      "logps/chosen": -0.2793554961681366,
      "logps/rejected": -0.2893218398094177,
      "loss": 0.3135538399219513,
      "loss/core": 0.3135538399219513,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5822449922561646,
      "rewards/margins": 0.09157203137874603,
      "rewards/rejected": 1.4906729459762573,
      "step": 540
    },
    {
      "epoch": 0.520721366296429,
      "grad_norm": 1128.0,
      "learning_rate": 2.0277374684296498e-07,
      "logits/chosen": -2.306612730026245,
      "logits/rejected": -2.1498613357543945,
      "logps/chosen": -0.2859051823616028,
      "logps/rejected": -0.29374048113822937,
      "loss": 0.9199239611625671,
      "loss/core": 0.9199239611625671,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.6144795417785645,
      "rewards/margins": 1.7432453632354736,
      "rewards/rejected": 0.8712340593338013,
      "step": 545
    },
    {
      "epoch": 0.5254986265376806,
      "grad_norm": 12.8125,
      "learning_rate": 1.980220772955602e-07,
      "logits/chosen": -1.8739219903945923,
      "logits/rejected": -1.8925116062164307,
      "logps/chosen": -0.27485448122024536,
      "logps/rejected": -0.3017066717147827,
      "loss": 0.7916240692138672,
      "loss/core": 0.7916240692138672,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.78112530708313,
      "rewards/margins": 1.196991205215454,
      "rewards/rejected": 1.5841339826583862,
      "step": 550
    },
    {
      "epoch": 0.5302758867789323,
      "grad_norm": 5.34375,
      "learning_rate": 1.932899545191433e-07,
      "logits/chosen": -2.0721099376678467,
      "logits/rejected": -2.1279807090759277,
      "logps/chosen": -0.311161607503891,
      "logps/rejected": -0.27461129426956177,
      "loss": 0.827431857585907,
      "loss/core": 0.827431857585907,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.3001389503479004,
      "rewards/margins": 2.3012726306915283,
      "rewards/rejected": 0.998866081237793,
      "step": 555
    },
    {
      "epoch": 0.5350531470201839,
      "grad_norm": 54.25,
      "learning_rate": 1.885791580715609e-07,
      "logits/chosen": -2.157796859741211,
      "logits/rejected": -2.144073009490967,
      "logps/chosen": -0.27682000398635864,
      "logps/rejected": -0.28132444620132446,
      "loss": 0.6787029504776001,
      "loss/core": 0.6787029504776001,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.8043270111083984,
      "rewards/margins": 1.2791929244995117,
      "rewards/rejected": 1.5251343250274658,
      "step": 560
    },
    {
      "epoch": 0.5398304072614356,
      "grad_norm": 21.0,
      "learning_rate": 1.8389145949069951e-07,
      "logits/chosen": -2.05513596534729,
      "logits/rejected": -2.0813965797424316,
      "logps/chosen": -0.28800255060195923,
      "logps/rejected": -0.3045352101325989,
      "loss": 1.3088706731796265,
      "loss/core": 1.3088706731796265,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.775949478149414,
      "rewards/margins": 1.1087366342544556,
      "rewards/rejected": 1.6672130823135376,
      "step": 565
    },
    {
      "epoch": 0.5446076675026872,
      "grad_norm": 66.5,
      "learning_rate": 1.792286216282824e-07,
      "logits/chosen": -2.0531630516052246,
      "logits/rejected": -2.1032092571258545,
      "logps/chosen": -0.2883487045764923,
      "logps/rejected": -0.29088732600212097,
      "loss": 0.902314305305481,
      "loss/core": 0.902314305305481,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.3471977710723877,
      "rewards/margins": 1.6828256845474243,
      "rewards/rejected": 1.6643718481063843,
      "step": 570
    },
    {
      "epoch": 0.5493849277439389,
      "grad_norm": 8.6875,
      "learning_rate": 1.745923979869336e-07,
      "logits/chosen": -2.1334240436553955,
      "logits/rejected": -2.164499044418335,
      "logps/chosen": -0.3011549413204193,
      "logps/rejected": -0.32307350635528564,
      "loss": 0.3328213393688202,
      "loss/core": 0.3328213393688202,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8378055095672607,
      "rewards/margins": 1.2759101390838623,
      "rewards/rejected": 0.5618952512741089,
      "step": 575
    },
    {
      "epoch": 0.5541621879851905,
      "grad_norm": 0.490234375,
      "learning_rate": 1.6998453206075708e-07,
      "logits/chosen": -2.4017269611358643,
      "logits/rejected": -2.458293914794922,
      "logps/chosen": -0.26788607239723206,
      "logps/rejected": -0.28343814611434937,
      "loss": 0.1603700816631317,
      "loss/core": 0.1603700816631317,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.406388282775879,
      "rewards/margins": 0.4723166823387146,
      "rewards/rejected": 0.9340717196464539,
      "step": 580
    },
    {
      "epoch": 0.5589394482264421,
      "grad_norm": 32.0,
      "learning_rate": 1.6540675667967973e-07,
      "logits/chosen": -2.1147303581237793,
      "logits/rejected": -2.0781502723693848,
      "logps/chosen": -0.2706436514854431,
      "logps/rejected": -0.2854623794555664,
      "loss": 1.4029546976089478,
      "loss/core": 1.4029546976089478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.21844220161438,
      "rewards/margins": 0.8768823742866516,
      "rewards/rejected": 1.341559648513794,
      "step": 585
    },
    {
      "epoch": 0.5637167084676937,
      "grad_norm": 4.40625,
      "learning_rate": 1.60860793357805e-07,
      "logits/chosen": -2.001035690307617,
      "logits/rejected": -2.096832275390625,
      "logps/chosen": -0.29010075330734253,
      "logps/rejected": -0.2887147068977356,
      "loss": 0.8843881487846375,
      "loss/core": 0.8843881487846375,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.5953457355499268,
      "rewards/margins": 2.580474376678467,
      "rewards/rejected": 1.01487135887146,
      "step": 590
    },
    {
      "epoch": 0.5684939687089454,
      "grad_norm": 3.734375,
      "learning_rate": 1.5634835164602196e-07,
      "logits/chosen": -2.020965099334717,
      "logits/rejected": -2.015770673751831,
      "logps/chosen": -0.27582883834838867,
      "logps/rejected": -0.2730068266391754,
      "loss": 0.1141238808631897,
      "loss/core": 0.1141238808631897,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.7046051025390625,
      "rewards/margins": 0.826572060585022,
      "rewards/rejected": 0.8780330419540405,
      "step": 595
    },
    {
      "epoch": 0.5732712289501971,
      "grad_norm": 24.375,
      "learning_rate": 1.518711284891132e-07,
      "logits/chosen": -2.250802755355835,
      "logits/rejected": -2.2414631843566895,
      "logps/chosen": -0.26670363545417786,
      "logps/rejected": -0.286176860332489,
      "loss": 0.2968599498271942,
      "loss/core": 0.2968599498271942,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8693134784698486,
      "rewards/margins": 0.8250168561935425,
      "rewards/rejected": 1.0442965030670166,
      "step": 600
    },
    {
      "epoch": 0.5780484891914487,
      "grad_norm": 129.0,
      "learning_rate": 1.47430807587603e-07,
      "logits/chosen": -2.0972466468811035,
      "logits/rejected": -2.1712050437927246,
      "logps/chosen": -0.2813746929168701,
      "logps/rejected": -0.2767219841480255,
      "loss": 1.7541176080703735,
      "loss/core": 1.7541176080703735,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.171405792236328,
      "rewards/margins": 2.975759267807007,
      "rewards/rejected": 1.1956462860107422,
      "step": 605
    },
    {
      "epoch": 0.5828257494327004,
      "grad_norm": 104.0,
      "learning_rate": 1.430290587645865e-07,
      "logits/chosen": -2.0927460193634033,
      "logits/rejected": -2.1099424362182617,
      "logps/chosen": -0.27945631742477417,
      "logps/rejected": -0.27728086709976196,
      "loss": 1.8355001211166382,
      "loss/core": 1.8355001211166382,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 4.239993095397949,
      "rewards/margins": 2.323775291442871,
      "rewards/rejected": 1.9162174463272095,
      "step": 610
    },
    {
      "epoch": 0.587603009673952,
      "grad_norm": 13.0,
      "learning_rate": 1.3866753733777765e-07,
      "logits/chosen": -1.9469194412231445,
      "logits/rejected": -2.012991189956665,
      "logps/chosen": -0.3159197270870209,
      "logps/rejected": -0.3063480257987976,
      "loss": 0.5646291971206665,
      "loss/core": 0.5646291971206665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.7208328247070312,
      "rewards/margins": 1.2584408521652222,
      "rewards/rejected": 1.4623918533325195,
      "step": 615
    },
    {
      "epoch": 0.5923802699152036,
      "grad_norm": 6.78125,
      "learning_rate": 1.343478834970121e-07,
      "logits/chosen": -2.1200366020202637,
      "logits/rejected": -2.1472256183624268,
      "logps/chosen": -0.286575585603714,
      "logps/rejected": -0.29443609714508057,
      "loss": 0.37433984875679016,
      "loss/core": 0.37433984875679016,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6850149631500244,
      "rewards/margins": 0.25270089507102966,
      "rewards/rejected": 1.432314157485962,
      "step": 620
    },
    {
      "epoch": 0.5971575301564552,
      "grad_norm": 1520.0,
      "learning_rate": 1.3007172168743852e-07,
      "logits/chosen": -1.9102928638458252,
      "logits/rejected": -1.938646912574768,
      "logps/chosen": -0.3089125454425812,
      "logps/rejected": -0.3455188274383545,
      "loss": 2.0537664890289307,
      "loss/core": 2.0537664890289307,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.8455417156219482,
      "rewards/margins": 2.4277970790863037,
      "rewards/rejected": 1.4177448749542236,
      "step": 625
    },
    {
      "epoch": 0.6019347903977069,
      "grad_norm": 178.0,
      "learning_rate": 1.25840659998631e-07,
      "logits/chosen": -1.9074939489364624,
      "logits/rejected": -2.022514820098877,
      "logps/chosen": -0.27223581075668335,
      "logps/rejected": -0.28289422392845154,
      "loss": 0.8955941200256348,
      "loss/core": 0.8955941200256348,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9453296661376953,
      "rewards/margins": 1.6964915990829468,
      "rewards/rejected": 1.2488377094268799,
      "step": 630
    },
    {
      "epoch": 0.6067120506389586,
      "grad_norm": 286.0,
      "learning_rate": 1.2165628955985313e-07,
      "logits/chosen": -1.9614832401275635,
      "logits/rejected": -2.0468192100524902,
      "logps/chosen": -0.30598872900009155,
      "logps/rejected": -0.30102452635765076,
      "loss": 0.4347690939903259,
      "loss/core": 0.4347690939903259,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.2950398921966553,
      "rewards/margins": 1.2899543046951294,
      "rewards/rejected": 1.0050859451293945,
      "step": 635
    },
    {
      "epoch": 0.6114893108802102,
      "grad_norm": 2.59375,
      "learning_rate": 1.175201839416988e-07,
      "logits/chosen": -2.1878843307495117,
      "logits/rejected": -2.2639148235321045,
      "logps/chosen": -0.3026634752750397,
      "logps/rejected": -0.3000384271144867,
      "loss": 0.3829612135887146,
      "loss/core": 0.3829612135887146,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9458287954330444,
      "rewards/margins": 1.161188006401062,
      "rewards/rejected": 0.7846406102180481,
      "step": 640
    },
    {
      "epoch": 0.6162665711214619,
      "grad_norm": 15.0,
      "learning_rate": 1.1343389856433658e-07,
      "logits/chosen": -2.133284091949463,
      "logits/rejected": -2.1109793186187744,
      "logps/chosen": -0.2913122773170471,
      "logps/rejected": -0.3065160810947418,
      "loss": 0.20538568496704102,
      "loss/core": 0.20538568496704102,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.6671216487884521,
      "rewards/margins": 0.5487877726554871,
      "rewards/rejected": 1.1183340549468994,
      "step": 645
    },
    {
      "epoch": 0.6210438313627135,
      "grad_norm": 3.78125,
      "learning_rate": 1.0939897011258e-07,
      "logits/chosen": -2.198296546936035,
      "logits/rejected": -2.151305913925171,
      "logps/chosen": -0.2899485230445862,
      "logps/rejected": -0.30108729004859924,
      "loss": 0.1469523161649704,
      "loss/core": 0.1469523161649704,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.3288962841033936,
      "rewards/margins": 0.5087905526161194,
      "rewards/rejected": 0.8201059103012085,
      "step": 650
    },
    {
      "epoch": 0.6258210916039652,
      "grad_norm": 53.0,
      "learning_rate": 1.0541691595800336e-07,
      "logits/chosen": -2.037231922149658,
      "logits/rejected": -2.0994153022766113,
      "logps/chosen": -0.28296002745628357,
      "logps/rejected": -0.29361408948898315,
      "loss": 0.24921326339244843,
      "loss/core": 0.24921326339244843,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.997532844543457,
      "rewards/margins": 1.048076868057251,
      "rewards/rejected": 0.9494560360908508,
      "step": 655
    },
    {
      "epoch": 0.6305983518452167,
      "grad_norm": 2.390625,
      "learning_rate": 1.0148923358832021e-07,
      "logits/chosen": -2.0072593688964844,
      "logits/rejected": -2.000403642654419,
      "logps/chosen": -0.29590821266174316,
      "logps/rejected": -0.3091500699520111,
      "loss": 1.2735823392868042,
      "loss/core": 1.2735823392868042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.7570533752441406,
      "rewards/margins": 1.4806814193725586,
      "rewards/rejected": 1.2763724327087402,
      "step": 660
    },
    {
      "epoch": 0.6353756120864684,
      "grad_norm": 134.0,
      "learning_rate": 9.761740004423926e-08,
      "logits/chosen": -2.169792652130127,
      "logits/rejected": -2.1452460289001465,
      "logps/chosen": -0.28445929288864136,
      "logps/rejected": -0.2821694612503052,
      "loss": 0.505510151386261,
      "loss/core": 0.505510151386261,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.346339702606201,
      "rewards/margins": 0.8737912178039551,
      "rewards/rejected": 1.472548484802246,
      "step": 665
    },
    {
      "epoch": 0.64015287232772,
      "grad_norm": 1.421875,
      "learning_rate": 9.380287136400999e-08,
      "logits/chosen": -1.958132028579712,
      "logits/rejected": -2.0062148571014404,
      "logps/chosen": -0.2736981213092804,
      "logps/rejected": -0.32142865657806396,
      "loss": 0.9827208518981934,
      "loss/core": 0.9827208518981934,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.6511759757995605,
      "rewards/margins": 2.406550168991089,
      "rewards/rejected": 1.2446256875991821,
      "step": 670
    },
    {
      "epoch": 0.6449301325689717,
      "grad_norm": 15.375,
      "learning_rate": 9.004708203586628e-08,
      "logits/chosen": -2.010897159576416,
      "logits/rejected": -2.05232834815979,
      "logps/chosen": -0.28901082277297974,
      "logps/rejected": -0.2891398072242737,
      "loss": 0.12457512319087982,
      "loss/core": 0.12457512319087982,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.6163723468780518,
      "rewards/margins": 0.7447583079338074,
      "rewards/rejected": 0.8716139793395996,
      "step": 675
    },
    {
      "epoch": 0.6497073928102234,
      "grad_norm": 1.7265625,
      "learning_rate": 8.635144445857407e-08,
      "logits/chosen": -2.133448362350464,
      "logits/rejected": -2.0533690452575684,
      "logps/chosen": -0.28220346570014954,
      "logps/rejected": -0.2999503016471863,
      "loss": 0.41169777512550354,
      "loss/core": 0.41169777512550354,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.1444594860076904,
      "rewards/margins": 0.8426515460014343,
      "rewards/rejected": 1.3018076419830322,
      "step": 680
    },
    {
      "epoch": 0.654484653051475,
      "grad_norm": 4.28125,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -2.0622639656066895,
      "logits/rejected": -2.0482723712921143,
      "logps/chosen": -0.2543559670448303,
      "logps/rejected": -0.2783207893371582,
      "loss": 0.9395847320556641,
      "loss/core": 0.9395847320556641,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.8786520957946777,
      "rewards/margins": 1.356764554977417,
      "rewards/rejected": 1.5218875408172607,
      "step": 685
    },
    {
      "epoch": 0.6592619132927267,
      "grad_norm": 3.203125,
      "learning_rate": 7.91461605259007e-08,
      "logits/chosen": -1.8647575378417969,
      "logits/rejected": -1.887168526649475,
      "logps/chosen": -0.3094645142555237,
      "logps/rejected": -0.32861852645874023,
      "loss": 0.20163805782794952,
      "loss/core": 0.20163805782794952,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.7553918361663818,
      "rewards/margins": 0.6693966388702393,
      "rewards/rejected": 1.0859953165054321,
      "step": 690
    },
    {
      "epoch": 0.6640391735339782,
      "grad_norm": 9.375,
      "learning_rate": 7.563922378313217e-08,
      "logits/chosen": -2.2469770908355713,
      "logits/rejected": -2.256824016571045,
      "logps/chosen": -0.3077448308467865,
      "logps/rejected": -0.2938057780265808,
      "loss": 0.3713051676750183,
      "loss/core": 0.3713051676750183,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9559170007705688,
      "rewards/margins": 1.282508134841919,
      "rewards/rejected": 0.6734089851379395,
      "step": 695
    },
    {
      "epoch": 0.6688164337752299,
      "grad_norm": 388.0,
      "learning_rate": 7.219785699746572e-08,
      "logits/chosen": -2.08168888092041,
      "logits/rejected": -2.1877474784851074,
      "logps/chosen": -0.2970959544181824,
      "logps/rejected": -0.3301224112510681,
      "loss": 0.46775469183921814,
      "loss/core": 0.46775469183921814,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.227623224258423,
      "rewards/margins": 0.7473882436752319,
      "rewards/rejected": 1.4802348613739014,
      "step": 700
    },
    {
      "epoch": 0.6735936940164815,
      "grad_norm": 84.5,
      "learning_rate": 6.882335432620779e-08,
      "logits/chosen": -2.1753158569335938,
      "logits/rejected": -2.1324844360351562,
      "logps/chosen": -0.28070613741874695,
      "logps/rejected": -0.2675221264362335,
      "loss": 0.8060401082038879,
      "loss/core": 0.8060401082038879,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.4011025428771973,
      "rewards/margins": 1.8703250885009766,
      "rewards/rejected": 1.5307778120040894,
      "step": 705
    },
    {
      "epoch": 0.6783709542577332,
      "grad_norm": 370.0,
      "learning_rate": 6.551698478180589e-08,
      "logits/chosen": -1.862591028213501,
      "logits/rejected": -1.8508355617523193,
      "logps/chosen": -0.27737393975257874,
      "logps/rejected": -0.3012697696685791,
      "loss": 1.3350226879119873,
      "loss/core": 1.3350226879119873,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.5951266288757324,
      "rewards/margins": 1.7325704097747803,
      "rewards/rejected": 1.8625566959381104,
      "step": 710
    },
    {
      "epoch": 0.6831482144989849,
      "grad_norm": 3.78125,
      "learning_rate": 6.22799917546252e-08,
      "logits/chosen": -2.230132579803467,
      "logits/rejected": -2.2110886573791504,
      "logps/chosen": -0.2842472493648529,
      "logps/rejected": -0.3385520279407501,
      "loss": 0.5101768374443054,
      "loss/core": 0.5101768374443054,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8642584085464478,
      "rewards/margins": 1.1836979389190674,
      "rewards/rejected": 0.6805603504180908,
      "step": 715
    },
    {
      "epoch": 0.6879254747402365,
      "grad_norm": 23.625,
      "learning_rate": 5.9113592545359944e-08,
      "logits/chosen": -1.9779284000396729,
      "logits/rejected": -1.9853112697601318,
      "logps/chosen": -0.2979108393192291,
      "logps/rejected": -0.30363336205482483,
      "loss": 0.4835893511772156,
      "loss/core": 0.4835893511772156,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.0863685607910156,
      "rewards/margins": 0.6202920079231262,
      "rewards/rejected": 1.4660766124725342,
      "step": 720
    },
    {
      "epoch": 0.6927027349814882,
      "grad_norm": 3.953125,
      "learning_rate": 5.601897790725643e-08,
      "logits/chosen": -2.102151393890381,
      "logits/rejected": -2.2167160511016846,
      "logps/chosen": -0.28644677996635437,
      "logps/rejected": -0.27808719873428345,
      "loss": 0.17492833733558655,
      "loss/core": 0.17492833733558655,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.5772634744644165,
      "rewards/margins": 0.8539497256278992,
      "rewards/rejected": 0.7233136892318726,
      "step": 725
    },
    {
      "epoch": 0.6974799952227397,
      "grad_norm": 398.0,
      "learning_rate": 5.299731159831952e-08,
      "logits/chosen": -2.1961236000061035,
      "logits/rejected": -2.246107339859009,
      "logps/chosen": -0.287611722946167,
      "logps/rejected": -0.2948874533176422,
      "loss": 0.19452638924121857,
      "loss/core": 0.19452638924121857,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.0076961517333984,
      "rewards/margins": 0.14150258898735046,
      "rewards/rejected": 0.8661934733390808,
      "step": 730
    },
    {
      "epoch": 0.7022572554639914,
      "grad_norm": 58.0,
      "learning_rate": 5.0049729943671013e-08,
      "logits/chosen": -2.0291497707366943,
      "logits/rejected": -2.0758984088897705,
      "logps/chosen": -0.32085052132606506,
      "logps/rejected": -0.31806251406669617,
      "loss": 0.14255869388580322,
      "loss/core": 0.14255869388580322,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.4092836380004883,
      "rewards/margins": 0.7487727403640747,
      "rewards/rejected": 0.6605110168457031,
      "step": 735
    },
    {
      "epoch": 0.707034515705243,
      "grad_norm": 5.46875,
      "learning_rate": 4.7177341408223994e-08,
      "logits/chosen": -2.077390193939209,
      "logits/rejected": -2.095350742340088,
      "logps/chosen": -0.3161494731903076,
      "logps/rejected": -0.3177900016307831,
      "loss": 1.4744199514389038,
      "loss/core": 1.4744199514389038,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.533155918121338,
      "rewards/margins": 0.720278799533844,
      "rewards/rejected": 1.8128772974014282,
      "step": 740
    },
    {
      "epoch": 0.7118117759464947,
      "grad_norm": 384.0,
      "learning_rate": 4.438122617983442e-08,
      "logits/chosen": -2.031482696533203,
      "logits/rejected": -2.0756478309631348,
      "logps/chosen": -0.29046431183815,
      "logps/rejected": -0.2840021848678589,
      "loss": 0.611270010471344,
      "loss/core": 0.611270010471344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.1211209297180176,
      "rewards/margins": 1.6258805990219116,
      "rewards/rejected": 1.4952404499053955,
      "step": 745
    },
    {
      "epoch": 0.7165890361877463,
      "grad_norm": 318.0,
      "learning_rate": 4.1662435763087114e-08,
      "logits/chosen": -2.16795015335083,
      "logits/rejected": -2.160215377807617,
      "logps/chosen": -0.3122718930244446,
      "logps/rejected": -0.3045649826526642,
      "loss": 0.5550835728645325,
      "loss/core": 0.5550835728645325,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.2003018856048584,
      "rewards/margins": -0.04670139029622078,
      "rewards/rejected": 1.2470033168792725,
      "step": 750
    },
    {
      "epoch": 0.721366296428998,
      "grad_norm": 23.125,
      "learning_rate": 3.902199258386732e-08,
      "logits/chosen": -1.905664086341858,
      "logits/rejected": -1.815201997756958,
      "logps/chosen": -0.3100651502609253,
      "logps/rejected": -0.2970584034919739,
      "loss": 0.3808279037475586,
      "loss/core": 0.3808279037475586,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.371333122253418,
      "rewards/margins": 0.689149796962738,
      "rewards/rejected": 1.6821835041046143,
      "step": 755
    },
    {
      "epoch": 0.7261435566702497,
      "grad_norm": 20.0,
      "learning_rate": 3.646088960486862e-08,
      "logits/chosen": -2.1599106788635254,
      "logits/rejected": -2.178335189819336,
      "logps/chosen": -0.3025417923927307,
      "logps/rejected": -0.2942703664302826,
      "loss": 0.27844661474227905,
      "loss/core": 0.27844661474227905,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9924026727676392,
      "rewards/margins": 1.1353952884674072,
      "rewards/rejected": 0.8570073843002319,
      "step": 760
    },
    {
      "epoch": 0.7309208169115012,
      "grad_norm": 41.25,
      "learning_rate": 3.398008995217988e-08,
      "logits/chosen": -2.05729603767395,
      "logits/rejected": -2.128960132598877,
      "logps/chosen": -0.2781466543674469,
      "logps/rejected": -0.2730472981929779,
      "loss": 0.23510298132896423,
      "loss/core": 0.23510298132896423,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.773431420326233,
      "rewards/margins": 0.7215307354927063,
      "rewards/rejected": 1.0519007444381714,
      "step": 765
    },
    {
      "epoch": 0.7356980771527529,
      "grad_norm": 50.25,
      "learning_rate": 3.1580526553093315e-08,
      "logits/chosen": -1.9602434635162354,
      "logits/rejected": -1.927735686302185,
      "logps/chosen": -0.31485968828201294,
      "logps/rejected": -0.28551191091537476,
      "loss": 2.1671090126037598,
      "loss/core": 2.1671090126037598,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.6825318336486816,
      "rewards/margins": 1.338300108909607,
      "rewards/rejected": 2.3442318439483643,
      "step": 770
    },
    {
      "epoch": 0.7404753373940045,
      "grad_norm": 1.8984375,
      "learning_rate": 2.9263101785268252e-08,
      "logits/chosen": -2.0744194984436035,
      "logits/rejected": -2.105100154876709,
      "logps/chosen": -0.3076358139514923,
      "logps/rejected": -0.3005218207836151,
      "loss": 0.14362755417823792,
      "loss/core": 0.14362755417823792,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5125480890274048,
      "rewards/margins": 0.5728277564048767,
      "rewards/rejected": 0.9397203326225281,
      "step": 775
    },
    {
      "epoch": 0.7452525976352562,
      "grad_norm": 19.125,
      "learning_rate": 2.7028687137384264e-08,
      "logits/chosen": -2.156768798828125,
      "logits/rejected": -2.170663356781006,
      "logps/chosen": -0.3037753999233246,
      "logps/rejected": -0.3085966110229492,
      "loss": 0.22171640396118164,
      "loss/core": 0.22171640396118164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8746483325958252,
      "rewards/margins": 0.6551548838615417,
      "rewards/rejected": 1.2194935083389282,
      "step": 780
    },
    {
      "epoch": 0.7500298578765078,
      "grad_norm": 1.6328125,
      "learning_rate": 2.4878122881409447e-08,
      "logits/chosen": -1.9704217910766602,
      "logits/rejected": -1.92739999294281,
      "logps/chosen": -0.3119404911994934,
      "logps/rejected": -0.2957368493080139,
      "loss": 1.2243354320526123,
      "loss/core": 1.2243354320526123,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 3.421541690826416,
      "rewards/margins": 2.0960593223571777,
      "rewards/rejected": 1.3254823684692383,
      "step": 785
    },
    {
      "epoch": 0.7548071181177595,
      "grad_norm": 244.0,
      "learning_rate": 2.2812217756608937e-08,
      "logits/chosen": -2.1468310356140137,
      "logits/rejected": -1.9746036529541016,
      "logps/chosen": -0.28714293241500854,
      "logps/rejected": -0.3162146210670471,
      "loss": 1.1724170446395874,
      "loss/core": 1.1724170446395874,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.608785390853882,
      "rewards/margins": 0.6583928465843201,
      "rewards/rejected": 1.9503923654556274,
      "step": 790
    },
    {
      "epoch": 0.7595843783590112,
      "grad_norm": 24.125,
      "learning_rate": 2.0831748665410763e-08,
      "logits/chosen": -1.9510393142700195,
      "logits/rejected": -1.968506097793579,
      "logps/chosen": -0.30975866317749023,
      "logps/rejected": -0.2976404130458832,
      "loss": 1.044623851776123,
      "loss/core": 1.044623851776123,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.357090473175049,
      "rewards/margins": 1.1862857341766357,
      "rewards/rejected": 1.1708046197891235,
      "step": 795
    },
    {
      "epoch": 0.7643616386002627,
      "grad_norm": 4.53125,
      "learning_rate": 1.8937460381244967e-08,
      "logits/chosen": -2.131096363067627,
      "logits/rejected": -2.1090807914733887,
      "logps/chosen": -0.2900455892086029,
      "logps/rejected": -0.28949636220932007,
      "loss": 0.3053150773048401,
      "loss/core": 0.3053150773048401,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.288881778717041,
      "rewards/margins": 1.1297084093093872,
      "rewards/rejected": 1.1591734886169434,
      "step": 800
    },
    {
      "epoch": 0.7691388988415144,
      "grad_norm": 9.5625,
      "learning_rate": 1.713006526846439e-08,
      "logits/chosen": -2.0039420127868652,
      "logits/rejected": -1.9931786060333252,
      "logps/chosen": -0.27616554498672485,
      "logps/rejected": -0.27139753103256226,
      "loss": 0.5071675181388855,
      "loss/core": 0.5071675181388855,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.244047164916992,
      "rewards/margins": 0.266120582818985,
      "rewards/rejected": 1.9779266119003296,
      "step": 805
    },
    {
      "epoch": 0.773916159082766,
      "grad_norm": 28.75,
      "learning_rate": 1.541024301445404e-08,
      "logits/chosen": -2.2073111534118652,
      "logits/rejected": -2.1122589111328125,
      "logps/chosen": -0.3092059791088104,
      "logps/rejected": -0.32349738478660583,
      "loss": 0.23964479565620422,
      "loss/core": 0.23964479565620422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.5212091207504272,
      "rewards/margins": 0.11138463020324707,
      "rewards/rejected": 1.409824252128601,
      "step": 810
    },
    {
      "epoch": 0.7786934193240177,
      "grad_norm": 35.0,
      "learning_rate": 1.3778640374027983e-08,
      "logits/chosen": -2.0743167400360107,
      "logits/rejected": -2.162884473800659,
      "logps/chosen": -0.2836160361766815,
      "logps/rejected": -0.28452104330062866,
      "loss": 0.24607639014720917,
      "loss/core": 0.24607639014720917,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.1251344680786133,
      "rewards/margins": 1.0335019826889038,
      "rewards/rejected": 1.09163236618042,
      "step": 815
    },
    {
      "epoch": 0.7834706795652693,
      "grad_norm": 3.296875,
      "learning_rate": 1.2235870926211616e-08,
      "logits/chosen": -2.079317569732666,
      "logits/rejected": -2.112313747406006,
      "logps/chosen": -0.288349449634552,
      "logps/rejected": -0.31876736879348755,
      "loss": 0.2414264678955078,
      "loss/core": 0.2414264678955078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9595178365707397,
      "rewards/margins": 1.2891201972961426,
      "rewards/rejected": 0.6703976988792419,
      "step": 820
    },
    {
      "epoch": 0.788247939806521,
      "grad_norm": 60.5,
      "learning_rate": 1.0782514843499652e-08,
      "logits/chosen": -2.1222786903381348,
      "logits/rejected": -2.2262613773345947,
      "logps/chosen": -0.30225175619125366,
      "logps/rejected": -0.2850930094718933,
      "loss": 0.308658629655838,
      "loss/core": 0.308658629655838,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.461728096008301,
      "rewards/margins": 1.250454068183899,
      "rewards/rejected": 1.2112740278244019,
      "step": 825
    },
    {
      "epoch": 0.7930252000477725,
      "grad_norm": 8.625,
      "learning_rate": 9.419118673676924e-09,
      "logits/chosen": -2.156327962875366,
      "logits/rejected": -2.1050965785980225,
      "logps/chosen": -0.2788023054599762,
      "logps/rejected": -0.3164371848106384,
      "loss": 0.30850037932395935,
      "loss/core": 0.30850037932395935,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7236570119857788,
      "rewards/margins": 0.15539640188217163,
      "rewards/rejected": 1.5682607889175415,
      "step": 830
    },
    {
      "epoch": 0.7978024602890242,
      "grad_norm": 173.0,
      "learning_rate": 8.14619513428405e-09,
      "logits/chosen": -2.0721898078918457,
      "logits/rejected": -2.071643590927124,
      "logps/chosen": -0.2557886242866516,
      "logps/rejected": -0.30318066477775574,
      "loss": 0.3867768943309784,
      "loss/core": 0.3867768943309784,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3934996128082275,
      "rewards/margins": 1.5836869478225708,
      "rewards/rejected": 0.8098125457763672,
      "step": 835
    },
    {
      "epoch": 0.8025797205302759,
      "grad_norm": 2672.0,
      "learning_rate": 6.96422291980539e-09,
      "logits/chosen": -2.0129635334014893,
      "logits/rejected": -2.0625948905944824,
      "logps/chosen": -0.32838812470436096,
      "logps/rejected": -0.3022240698337555,
      "loss": 0.8802303075790405,
      "loss/core": 0.8802303075790405,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.7439942359924316,
      "rewards/margins": 1.7150169610977173,
      "rewards/rejected": 1.0289772748947144,
      "step": 840
    },
    {
      "epoch": 0.8073569807715275,
      "grad_norm": 3.296875,
      "learning_rate": 5.8736465216517594e-09,
      "logits/chosen": -2.1509671211242676,
      "logits/rejected": -2.22468638420105,
      "logps/chosen": -0.29492026567459106,
      "logps/rejected": -0.3140087425708771,
      "loss": 0.15912021696567535,
      "loss/core": 0.15912021696567535,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.4049441814422607,
      "rewards/margins": 0.977611243724823,
      "rewards/rejected": 0.4273327887058258,
      "step": 845
    },
    {
      "epoch": 0.8121342410127792,
      "grad_norm": 37.0,
      "learning_rate": 4.874876061005173e-09,
      "logits/chosen": -2.2683005332946777,
      "logits/rejected": -2.3160834312438965,
      "logps/chosen": -0.25317704677581787,
      "logps/rejected": -0.2489752322435379,
      "loss": 0.0958312377333641,
      "loss/core": 0.0958312377333641,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.54629647731781,
      "rewards/margins": 0.6603418588638306,
      "rewards/rejected": 0.8859546780586243,
      "step": 850
    },
    {
      "epoch": 0.8169115012540308,
      "grad_norm": 59.5,
      "learning_rate": 3.968287134589188e-09,
      "logits/chosen": -1.9828602075576782,
      "logits/rejected": -2.0203537940979004,
      "logps/chosen": -0.30375003814697266,
      "logps/rejected": -0.2930416464805603,
      "loss": 0.8324758410453796,
      "loss/core": 0.8324758410453796,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.6595799922943115,
      "rewards/margins": 1.4085386991500854,
      "rewards/rejected": 1.251041293144226,
      "step": 855
    },
    {
      "epoch": 0.8216887614952825,
      "grad_norm": 2.515625,
      "learning_rate": 3.154220673422192e-09,
      "logits/chosen": -1.9286237955093384,
      "logits/rejected": -2.016119956970215,
      "logps/chosen": -0.3028337359428406,
      "logps/rejected": -0.30019161105155945,
      "loss": 1.6009376049041748,
      "loss/core": 1.6009376049041748,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.053446292877197,
      "rewards/margins": 2.1368956565856934,
      "rewards/rejected": 1.916550874710083,
      "step": 860
    },
    {
      "epoch": 0.826466021736534,
      "grad_norm": 51.25,
      "learning_rate": 2.4329828146074096e-09,
      "logits/chosen": -1.9394134283065796,
      "logits/rejected": -2.0103347301483154,
      "logps/chosen": -0.2917220890522003,
      "logps/rejected": -0.2960386276245117,
      "loss": 0.590233325958252,
      "loss/core": 0.590233325958252,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.443800926208496,
      "rewards/margins": 0.8796428442001343,
      "rewards/rejected": 1.5641582012176514,
      "step": 865
    },
    {
      "epoch": 0.8312432819777857,
      "grad_norm": 13.125,
      "learning_rate": 1.8048447862070714e-09,
      "logits/chosen": -2.090069532394409,
      "logits/rejected": -2.098034381866455,
      "logps/chosen": -0.3098277449607849,
      "logps/rejected": -0.29889780282974243,
      "loss": 1.0631767511367798,
      "loss/core": 1.0631767511367798,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.1978888511657715,
      "rewards/margins": 2.2793126106262207,
      "rewards/rejected": 0.9185762405395508,
      "step": 870
    },
    {
      "epoch": 0.8360205422190374,
      "grad_norm": 7.15625,
      "learning_rate": 1.2700428052447033e-09,
      "logits/chosen": -2.1596736907958984,
      "logits/rejected": -2.207904100418091,
      "logps/chosen": -0.30997827649116516,
      "logps/rejected": -0.3309651017189026,
      "loss": 0.7209604382514954,
      "loss/core": 0.7209604382514954,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8416748046875,
      "rewards/margins": 0.8194574117660522,
      "rewards/rejected": 1.0222175121307373,
      "step": 875
    },
    {
      "epoch": 0.840797802460289,
      "grad_norm": 62.25,
      "learning_rate": 8.287779888734858e-10,
      "logits/chosen": -2.0957977771759033,
      "logits/rejected": -2.079482078552246,
      "logps/chosen": -0.291581928730011,
      "logps/rejected": -0.3347880244255066,
      "loss": 0.9310186505317688,
      "loss/core": 0.9310186505317688,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.448652982711792,
      "rewards/margins": 0.716099739074707,
      "rewards/rejected": 1.7325531244277954,
      "step": 880
    },
    {
      "epoch": 0.8455750627015407,
      "grad_norm": 23.75,
      "learning_rate": 4.812162787445062e-10,
      "logits/chosen": -2.0210766792297363,
      "logits/rejected": -2.097437858581543,
      "logps/chosen": -0.28354987502098083,
      "logps/rejected": -0.27728238701820374,
      "loss": 0.42107605934143066,
      "loss/core": 0.42107605934143066,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.7150750160217285,
      "rewards/margins": 1.5944372415542603,
      "rewards/rejected": 1.1206376552581787,
      "step": 885
    },
    {
      "epoch": 0.8503523229427923,
      "grad_norm": 296.0,
      "learning_rate": 2.2748837860270265e-10,
      "logits/chosen": -1.9509280920028687,
      "logits/rejected": -2.072624921798706,
      "logps/chosen": -0.3031921684741974,
      "logps/rejected": -0.2749203145503998,
      "loss": 0.6249431371688843,
      "loss/core": 0.6249431371688843,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.4241182804107666,
      "rewards/margins": 0.9915217161178589,
      "rewards/rejected": 1.4325960874557495,
      "step": 890
    },
    {
      "epoch": 0.855129583184044,
      "grad_norm": 52.0,
      "learning_rate": 6.768970513457151e-11,
      "logits/chosen": -2.0221574306488037,
      "logits/rejected": -2.0542521476745605,
      "logps/chosen": -0.27334579825401306,
      "logps/rejected": -0.291655033826828,
      "loss": 0.3214346766471863,
      "loss/core": 0.3214346766471863,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.2918426990509033,
      "rewards/margins": 0.789661705493927,
      "rewards/rejected": 1.502180814743042,
      "step": 895
    },
    {
      "epoch": 0.8599068434252956,
      "grad_norm": 17.125,
      "learning_rate": 1.8803520859811406e-12,
      "logits/chosen": -2.0327274799346924,
      "logits/rejected": -2.152374744415283,
      "logps/chosen": -0.271484911441803,
      "logps/rejected": -0.2696631848812103,
      "loss": 0.9997545480728149,
      "loss/core": 0.9997545480728149,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 3.091799259185791,
      "rewards/margins": 2.2562103271484375,
      "rewards/rejected": 0.8355890512466431,
      "step": 900
    },
    {
      "epoch": 0.8599068434252956,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 0.7102717447943158,
      "train_runtime": 7093.1544,
      "train_samples_per_second": 2.03,
      "train_steps_per_second": 0.127
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}