{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8599068434252956,
  "eval_steps": 500,
  "global_step": 900,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004777260241251642,
      "grad_norm": 2.296875,
      "learning_rate": 2.222222222222222e-08,
      "logits/chosen": -2.3184335231781006,
      "logits/rejected": -2.304084300994873,
      "logps/chosen": -0.31140023469924927,
      "logps/rejected": -0.29028692841529846,
      "loss": 0.3946344256401062,
      "loss/core": 0.3946344256401062,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.209475040435791,
      "rewards/margins": 1.5164874792099,
      "rewards/rejected": 0.6929876208305359,
      "step": 5
    },
    {
      "epoch": 0.009554520482503284,
      "grad_norm": 183.0,
      "learning_rate": 5e-08,
      "logits/chosen": -2.064580202102661,
      "logits/rejected": -2.0450222492218018,
      "logps/chosen": -0.28674912452697754,
      "logps/rejected": -0.3005256652832031,
      "loss": 1.1264296770095825,
      "loss/core": 1.1264296770095825,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.9985833168029785,
      "rewards/margins": 0.33723002672195435,
      "rewards/rejected": 2.66135311126709,
      "step": 10
    },
    {
      "epoch": 0.014331780723754926,
      "grad_norm": 12.625,
      "learning_rate": 7.777777777777778e-08,
      "logits/chosen": -2.228989601135254,
      "logits/rejected": -2.2271804809570312,
      "logps/chosen": -0.2594456374645233,
      "logps/rejected": -0.2722489833831787,
      "loss": 0.26212042570114136,
      "loss/core": 0.26212042570114136,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.754608154296875,
      "rewards/margins": 0.7076910734176636,
      "rewards/rejected": 1.046917200088501,
      "step": 15
    },
    {
      "epoch": 0.01910904096500657,
      "grad_norm": 4.59375,
      "learning_rate": 1.0555555555555555e-07,
      "logits/chosen": -2.1681175231933594,
      "logits/rejected": -2.185380458831787,
      "logps/chosen": -0.2745749354362488,
      "logps/rejected": -0.26860952377319336,
      "loss": 1.016000747680664,
      "loss/core": 1.016000747680664,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.6250548362731934,
      "rewards/margins": 2.429143190383911,
      "rewards/rejected": 1.1959115266799927,
      "step": 20
    },
    {
      "epoch": 0.02388630120625821,
      "grad_norm": 4.40625,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -2.010061740875244,
      "logits/rejected": -2.1256415843963623,
      "logps/chosen": -0.27929872274398804,
      "logps/rejected": -0.2702064514160156,
      "loss": 0.17983713746070862,
      "loss/core": 0.17983713746070862,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.1464362144470215,
      "rewards/margins": 1.1600453853607178,
      "rewards/rejected": 0.9863905906677246,
      "step": 25
    },
    {
      "epoch": 0.028663561447509853,
      "grad_norm": 5.78125,
      "learning_rate": 1.611111111111111e-07,
      "logits/chosen": -2.1331052780151367,
      "logits/rejected": -2.1351206302642822,
      "logps/chosen": -0.27982744574546814,
      "logps/rejected": -0.26132339239120483,
      "loss": 0.5742586851119995,
      "loss/core": 0.5742586851119995,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.813447952270508,
      "rewards/margins": 1.750083327293396,
      "rewards/rejected": 1.0633647441864014,
      "step": 30
    },
    {
      "epoch": 0.033440821688761495,
      "grad_norm": 1.53125,
      "learning_rate": 1.8888888888888888e-07,
      "logits/chosen": -2.273294687271118,
      "logits/rejected": -2.2828454971313477,
      "logps/chosen": -0.2884656488895416,
      "logps/rejected": -0.2767728865146637,
      "loss": 0.8869408369064331,
      "loss/core": 0.8869408369064331,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.5600028038024902,
      "rewards/margins": 1.78105890750885,
      "rewards/rejected": 0.7789438366889954,
      "step": 35
    },
    {
      "epoch": 0.03821808193001314,
      "grad_norm": 27.75,
      "learning_rate": 2.1666666666666667e-07,
      "logits/chosen": -2.2540197372436523,
      "logits/rejected": -2.351133108139038,
      "logps/chosen": -0.2658698558807373,
      "logps/rejected": -0.2907714247703552,
      "loss": 0.6574953198432922,
      "loss/core": 0.6574953198432922,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.926403522491455,
      "rewards/margins": 2.233344554901123,
      "rewards/rejected": 0.6930587887763977,
      "step": 40
    },
    {
      "epoch": 0.04299534217126478,
      "grad_norm": 4.46875,
      "learning_rate": 2.4444444444444445e-07,
      "logits/chosen": -2.090873956680298,
      "logits/rejected": -2.14512300491333,
      "logps/chosen": -0.27041956782341003,
      "logps/rejected": -0.27646249532699585,
      "loss": 0.8163895606994629,
      "loss/core": 0.8163895606994629,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.0477185249328613,
      "rewards/margins": 1.7669315338134766,
      "rewards/rejected": 1.2807869911193848,
      "step": 45
    },
    {
      "epoch": 0.04777260241251642,
      "grad_norm": 66.0,
      "learning_rate": 2.7222222222222216e-07,
      "logits/chosen": -2.188749074935913,
      "logits/rejected": -2.2616398334503174,
      "logps/chosen": -0.2877681851387024,
      "logps/rejected": -0.26972633600234985,
      "loss": 0.21321019530296326,
      "loss/core": 0.21321019530296326,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6582177877426147,
      "rewards/margins": 0.44566529989242554,
      "rewards/rejected": 1.2125524282455444,
      "step": 50
    },
    {
      "epoch": 0.05254986265376806,
      "grad_norm": 236.0,
      "learning_rate": 3e-07,
      "logits/chosen": -2.0755667686462402,
      "logits/rejected": -2.1377971172332764,
      "logps/chosen": -0.28878724575042725,
      "logps/rejected": -0.28834742307662964,
      "loss": 0.5084811449050903,
      "loss/core": 0.5084811449050903,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.339972496032715,
      "rewards/margins": 1.6370515823364258,
      "rewards/rejected": 0.7029210329055786,
      "step": 55
    },
    {
      "epoch": 0.057327122895019705,
      "grad_norm": 121.0,
      "learning_rate": 3.2777777777777776e-07,
      "logits/chosen": -2.1755266189575195,
      "logits/rejected": -2.122792959213257,
      "logps/chosen": -0.2767593562602997,
      "logps/rejected": -0.2760574519634247,
      "loss": 1.5027045011520386,
      "loss/core": 1.5027045011520386,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.761892318725586,
      "rewards/margins": 2.0544276237487793,
      "rewards/rejected": 1.707464575767517,
      "step": 60
    },
    {
      "epoch": 0.06210438313627135,
      "grad_norm": 264.0,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -2.1052839756011963,
      "logits/rejected": -2.1663551330566406,
      "logps/chosen": -0.2984526455402374,
      "logps/rejected": -0.28494197130203247,
      "loss": 1.0875555276870728,
      "loss/core": 1.0875555276870728,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.2381019592285156,
      "rewards/margins": 2.0958991050720215,
      "rewards/rejected": 1.1422029733657837,
      "step": 65
    },
    {
      "epoch": 0.06688164337752299,
      "grad_norm": 16.0,
      "learning_rate": 3.8333333333333335e-07,
      "logits/chosen": -2.134707450866699,
      "logits/rejected": -2.046159267425537,
      "logps/chosen": -0.2798652648925781,
      "logps/rejected": -0.27488410472869873,
      "loss": 0.4238152503967285,
      "loss/core": 0.4238152503967285,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.4914584159851074,
      "rewards/margins": 1.004212498664856,
      "rewards/rejected": 1.4872459173202515,
      "step": 70
    },
    {
      "epoch": 0.07165890361877464,
      "grad_norm": 1.65625,
      "learning_rate": 4.1111111111111107e-07,
      "logits/chosen": -2.0351903438568115,
      "logits/rejected": -2.068025588989258,
      "logps/chosen": -0.29988136887550354,
      "logps/rejected": -0.3065613806247711,
      "loss": 0.5185372829437256,
      "loss/core": 0.5185372829437256,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2677860260009766,
      "rewards/margins": 1.2009799480438232,
      "rewards/rejected": 1.0668058395385742,
      "step": 75
    },
    {
      "epoch": 0.07643616386002627,
      "grad_norm": 0.8359375,
      "learning_rate": 4.3888888888888884e-07,
      "logits/chosen": -2.2651379108428955,
      "logits/rejected": -2.265864849090576,
      "logps/chosen": -0.2954421639442444,
      "logps/rejected": -0.2877275347709656,
      "loss": 0.18425166606903076,
      "loss/core": 0.18425166606903076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3546873331069946,
      "rewards/margins": 0.4060265123844147,
      "rewards/rejected": 0.9486608505249023,
      "step": 80
    },
    {
      "epoch": 0.08121342410127792,
      "grad_norm": 744.0,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -2.2166881561279297,
      "logits/rejected": -2.2303507328033447,
      "logps/chosen": -0.27470141649246216,
      "logps/rejected": -0.2800918519496918,
      "loss": 0.5587457418441772,
      "loss/core": 0.5587457418441772,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6758487224578857,
      "rewards/margins": 0.03965097665786743,
      "rewards/rejected": 1.6361976861953735,
      "step": 85
    },
    {
      "epoch": 0.08599068434252956,
      "grad_norm": 15.6875,
      "learning_rate": 4.944444444444445e-07,
      "logits/chosen": -2.1294827461242676,
      "logits/rejected": -2.2263193130493164,
      "logps/chosen": -0.27022016048431396,
      "logps/rejected": -0.31300783157348633,
      "loss": 0.8455947041511536,
      "loss/core": 0.8455947041511536,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.5631749629974365,
      "rewards/margins": 1.0474013090133667,
      "rewards/rejected": 1.515773892402649,
      "step": 90
    },
    {
      "epoch": 0.09076794458378121,
      "grad_norm": 22.0,
      "learning_rate": 4.999699149323369e-07,
      "logits/chosen": -2.137500524520874,
      "logits/rejected": -2.064415454864502,
      "logps/chosen": -0.28990474343299866,
      "logps/rejected": -0.2970162034034729,
      "loss": 0.4857604503631592,
      "loss/core": 0.4857604503631592,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.2846360206604004,
      "rewards/margins": 1.2684682607650757,
      "rewards/rejected": 1.0161678791046143,
      "step": 95
    },
    {
      "epoch": 0.09554520482503284,
      "grad_norm": 4.4375,
      "learning_rate": 4.998477067547739e-07,
      "logits/chosen": -2.11228346824646,
      "logits/rejected": -2.1553733348846436,
      "logps/chosen": -0.26913318037986755,
      "logps/rejected": -0.2833011746406555,
      "loss": 1.1113026142120361,
      "loss/core": 1.1113026142120361,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.711238145828247,
      "rewards/margins": 2.09954833984375,
      "rewards/rejected": 1.611689567565918,
      "step": 100
    },
    {
      "epoch": 0.10032246506628449,
      "grad_norm": 24.25,
      "learning_rate": 4.996315410727229e-07,
      "logits/chosen": -2.0640909671783447,
      "logits/rejected": -2.0353593826293945,
      "logps/chosen": -0.317560613155365,
      "logps/rejected": -0.304330050945282,
      "loss": 0.93501216173172,
      "loss/core": 0.93501216173172,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.3537240028381348,
      "rewards/margins": 2.5598361492156982,
      "rewards/rejected": 0.7938874959945679,
      "step": 105
    },
    {
      "epoch": 0.10509972530753613,
      "grad_norm": 1.7890625,
      "learning_rate": 4.993214991772562e-07,
      "logits/chosen": -2.1243863105773926,
      "logits/rejected": -2.126587390899658,
      "logps/chosen": -0.2978929877281189,
      "logps/rejected": -0.3149787485599518,
      "loss": 0.20066983997821808,
      "loss/core": 0.20066983997821808,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6585355997085571,
      "rewards/margins": 0.7344213724136353,
      "rewards/rejected": 0.9241144061088562,
      "step": 110
    },
    {
      "epoch": 0.10987698554878778,
      "grad_norm": 10.8125,
      "learning_rate": 4.989176976624511e-07,
      "logits/chosen": -2.0023255348205566,
      "logits/rejected": -2.1182520389556885,
      "logps/chosen": -0.30129915475845337,
      "logps/rejected": -0.29388052225112915,
      "loss": 1.1868757009506226,
      "loss/core": 1.1868757009506226,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8971168994903564,
      "rewards/margins": 1.7273696660995483,
      "rewards/rejected": 1.1697468757629395,
      "step": 115
    },
    {
      "epoch": 0.11465424579003941,
      "grad_norm": 51.0,
      "learning_rate": 4.984202883815428e-07,
      "logits/chosen": -2.2406866550445557,
      "logits/rejected": -2.252915620803833,
      "logps/chosen": -0.30535244941711426,
      "logps/rejected": -0.30678510665893555,
      "loss": 0.48086509108543396,
      "loss/core": 0.48086509108543396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3497025966644287,
      "rewards/margins": 1.1640708446502686,
      "rewards/rejected": 1.1856318712234497,
      "step": 120
    },
    {
      "epoch": 0.11943150603129106,
      "grad_norm": 87.0,
      "learning_rate": 4.978294583898195e-07,
      "logits/chosen": -2.0294809341430664,
      "logits/rejected": -1.9394248723983765,
      "logps/chosen": -0.29434821009635925,
      "logps/rejected": -0.3001564145088196,
      "loss": 0.9685723185539246,
      "loss/core": 0.9685723185539246,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.8707126379013062,
      "rewards/margins": -0.364132821559906,
      "rewards/rejected": 2.2348453998565674,
      "step": 125
    },
    {
      "epoch": 0.1242087662725427,
      "grad_norm": 58.5,
      "learning_rate": 4.971454298742779e-07,
      "logits/chosen": -2.348689079284668,
      "logits/rejected": -2.265820026397705,
      "logps/chosen": -0.27629876136779785,
      "logps/rejected": -0.302666574716568,
      "loss": 0.6233381032943726,
      "loss/core": 0.6233381032943726,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.2184367179870605,
      "rewards/margins": 0.2606551945209503,
      "rewards/rejected": 1.9577815532684326,
      "step": 130
    },
    {
      "epoch": 0.12898602651379434,
      "grad_norm": 11.0625,
      "learning_rate": 4.963684600700678e-07,
      "logits/chosen": -2.0810961723327637,
      "logits/rejected": -2.177532434463501,
      "logps/chosen": -0.34504374861717224,
      "logps/rejected": -0.3136570155620575,
      "loss": 0.6652131676673889,
      "loss/core": 0.6652131676673889,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.7010741233825684,
      "rewards/margins": 1.2287189960479736,
      "rewards/rejected": 1.4723551273345947,
      "step": 135
    },
    {
      "epoch": 0.13376328675504598,
      "grad_norm": 57.75,
      "learning_rate": 4.954988411637571e-07,
      "logits/chosen": -1.9434146881103516,
      "logits/rejected": -1.9310106039047241,
      "logps/chosen": -0.3319775462150574,
      "logps/rejected": -0.34657081961631775,
      "loss": 1.2185124158859253,
      "loss/core": 1.2185124158859253,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.100647449493408,
      "rewards/margins": 0.7573388814926147,
      "rewards/rejected": 2.343308925628662,
      "step": 140
    },
    {
      "epoch": 0.13854054699629761,
      "grad_norm": 6.4375,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -2.0375614166259766,
      "logits/rejected": -2.0471043586730957,
      "logps/chosen": -0.3008027672767639,
      "logps/rejected": -0.28672724962234497,
      "loss": 0.20575180649757385,
      "loss/core": 0.20575180649757385,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7885154485702515,
      "rewards/margins": 0.952106773853302,
      "rewards/rejected": 0.8364084362983704,
      "step": 145
    },
    {
      "epoch": 0.14331780723754928,
      "grad_norm": 211.0,
      "learning_rate": 4.93482998875813e-07,
      "logits/chosen": -2.004568576812744,
      "logits/rejected": -2.028262138366699,
      "logps/chosen": -0.2854091227054596,
      "logps/rejected": -0.27151355147361755,
      "loss": 0.6710211038589478,
      "loss/core": 0.6710211038589478,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.637166738510132,
      "rewards/margins": 1.3770954608917236,
      "rewards/rejected": 1.2600712776184082,
      "step": 150
    },
    {
      "epoch": 0.1480950674788009,
      "grad_norm": 5.90625,
      "learning_rate": 4.923375335700223e-07,
      "logits/chosen": -1.968898057937622,
      "logits/rejected": -2.025012731552124,
      "logps/chosen": -0.2497386932373047,
      "logps/rejected": -0.2664481997489929,
      "loss": 0.38974815607070923,
      "loss/core": 0.38974815607070923,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.618589401245117,
      "rewards/margins": 1.195473551750183,
      "rewards/rejected": 1.423115849494934,
      "step": 155
    },
    {
      "epoch": 0.15287232772005255,
      "grad_norm": 58.5,
      "learning_rate": 4.911009350287347e-07,
      "logits/chosen": -2.1655495166778564,
      "logits/rejected": -2.139359951019287,
      "logps/chosen": -0.27988573908805847,
      "logps/rejected": -0.2946188449859619,
      "loss": 0.5953173041343689,
      "loss/core": 0.5953173041343689,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.644726276397705,
      "rewards/margins": 1.5004968643188477,
      "rewards/rejected": 1.144229531288147,
      "step": 160
    },
    {
      "epoch": 0.15764958796130418,
      "grad_norm": 5.3125,
      "learning_rate": 4.897736682860885e-07,
      "logits/chosen": -2.109489917755127,
      "logits/rejected": -2.0843923091888428,
      "logps/chosen": -0.3071993589401245,
      "logps/rejected": -0.30547797679901123,
      "loss": 0.1960802972316742,
      "loss/core": 0.1960802972316742,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.7830066680908203,
      "rewards/margins": 0.7959617376327515,
      "rewards/rejected": 0.9870448112487793,
      "step": 165
    },
    {
      "epoch": 0.16242684820255585,
      "grad_norm": 1.2578125,
      "learning_rate": 4.883562324728241e-07,
      "logits/chosen": -2.0669140815734863,
      "logits/rejected": -1.9952644109725952,
      "logps/chosen": -0.3106909692287445,
      "logps/rejected": -0.3281523883342743,
      "loss": 0.20468628406524658,
      "loss/core": 0.20468628406524658,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8208732604980469,
      "rewards/margins": 1.240056037902832,
      "rewards/rejected": 0.5808171033859253,
      "step": 170
    },
    {
      "epoch": 0.16720410844380748,
      "grad_norm": 70.5,
      "learning_rate": 4.868491606285823e-07,
      "logits/chosen": -2.2352006435394287,
      "logits/rejected": -2.256232261657715,
      "logps/chosen": -0.27431878447532654,
      "logps/rejected": -0.28181517124176025,
      "loss": 0.1919003427028656,
      "loss/core": 0.1919003427028656,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.784128189086914,
      "rewards/margins": 1.083038568496704,
      "rewards/rejected": 0.7010896801948547,
      "step": 175
    },
    {
      "epoch": 0.17198136868505912,
      "grad_norm": 14.6875,
      "learning_rate": 4.852530195014489e-07,
      "logits/chosen": -2.1959755420684814,
      "logits/rejected": -2.148374080657959,
      "logps/chosen": -0.2624792456626892,
      "logps/rejected": -0.2835710942745209,
      "loss": 0.6657207608222961,
      "loss/core": 0.6657207608222961,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.457181692123413,
      "rewards/margins": 1.1070095300674438,
      "rewards/rejected": 1.3501724004745483,
      "step": 180
    },
    {
      "epoch": 0.17675862892631075,
      "grad_norm": 632.0,
      "learning_rate": 4.835684093348244e-07,
      "logits/chosen": -2.2028822898864746,
      "logits/rejected": -2.2440648078918457,
      "logps/chosen": -0.3046368956565857,
      "logps/rejected": -0.3071895241737366,
      "loss": 0.6476051807403564,
      "loss/core": 0.6476051807403564,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.8896329402923584,
      "rewards/margins": 0.37541091442108154,
      "rewards/rejected": 1.5142220258712769,
      "step": 185
    },
    {
      "epoch": 0.18153588916756241,
      "grad_norm": 436.0,
      "learning_rate": 4.817959636416969e-07,
      "logits/chosen": -2.134538412094116,
      "logits/rejected": -2.1165084838867188,
      "logps/chosen": -0.2866383194923401,
      "logps/rejected": -0.27144110202789307,
      "loss": 0.6611530780792236,
      "loss/core": 0.6611530780792236,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.6436305046081543,
      "rewards/margins": 1.4646612405776978,
      "rewards/rejected": 1.178969144821167,
      "step": 190
    },
    {
      "epoch": 0.18631314940881405,
      "grad_norm": 6.3125,
      "learning_rate": 4.799363489664039e-07,
      "logits/chosen": -1.9559043645858765,
      "logits/rejected": -1.9407514333724976,
      "logps/chosen": -0.28489893674850464,
      "logps/rejected": -0.2946854531764984,
      "loss": 0.2528533637523651,
      "loss/core": 0.2528533637523651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.681348204612732,
      "rewards/margins": 0.3501024842262268,
      "rewards/rejected": 1.33124577999115,
      "step": 195
    },
    {
      "epoch": 0.19109040965006568,
      "grad_norm": 8.6875,
      "learning_rate": 4.779902646339721e-07,
      "logits/chosen": -2.1071603298187256,
      "logits/rejected": -2.1380481719970703,
      "logps/chosen": -0.2741830050945282,
      "logps/rejected": -0.27496692538261414,
      "loss": 0.46219339966773987,
      "loss/core": 0.46219339966773987,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.5044608116149902,
      "rewards/margins": 1.1724002361297607,
      "rewards/rejected": 1.33206045627594,
      "step": 200
    },
    {
      "epoch": 0.19586766989131732,
      "grad_norm": 3.203125,
      "learning_rate": 4.759584424871301e-07,
      "logits/chosen": -2.0774598121643066,
      "logits/rejected": -2.142284631729126,
      "logps/chosen": -0.2829883396625519,
      "logps/rejected": -0.29093074798583984,
      "loss": 0.08258922398090363,
      "loss/core": 0.08258922398090363,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.3614203929901123,
      "rewards/margins": 0.7309275269508362,
      "rewards/rejected": 0.6304928660392761,
      "step": 205
    },
    {
      "epoch": 0.20064493013256898,
      "grad_norm": 10.9375,
      "learning_rate": 4.738416466110917e-07,
      "logits/chosen": -2.0302059650421143,
      "logits/rejected": -2.0086560249328613,
      "logps/chosen": -0.288141131401062,
      "logps/rejected": -0.2887657582759857,
      "loss": 0.8564616441726685,
      "loss/core": 0.8564616441726685,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.526529550552368,
      "rewards/margins": 1.0289502143859863,
      "rewards/rejected": 1.4975792169570923,
      "step": 210
    },
    {
      "epoch": 0.20542219037382062,
      "grad_norm": 10.8125,
      "learning_rate": 4.716406730462153e-07,
      "logits/chosen": -1.9449901580810547,
      "logits/rejected": -1.9634069204330444,
      "logps/chosen": -0.29736560583114624,
      "logps/rejected": -0.2745818793773651,
      "loss": 0.24143390357494354,
      "loss/core": 0.24143390357494354,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.0462050437927246,
      "rewards/margins": 1.0101492404937744,
      "rewards/rejected": 1.0360558032989502,
      "step": 215
    },
    {
      "epoch": 0.21019945061507225,
      "grad_norm": 3.0,
      "learning_rate": 4.693563494886454e-07,
      "logits/chosen": -2.025031089782715,
      "logits/rejected": -2.0499961376190186,
      "logps/chosen": -0.28308767080307007,
      "logps/rejected": -0.27393776178359985,
      "loss": 0.16485320031642914,
      "loss/core": 0.16485320031642914,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.761115312576294,
      "rewards/margins": 0.9114633798599243,
      "rewards/rejected": 0.849652111530304,
      "step": 220
    },
    {
      "epoch": 0.2149767108563239,
      "grad_norm": 15.3125,
      "learning_rate": 4.6698953497905016e-07,
      "logits/chosen": -2.0364017486572266,
      "logits/rejected": -2.0577664375305176,
      "logps/chosen": -0.31471285223960876,
      "logps/rejected": -0.30929452180862427,
      "loss": 1.460294485092163,
      "loss/core": 1.460294485092163,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.0360703468322754,
      "rewards/margins": 1.0577287673950195,
      "rewards/rejected": 1.9783413410186768,
      "step": 225
    },
    {
      "epoch": 0.21975397109757555,
      "grad_norm": 54.5,
      "learning_rate": 4.645411195795709e-07,
      "logits/chosen": -2.118978500366211,
      "logits/rejected": -2.1093850135803223,
      "logps/chosen": -0.27341052889823914,
      "logps/rejected": -0.26687008142471313,
      "loss": 0.740382194519043,
      "loss/core": 0.740382194519043,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.039541006088257,
      "rewards/margins": 1.364072561264038,
      "rewards/rejected": 1.6754686832427979,
      "step": 230
    },
    {
      "epoch": 0.2245312313388272,
      "grad_norm": 2.84375,
      "learning_rate": 4.6201202403910643e-07,
      "logits/chosen": -2.160707950592041,
      "logits/rejected": -2.255814790725708,
      "logps/chosen": -0.28612154722213745,
      "logps/rejected": -0.28075844049453735,
      "loss": 0.19919511675834656,
      "loss/core": 0.19919511675834656,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.6981313228607178,
      "rewards/margins": 1.023926854133606,
      "rewards/rejected": 0.6742045879364014,
      "step": 235
    },
    {
      "epoch": 0.22930849158007882,
      "grad_norm": 7.90625,
      "learning_rate": 4.594031994470573e-07,
      "logits/chosen": -2.0647361278533936,
      "logits/rejected": -2.1183581352233887,
      "logps/chosen": -0.2742571234703064,
      "logps/rejected": -0.2913045287132263,
      "loss": 1.243248701095581,
      "loss/core": 1.243248701095581,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.7567174434661865,
      "rewards/margins": 0.8206822276115417,
      "rewards/rejected": 1.936035394668579,
      "step": 240
    },
    {
      "epoch": 0.23408575182133046,
      "grad_norm": 1112.0,
      "learning_rate": 4.567156268756593e-07,
      "logits/chosen": -2.120326519012451,
      "logits/rejected": -2.2658631801605225,
      "logps/chosen": -0.30770888924598694,
      "logps/rejected": -0.292914479970932,
      "loss": 0.6964072585105896,
      "loss/core": 0.6964072585105896,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.1233694553375244,
      "rewards/margins": 1.4673677682876587,
      "rewards/rejected": 0.6560018062591553,
      "step": 245
    },
    {
      "epoch": 0.23886301206258212,
      "grad_norm": 10.625,
      "learning_rate": 4.5395031701104303e-07,
      "logits/chosen": -1.9557708501815796,
      "logits/rejected": -1.9600833654403687,
      "logps/chosen": -0.30549654364585876,
      "logps/rejected": -0.3048430383205414,
      "loss": 1.1260910034179688,
      "loss/core": 1.1260910034179688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.3460769653320312,
      "rewards/margins": 1.7832248210906982,
      "rewards/rejected": 1.562852144241333,
      "step": 250
    },
    {
      "epoch": 0.24364027230383375,
      "grad_norm": 4.09375,
      "learning_rate": 4.511083097731555e-07,
      "logits/chosen": -2.1089088916778564,
      "logits/rejected": -2.0778651237487793,
      "logps/chosen": -0.287789523601532,
      "logps/rejected": -0.2891516089439392,
      "loss": 1.2482858896255493,
      "loss/core": 1.2482858896255493,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.746084213256836,
      "rewards/margins": 1.564002275466919,
      "rewards/rejected": 1.1820822954177856,
      "step": 255
    },
    {
      "epoch": 0.2484175325450854,
      "grad_norm": 108.0,
      "learning_rate": 4.481906739246894e-07,
      "logits/chosen": -2.062340259552002,
      "logits/rejected": -2.0389857292175293,
      "logps/chosen": -0.3061924874782562,
      "logps/rejected": -0.30993199348449707,
      "loss": 0.4140895903110504,
      "loss/core": 0.4140895903110504,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.9362844228744507,
      "rewards/margins": 0.6997807025909424,
      "rewards/rejected": 1.2365038394927979,
      "step": 260
    },
    {
      "epoch": 0.25319479278633705,
      "grad_norm": 2.8125,
      "learning_rate": 4.451985066691648e-07,
      "logits/chosen": -2.0385899543762207,
      "logits/rejected": -2.0162370204925537,
      "logps/chosen": -0.27738499641418457,
      "logps/rejected": -0.28945407271385193,
      "loss": 0.23780469596385956,
      "loss/core": 0.23780469596385956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.11801815032959,
      "rewards/margins": 1.098084568977356,
      "rewards/rejected": 1.0199334621429443,
      "step": 265
    },
    {
      "epoch": 0.2579720530275887,
      "grad_norm": 1.4375,
      "learning_rate": 4.421329332383158e-07,
      "logits/chosen": -2.0890111923217773,
      "logits/rejected": -2.0384275913238525,
      "logps/chosen": -0.30751222372055054,
      "logps/rejected": -0.30553263425827026,
      "loss": 0.3220127820968628,
      "loss/core": 0.3220127820968628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6447738409042358,
      "rewards/margins": 0.24662013351917267,
      "rewards/rejected": 1.3981537818908691,
      "step": 270
    },
    {
      "epoch": 0.2627493132688403,
      "grad_norm": 3.703125,
      "learning_rate": 4.3899510646893696e-07,
      "logits/chosen": -2.207843780517578,
      "logits/rejected": -2.2784366607666016,
      "logps/chosen": -0.2971876561641693,
      "logps/rejected": -0.2956683933734894,
      "loss": 0.05450323969125748,
      "loss/core": 0.05450323969125748,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0277817249298096,
      "rewards/margins": 0.44771966338157654,
      "rewards/rejected": 0.5800620317459106,
      "step": 275
    },
    {
      "epoch": 0.26752657351009196,
      "grad_norm": 23.0,
      "learning_rate": 4.357862063693485e-07,
      "logits/chosen": -2.0210609436035156,
      "logits/rejected": -1.9996906518936157,
      "logps/chosen": -0.3016383945941925,
      "logps/rejected": -0.3017963767051697,
      "loss": 0.21795885264873505,
      "loss/core": 0.21795885264873505,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8103902339935303,
      "rewards/margins": 0.9846366047859192,
      "rewards/rejected": 0.825753390789032,
      "step": 280
    },
    {
      "epoch": 0.2723038337513436,
      "grad_norm": 8.5625,
      "learning_rate": 4.3250743967564364e-07,
      "logits/chosen": -2.2049059867858887,
      "logits/rejected": -2.1598610877990723,
      "logps/chosen": -0.2876811623573303,
      "logps/rejected": -0.2909473478794098,
      "loss": 1.0883954763412476,
      "loss/core": 1.0883954763412476,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.9615416526794434,
      "rewards/margins": 1.0239375829696655,
      "rewards/rejected": 1.9376041889190674,
      "step": 285
    },
    {
      "epoch": 0.27708109399259523,
      "grad_norm": 1056.0,
      "learning_rate": 4.29160039397884e-07,
      "logits/chosen": -2.1312131881713867,
      "logits/rejected": -2.1842284202575684,
      "logps/chosen": -0.2860090136528015,
      "logps/rejected": -0.2702611982822418,
      "loss": 2.010714292526245,
      "loss/core": 2.010714292526245,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 4.503174781799316,
      "rewards/margins": 3.201117992401123,
      "rewards/rejected": 1.3020570278167725,
      "step": 290
    },
    {
      "epoch": 0.28185835423384686,
      "grad_norm": 10.0625,
      "learning_rate": 4.2574526435641546e-07,
      "logits/chosen": -1.899706244468689,
      "logits/rejected": -1.976220726966858,
      "logps/chosen": -0.3136875629425049,
      "logps/rejected": -0.302241712808609,
      "loss": 1.1247197389602661,
      "loss/core": 1.1247197389602661,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.7293989658355713,
      "rewards/margins": 1.6034202575683594,
      "rewards/rejected": 1.125978708267212,
      "step": 295
    },
    {
      "epoch": 0.28663561447509855,
      "grad_norm": 5.375,
      "learning_rate": 4.22264398708477e-07,
      "logits/chosen": -2.0837273597717285,
      "logits/rejected": -2.1485800743103027,
      "logps/chosen": -0.28113582730293274,
      "logps/rejected": -0.3067568838596344,
      "loss": 2.029283046722412,
      "loss/core": 2.029283046722412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.4929652214050293,
      "rewards/margins": 1.580517053604126,
      "rewards/rejected": 1.9124481678009033,
      "step": 300
    },
    {
      "epoch": 0.2914128747163502,
      "grad_norm": 45.25,
      "learning_rate": 4.187187514652819e-07,
      "logits/chosen": -1.9658424854278564,
      "logits/rejected": -2.029053211212158,
      "logps/chosen": -0.3088286519050598,
      "logps/rejected": -0.30893000960350037,
      "loss": 2.0511975288391113,
      "loss/core": 2.0511975288391113,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.7278835773468018,
      "rewards/margins": 1.2690778970718384,
      "rewards/rejected": 2.458805561065674,
      "step": 305
    },
    {
      "epoch": 0.2961901349576018,
      "grad_norm": 32.0,
      "learning_rate": 4.151096559997519e-07,
      "logits/chosen": -1.9535623788833618,
      "logits/rejected": -1.9904283285140991,
      "logps/chosen": -0.2780826687812805,
      "logps/rejected": -0.2801373302936554,
      "loss": 0.36180371046066284,
      "loss/core": 0.36180371046066284,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.327801465988159,
      "rewards/margins": 0.9045225977897644,
      "rewards/rejected": 1.423278570175171,
      "step": 310
    },
    {
      "epoch": 0.30096739519885346,
      "grad_norm": 29.375,
      "learning_rate": 4.114384695450905e-07,
      "logits/chosen": -2.104870080947876,
      "logits/rejected": -2.0882060527801514,
      "logps/chosen": -0.3096676766872406,
      "logps/rejected": -0.303961843252182,
      "loss": 0.32151490449905396,
      "loss/core": 0.32151490449905396,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.1817171573638916,
      "rewards/margins": -0.025614654645323753,
      "rewards/rejected": 1.207331895828247,
      "step": 315
    },
    {
      "epoch": 0.3057446554401051,
      "grad_norm": 812.0,
      "learning_rate": 4.077065726843828e-07,
      "logits/chosen": -1.9013550281524658,
      "logits/rejected": -1.8669452667236328,
      "logps/chosen": -0.3241393268108368,
      "logps/rejected": -0.3211979269981384,
      "loss": 1.0354502201080322,
      "loss/core": 1.0354502201080322,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.091705799102783,
      "rewards/margins": 1.3894522190093994,
      "rewards/rejected": 1.7022536993026733,
      "step": 320
    },
    {
      "epoch": 0.31052191568135673,
      "grad_norm": 1.5234375,
      "learning_rate": 4.039153688314145e-07,
      "logits/chosen": -1.993546485900879,
      "logits/rejected": -2.022549867630005,
      "logps/chosen": -0.28880229592323303,
      "logps/rejected": -0.2948676645755768,
      "loss": 0.12079276889562607,
      "loss/core": 0.12079276889562607,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.4271008968353271,
      "rewards/margins": 0.6435669660568237,
      "rewards/rejected": 0.7835339903831482,
      "step": 325
    },
    {
      "epoch": 0.31529917592260837,
      "grad_norm": 836.0,
      "learning_rate": 4.0006628370290613e-07,
      "logits/chosen": -2.167954444885254,
      "logits/rejected": -2.0930395126342773,
      "logps/chosen": -0.3180980980396271,
      "logps/rejected": -0.3477565050125122,
      "loss": 0.3206109404563904,
      "loss/core": 0.3206109404563904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3264482021331787,
      "rewards/margins": 0.24738340079784393,
      "rewards/rejected": 1.0790647268295288,
      "step": 330
    },
    {
      "epoch": 0.32007643616386,
      "grad_norm": 3.53125,
      "learning_rate": 3.9616076478235826e-07,
      "logits/chosen": -2.2583556175231934,
      "logits/rejected": -2.293337106704712,
      "logps/chosen": -0.30557915568351746,
      "logps/rejected": -0.30756452679634094,
      "loss": 0.0511607900261879,
      "loss/core": 0.0511607900261879,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.0766465663909912,
      "rewards/margins": 0.5183663964271545,
      "rewards/rejected": 0.5582801699638367,
      "step": 335
    },
    {
      "epoch": 0.3248536964051117,
      "grad_norm": 5.65625,
      "learning_rate": 3.922002807757129e-07,
      "logits/chosen": -2.2513046264648438,
      "logits/rejected": -2.259812593460083,
      "logps/chosen": -0.28524649143218994,
      "logps/rejected": -0.283276230096817,
      "loss": 0.631820797920227,
      "loss/core": 0.631820797920227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1910934448242188,
      "rewards/margins": 1.2970854043960571,
      "rewards/rejected": 0.8940078616142273,
      "step": 340
    },
    {
      "epoch": 0.3296309566463633,
      "grad_norm": 79.0,
      "learning_rate": 3.8818632105903315e-07,
      "logits/chosen": -2.1031250953674316,
      "logits/rejected": -2.102226734161377,
      "logps/chosen": -0.26754727959632874,
      "logps/rejected": -0.28109580278396606,
      "loss": 0.7250244617462158,
      "loss/core": 0.7250244617462158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1810359954833984,
      "rewards/margins": 0.5469441413879395,
      "rewards/rejected": 1.6340919733047485,
      "step": 345
    },
    {
      "epoch": 0.33440821688761496,
      "grad_norm": 11.6875,
      "learning_rate": 3.841203951184094e-07,
      "logits/chosen": -2.2044973373413086,
      "logits/rejected": -2.2186226844787598,
      "logps/chosen": -0.3000710904598236,
      "logps/rejected": -0.30985429883003235,
      "loss": 0.6811296939849854,
      "loss/core": 0.6811296939849854,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.313596487045288,
      "rewards/margins": 0.9291432499885559,
      "rewards/rejected": 1.3844534158706665,
      "step": 350
    },
    {
      "epoch": 0.3391854771288666,
      "grad_norm": 31.25,
      "learning_rate": 3.800040319823038e-07,
      "logits/chosen": -2.172675848007202,
      "logits/rejected": -2.1876015663146973,
      "logps/chosen": -0.291890025138855,
      "logps/rejected": -0.2729564905166626,
      "loss": 0.8517450094223022,
      "loss/core": 0.8517450094223022,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.446463108062744,
      "rewards/margins": 1.6567842960357666,
      "rewards/rejected": 1.7896785736083984,
      "step": 355
    },
    {
      "epoch": 0.34396273737011823,
      "grad_norm": 89.0,
      "learning_rate": 3.75838779646545e-07,
      "logits/chosen": -1.9489272832870483,
      "logits/rejected": -1.9853594303131104,
      "logps/chosen": -0.2767919600009918,
      "logps/rejected": -0.279407799243927,
      "loss": 0.7631909251213074,
      "loss/core": 0.7631909251213074,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.232623338699341,
      "rewards/margins": 1.9174954891204834,
      "rewards/rejected": 1.3151280879974365,
      "step": 360
    },
    {
      "epoch": 0.34873999761136987,
      "grad_norm": 40.5,
      "learning_rate": 3.7162620449218993e-07,
      "logits/chosen": -2.235380172729492,
      "logits/rejected": -2.306373119354248,
      "logps/chosen": -0.3116162121295929,
      "logps/rejected": -0.306151807308197,
      "loss": 0.5377756357192993,
      "loss/core": 0.5377756357192993,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.417541265487671,
      "rewards/margins": 1.4117828607559204,
      "rewards/rejected": 1.0057584047317505,
      "step": 365
    },
    {
      "epoch": 0.3535172578526215,
      "grad_norm": 22.0,
      "learning_rate": 3.673678906964727e-07,
      "logits/chosen": -2.117574691772461,
      "logits/rejected": -2.0456650257110596,
      "logps/chosen": -0.2936505675315857,
      "logps/rejected": -0.30493101477622986,
      "loss": 0.7472105622291565,
      "loss/core": 0.7472105622291565,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.3911571502685547,
      "rewards/margins": 0.7618893384933472,
      "rewards/rejected": 1.629267692565918,
      "step": 370
    },
    {
      "epoch": 0.35829451809387314,
      "grad_norm": 7.15625,
      "learning_rate": 3.6306543963705936e-07,
      "logits/chosen": -1.9923632144927979,
      "logits/rejected": -1.9175786972045898,
      "logps/chosen": -0.2919101119041443,
      "logps/rejected": -0.28915977478027344,
      "loss": 1.7186968326568604,
      "loss/core": 1.7186968326568604,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 3.748534679412842,
      "rewards/margins": 2.381950855255127,
      "rewards/rejected": 1.3665834665298462,
      "step": 375
    },
    {
      "epoch": 0.36307177833512483,
      "grad_norm": 47.0,
      "learning_rate": 3.5872046928983623e-07,
      "logits/chosen": -1.9712073802947998,
      "logits/rejected": -2.0564160346984863,
      "logps/chosen": -0.28708744049072266,
      "logps/rejected": -0.308341920375824,
      "loss": 1.9703826904296875,
      "loss/core": 1.9703826904296875,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.5351765155792236,
      "rewards/margins": 1.9188482761383057,
      "rewards/rejected": 1.616328239440918,
      "step": 380
    },
    {
      "epoch": 0.36784903857637646,
      "grad_norm": 286.0,
      "learning_rate": 3.5433461362045447e-07,
      "logits/chosen": -2.1837782859802246,
      "logits/rejected": -2.109261989593506,
      "logps/chosen": -0.3028949201107025,
      "logps/rejected": -0.3316383957862854,
      "loss": 1.2228304147720337,
      "loss/core": 1.2228304147720337,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.1072325706481934,
      "rewards/margins": 1.2266871929168701,
      "rewards/rejected": 1.8805453777313232,
      "step": 385
    },
    {
      "epoch": 0.3726262988176281,
      "grad_norm": 2.671875,
      "learning_rate": 3.4990952196986305e-07,
      "logits/chosen": -2.0442094802856445,
      "logits/rejected": -2.072903633117676,
      "logps/chosen": -0.2997892498970032,
      "logps/rejected": -0.30010247230529785,
      "loss": 0.475435346364975,
      "loss/core": 0.475435346364975,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.0823211669921875,
      "rewards/margins": 1.195578694343567,
      "rewards/rejected": 0.8867424726486206,
      "step": 390
    },
    {
      "epoch": 0.37740355905887973,
      "grad_norm": 7.5625,
      "learning_rate": 3.4544685843405875e-07,
      "logits/chosen": -2.156062364578247,
      "logits/rejected": -2.2315170764923096,
      "logps/chosen": -0.2756079435348511,
      "logps/rejected": -0.2708030045032501,
      "loss": 0.1924670934677124,
      "loss/core": 0.1924670934677124,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.018162965774536,
      "rewards/margins": 1.0384140014648438,
      "rewards/rejected": 0.9797487258911133,
      "step": 395
    },
    {
      "epoch": 0.38218081930013137,
      "grad_norm": 25.25,
      "learning_rate": 3.4094830123828786e-07,
      "logits/chosen": -2.1258950233459473,
      "logits/rejected": -2.064730405807495,
      "logps/chosen": -0.27466386556625366,
      "logps/rejected": -0.28442296385765076,
      "loss": 0.35562631487846375,
      "loss/core": 0.35562631487846375,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.0544662475585938,
      "rewards/margins": 0.5271926522254944,
      "rewards/rejected": 1.5272737741470337,
      "step": 400
    },
    {
      "epoch": 0.386958079541383,
      "grad_norm": 1208.0,
      "learning_rate": 3.3641554210593414e-07,
      "logits/chosen": -2.2992489337921143,
      "logits/rejected": -2.3570144176483154,
      "logps/chosen": -0.266804575920105,
      "logps/rejected": -0.265764981508255,
      "loss": 1.0862951278686523,
      "loss/core": 1.0862951278686523,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.5454628467559814,
      "rewards/margins": 1.311475157737732,
      "rewards/rejected": 1.2339876890182495,
      "step": 405
    },
    {
      "epoch": 0.39173533978263464,
      "grad_norm": 11.25,
      "learning_rate": 3.3185028562233107e-07,
      "logits/chosen": -2.073944568634033,
      "logits/rejected": -2.1331162452697754,
      "logps/chosen": -0.3112749457359314,
      "logps/rejected": -0.3046428859233856,
      "loss": 0.09161730855703354,
      "loss/core": 0.09161730855703354,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.2061810493469238,
      "rewards/margins": 0.551508903503418,
      "rewards/rejected": 0.6546720266342163,
      "step": 410
    },
    {
      "epoch": 0.3965126000238863,
      "grad_norm": 4.96875,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.2970967292785645,
      "logits/rejected": -2.295583724975586,
      "logps/chosen": -0.29680368304252625,
      "logps/rejected": -0.28816667199134827,
      "loss": 0.5273622274398804,
      "loss/core": 0.5273622274398804,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.4427194595336914,
      "rewards/margins": 1.134141206741333,
      "rewards/rejected": 1.3085782527923584,
      "step": 415
    },
    {
      "epoch": 0.40128986026513797,
      "grad_norm": 508.0,
      "learning_rate": 3.226291594017137e-07,
      "logits/chosen": -2.098283290863037,
      "logits/rejected": -2.067476749420166,
      "logps/chosen": -0.3195353150367737,
      "logps/rejected": -0.3052719235420227,
      "loss": 1.6536753177642822,
      "loss/core": 1.6536753177642822,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.3768184185028076,
      "rewards/margins": 2.38930344581604,
      "rewards/rejected": 0.9875147938728333,
      "step": 420
    },
    {
      "epoch": 0.4060671205063896,
      "grad_norm": 23.625,
      "learning_rate": 3.1797675735315454e-07,
      "logits/chosen": -2.124053478240967,
      "logits/rejected": -2.2029757499694824,
      "logps/chosen": -0.31783485412597656,
      "logps/rejected": -0.31542786955833435,
      "loss": 1.095782995223999,
      "loss/core": 1.095782995223999,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.0307421684265137,
      "rewards/margins": 1.9506139755249023,
      "rewards/rejected": 1.0801277160644531,
      "step": 425
    },
    {
      "epoch": 0.41084438074764124,
      "grad_norm": 66.5,
      "learning_rate": 3.1329879202620047e-07,
      "logits/chosen": -2.1495888233184814,
      "logits/rejected": -2.263394832611084,
      "logps/chosen": -0.26696711778640747,
      "logps/rejected": -0.2735012173652649,
      "loss": 0.14888674020767212,
      "loss/core": 0.14888674020767212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.4852890968322754,
      "rewards/margins": 0.5966399908065796,
      "rewards/rejected": 0.8886489868164062,
      "step": 430
    },
    {
      "epoch": 0.41562164098889287,
      "grad_norm": 1328.0,
      "learning_rate": 3.0859702261229613e-07,
      "logits/chosen": -1.9643999338150024,
      "logits/rejected": -1.975927710533142,
      "logps/chosen": -0.2912464141845703,
      "logps/rejected": -0.2692493498325348,
      "loss": 1.1893305778503418,
      "loss/core": 1.1893305778503418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.8585808277130127,
      "rewards/margins": 2.5125555992126465,
      "rewards/rejected": 1.3460257053375244,
      "step": 435
    },
    {
      "epoch": 0.4203989012301445,
      "grad_norm": 42.5,
      "learning_rate": 3.0387321725463e-07,
      "logits/chosen": -2.1609604358673096,
      "logits/rejected": -2.1351842880249023,
      "logps/chosen": -0.29672715067863464,
      "logps/rejected": -0.2880227565765381,
      "loss": 0.2775880694389343,
      "loss/core": 0.2775880694389343,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.147207736968994,
      "rewards/margins": 0.9227482080459595,
      "rewards/rejected": 1.2244592905044556,
      "step": 440
    },
    {
      "epoch": 0.42517616147139614,
      "grad_norm": 97.0,
      "learning_rate": 2.991291523832075e-07,
      "logits/chosen": -2.0940496921539307,
      "logits/rejected": -2.1732332706451416,
      "logps/chosen": -0.285409152507782,
      "logps/rejected": -0.27999910712242126,
      "loss": 0.576115071773529,
      "loss/core": 0.576115071773529,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.9870572090148926,
      "rewards/margins": 1.8674147129058838,
      "rewards/rejected": 1.1196422576904297,
      "step": 445
    },
    {
      "epoch": 0.4299534217126478,
      "grad_norm": 1600.0,
      "learning_rate": 2.943666120468088e-07,
      "logits/chosen": -2.1286191940307617,
      "logits/rejected": -2.199714422225952,
      "logps/chosen": -0.2662820518016815,
      "logps/rejected": -0.30065736174583435,
      "loss": 0.9491519927978516,
      "loss/core": 0.9491519927978516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.870041608810425,
      "rewards/margins": 1.6734178066253662,
      "rewards/rejected": 1.1966238021850586,
      "step": 450
    },
    {
      "epoch": 0.4347306819538994,
      "grad_norm": 1.9921875,
      "learning_rate": 2.8958738724208073e-07,
      "logits/chosen": -1.8988832235336304,
      "logits/rejected": -1.9345617294311523,
      "logps/chosen": -0.30704838037490845,
      "logps/rejected": -0.3010723292827606,
      "loss": 0.18666288256645203,
      "loss/core": 0.18666288256645203,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.7061916589736938,
      "rewards/margins": 0.875575065612793,
      "rewards/rejected": 0.8306165933609009,
      "step": 455
    },
    {
      "epoch": 0.4395079421951511,
      "grad_norm": 10.3125,
      "learning_rate": 2.8479327524001633e-07,
      "logits/chosen": -2.0395889282226562,
      "logits/rejected": -2.1937108039855957,
      "logps/chosen": -0.2862888276576996,
      "logps/rejected": -0.28865212202072144,
      "loss": 0.35019630193710327,
      "loss/core": 0.35019630193710327,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.244964838027954,
      "rewards/margins": 0.8104959726333618,
      "rewards/rejected": 1.4344688653945923,
      "step": 460
    },
    {
      "epoch": 0.44428520243640274,
      "grad_norm": 32.5,
      "learning_rate": 2.7998607891007493e-07,
      "logits/chosen": -2.112020254135132,
      "logits/rejected": -2.0445303916931152,
      "logps/chosen": -0.2725178003311157,
      "logps/rejected": -0.30414050817489624,
      "loss": 1.0059139728546143,
      "loss/core": 1.0059139728546143,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.83925199508667,
      "rewards/margins": 0.9617875814437866,
      "rewards/rejected": 1.8774642944335938,
      "step": 465
    },
    {
      "epoch": 0.4490624626776544,
      "grad_norm": 102.5,
      "learning_rate": 2.7516760604219616e-07,
      "logits/chosen": -2.1016361713409424,
      "logits/rejected": -2.077967643737793,
      "logps/chosen": -0.2712469696998596,
      "logps/rejected": -0.29456794261932373,
      "loss": 0.46677565574645996,
      "loss/core": 0.46677565574645996,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.2126553058624268,
      "rewards/margins": 0.6549152135848999,
      "rewards/rejected": 1.557740330696106,
      "step": 470
    },
    {
      "epoch": 0.453839722918906,
      "grad_norm": 11.6875,
      "learning_rate": 2.703396686669646e-07,
      "logits/chosen": -2.209343433380127,
      "logits/rejected": -2.228372097015381,
      "logps/chosen": -0.27892497181892395,
      "logps/rejected": -0.2862241864204407,
      "loss": 0.31267350912094116,
      "loss/core": 0.31267350912094116,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.3414828777313232,
      "rewards/margins": 0.8732233047485352,
      "rewards/rejected": 1.468259572982788,
      "step": 475
    },
    {
      "epoch": 0.45861698316015764,
      "grad_norm": 1168.0,
      "learning_rate": 2.6550408237417884e-07,
      "logits/chosen": -1.982518196105957,
      "logits/rejected": -1.9973068237304688,
      "logps/chosen": -0.28591465950012207,
      "logps/rejected": -0.2720577120780945,
      "loss": 2.5910723209381104,
      "loss/core": 2.5910723209381104,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 5.177981376647949,
      "rewards/margins": 3.9536633491516113,
      "rewards/rejected": 1.2243181467056274,
      "step": 480
    },
    {
      "epoch": 0.4633942434014093,
      "grad_norm": 11.4375,
      "learning_rate": 2.6066266563008265e-07,
      "logits/chosen": -2.1890599727630615,
      "logits/rejected": -2.172191858291626,
      "logps/chosen": -0.2838982939720154,
      "logps/rejected": -0.30362966656684875,
      "loss": 0.41411668062210083,
      "loss/core": 0.41411668062210083,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.087114095687866,
      "rewards/margins": 1.0661743879318237,
      "rewards/rejected": 1.020939588546753,
      "step": 485
    },
    {
      "epoch": 0.4681715036426609,
      "grad_norm": 44.0,
      "learning_rate": 2.5581723909351404e-07,
      "logits/chosen": -2.1797804832458496,
      "logits/rejected": -2.127960681915283,
      "logps/chosen": -0.2894228994846344,
      "logps/rejected": -0.3229179084300995,
      "loss": 0.21129579842090607,
      "loss/core": 0.21129579842090607,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6542030572891235,
      "rewards/margins": 0.47612959146499634,
      "rewards/rejected": 1.1780736446380615,
      "step": 490
    },
    {
      "epoch": 0.47294876388391255,
      "grad_norm": 1896.0,
      "learning_rate": 2.509696249312301e-07,
      "logits/chosen": -2.0821642875671387,
      "logits/rejected": -2.087721347808838,
      "logps/chosen": -0.3256458342075348,
      "logps/rejected": -0.3126862943172455,
      "loss": 1.8491243124008179,
      "loss/core": 1.8491243124008179,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.5472946166992188,
      "rewards/margins": 1.9033483266830444,
      "rewards/rejected": 1.6439464092254639,
      "step": 495
    },
    {
      "epoch": 0.47772602412516424,
      "grad_norm": 400.0,
      "learning_rate": 2.461216461326642e-07,
      "logits/chosen": -2.1551461219787598,
      "logits/rejected": -2.116971015930176,
      "logps/chosen": -0.279205858707428,
      "logps/rejected": -0.27398478984832764,
      "loss": 1.2034151554107666,
      "loss/core": 1.2034151554107666,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.4934756755828857,
      "rewards/margins": 2.316671371459961,
      "rewards/rejected": 1.176804542541504,
      "step": 500
    },
    {
      "epoch": 0.4825032843664159,
      "grad_norm": 7.875,
      "learning_rate": 2.412751258243748e-07,
      "logits/chosen": -1.734514832496643,
      "logits/rejected": -1.755017638206482,
      "logps/chosen": -0.3091053366661072,
      "logps/rejected": -0.31018969416618347,
      "loss": 0.7064386606216431,
      "loss/core": 0.7064386606216431,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.180860996246338,
      "rewards/margins": 1.8545881509780884,
      "rewards/rejected": 1.32627272605896,
      "step": 505
    },
    {
      "epoch": 0.4872805446076675,
      "grad_norm": 502.0,
      "learning_rate": 2.3643188658444158e-07,
      "logits/chosen": -2.0605289936065674,
      "logits/rejected": -2.1547398567199707,
      "logps/chosen": -0.2701338529586792,
      "logps/rejected": -0.2844063639640808,
      "loss": 0.5944812297821045,
      "loss/core": 0.5944812297821045,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.014646053314209,
      "rewards/margins": 1.663463830947876,
      "rewards/rejected": 1.351182222366333,
      "step": 510
    },
    {
      "epoch": 0.49205780484891914,
      "grad_norm": 14.5,
      "learning_rate": 2.315937497570688e-07,
      "logits/chosen": -1.9274842739105225,
      "logits/rejected": -2.0313713550567627,
      "logps/chosen": -0.2993752360343933,
      "logps/rejected": -0.3588908612728119,
      "loss": 0.5572088956832886,
      "loss/core": 0.5572088956832886,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.2762691974639893,
      "rewards/margins": 1.200260043144226,
      "rewards/rejected": 1.0760090351104736,
      "step": 515
    },
    {
      "epoch": 0.4968350650901708,
      "grad_norm": 2.265625,
      "learning_rate": 2.2676253476765194e-07,
      "logits/chosen": -1.9769691228866577,
      "logits/rejected": -2.1776771545410156,
      "logps/chosen": -0.32004016637802124,
      "logps/rejected": -0.2949730157852173,
      "loss": 0.6923333406448364,
      "loss/core": 0.6923333406448364,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.3702404499053955,
      "rewards/margins": 1.4442033767700195,
      "rewards/rejected": 0.9260371923446655,
      "step": 520
    },
    {
      "epoch": 0.5016123253314224,
      "grad_norm": 0.89453125,
      "learning_rate": 2.2194005843856633e-07,
      "logits/chosen": -2.003140687942505,
      "logits/rejected": -2.0340893268585205,
      "logps/chosen": -0.28318530321121216,
      "logps/rejected": -0.281009316444397,
      "loss": 0.7909449338912964,
      "loss/core": 0.7909449338912964,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.3549766540527344,
      "rewards/margins": 2.027585744857788,
      "rewards/rejected": 1.3273913860321045,
      "step": 525
    },
    {
      "epoch": 0.5063895855726741,
      "grad_norm": 7.53125,
      "learning_rate": 2.1712813430593434e-07,
      "logits/chosen": -2.0626168251037598,
      "logits/rejected": -2.1819634437561035,
      "logps/chosen": -0.2981792986392975,
      "logps/rejected": -0.297743558883667,
      "loss": 0.17196998000144958,
      "loss/core": 0.17196998000144958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.795514464378357,
      "rewards/margins": 0.9798916578292847,
      "rewards/rejected": 0.8156226873397827,
      "step": 530
    },
    {
      "epoch": 0.5111668458139257,
      "grad_norm": 39.5,
      "learning_rate": 2.123285719376292e-07,
      "logits/chosen": -2.0366547107696533,
      "logits/rejected": -2.0415263175964355,
      "logps/chosen": -0.28957507014274597,
      "logps/rejected": -0.28460192680358887,
      "loss": 1.1312463283538818,
      "loss/core": 1.1312463283538818,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 3.3058319091796875,
      "rewards/margins": 1.5662612915039062,
      "rewards/rejected": 1.7395708560943604,
      "step": 535
    },
    {
      "epoch": 0.5159441060551774,
      "grad_norm": 43.5,
      "learning_rate": 2.0754317625276982e-07,
      "logits/chosen": -2.0326008796691895,
      "logits/rejected": -2.0860085487365723,
      "logps/chosen": -0.2913965582847595,
      "logps/rejected": -0.2849191427230835,
      "loss": 0.28950196504592896,
      "loss/core": 0.28950196504592896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.971967101097107,
      "rewards/margins": 1.0436313152313232,
      "rewards/rejected": 0.9283358454704285,
      "step": 540
    },
    {
      "epoch": 0.520721366296429,
      "grad_norm": 208.0,
      "learning_rate": 2.0277374684296498e-07,
      "logits/chosen": -2.021231174468994,
      "logits/rejected": -2.0813746452331543,
      "logps/chosen": -0.2889094650745392,
      "logps/rejected": -0.27322858572006226,
      "loss": 0.8897081613540649,
      "loss/core": 0.8897081613540649,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 3.6772232055664062,
      "rewards/margins": 2.866091012954712,
      "rewards/rejected": 0.8111323118209839,
      "step": 545
    },
    {
      "epoch": 0.5254986265376806,
      "grad_norm": 6.90625,
      "learning_rate": 1.980220772955602e-07,
      "logits/chosen": -1.9826974868774414,
      "logits/rejected": -2.0089569091796875,
      "logps/chosen": -0.2836936116218567,
      "logps/rejected": -0.28125476837158203,
      "loss": 0.17911246418952942,
      "loss/core": 0.17911246418952942,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.2390589714050293,
      "rewards/margins": 1.1507548093795776,
      "rewards/rejected": 1.0883041620254517,
      "step": 550
    },
    {
      "epoch": 0.5302758867789323,
      "grad_norm": 15.25,
      "learning_rate": 1.932899545191433e-07,
      "logits/chosen": -2.034287452697754,
      "logits/rejected": -2.1081697940826416,
      "logps/chosen": -0.3206217288970947,
      "logps/rejected": -0.30311864614486694,
      "loss": 1.040518045425415,
      "loss/core": 1.040518045425415,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.1932551860809326,
      "rewards/margins": 1.7633758783340454,
      "rewards/rejected": 1.429879069328308,
      "step": 555
    },
    {
      "epoch": 0.5350531470201839,
      "grad_norm": 14.6875,
      "learning_rate": 1.885791580715609e-07,
      "logits/chosen": -1.850868582725525,
      "logits/rejected": -1.8559157848358154,
      "logps/chosen": -0.3106183409690857,
      "logps/rejected": -0.29663991928100586,
      "loss": 0.6726303100585938,
      "loss/core": 0.6726303100585938,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.4848763942718506,
      "rewards/margins": 1.1928892135620117,
      "rewards/rejected": 1.291987419128418,
      "step": 560
    },
    {
      "epoch": 0.5398304072614356,
      "grad_norm": 45.25,
      "learning_rate": 1.8389145949069951e-07,
      "logits/chosen": -1.9359796047210693,
      "logits/rejected": -2.0416994094848633,
      "logps/chosen": -0.28728121519088745,
      "logps/rejected": -0.2942139506340027,
      "loss": 2.735677719116211,
      "loss/core": 2.735677719116211,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 5.07828950881958,
      "rewards/margins": 3.2359542846679688,
      "rewards/rejected": 1.8423347473144531,
      "step": 565
    },
    {
      "epoch": 0.5446076675026872,
      "grad_norm": 91.5,
      "learning_rate": 1.792286216282824e-07,
      "logits/chosen": -2.077359437942505,
      "logits/rejected": -2.0332412719726562,
      "logps/chosen": -0.3251068890094757,
      "logps/rejected": -0.31182152032852173,
      "loss": 0.7743596434593201,
      "loss/core": 0.7743596434593201,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.263739585876465,
      "rewards/margins": 0.2986152172088623,
      "rewards/rejected": 1.965124487876892,
      "step": 570
    },
    {
      "epoch": 0.5493849277439389,
      "grad_norm": 20.625,
      "learning_rate": 1.745923979869336e-07,
      "logits/chosen": -2.3297030925750732,
      "logits/rejected": -2.3048276901245117,
      "logps/chosen": -0.28046730160713196,
      "logps/rejected": -0.2963675260543823,
      "loss": 0.5231956839561462,
      "loss/core": 0.5231956839561462,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.400116443634033,
      "rewards/margins": 1.4617714881896973,
      "rewards/rejected": 0.9383450746536255,
      "step": 575
    },
    {
      "epoch": 0.5541621879851905,
      "grad_norm": 8.8125,
      "learning_rate": 1.6998453206075708e-07,
      "logits/chosen": -2.00852632522583,
      "logits/rejected": -2.128060817718506,
      "logps/chosen": -0.30730748176574707,
      "logps/rejected": -0.3067978024482727,
      "loss": 0.4244763255119324,
      "loss/core": 0.4244763255119324,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.4532878398895264,
      "rewards/margins": 1.449927568435669,
      "rewards/rejected": 1.0033600330352783,
      "step": 580
    },
    {
      "epoch": 0.5589394482264421,
      "grad_norm": 234.0,
      "learning_rate": 1.6540675667967973e-07,
      "logits/chosen": -2.2257282733917236,
      "logits/rejected": -2.239046096801758,
      "logps/chosen": -0.2853415906429291,
      "logps/rejected": -0.28488466143608093,
      "loss": 0.8494731187820435,
      "loss/core": 0.8494731187820435,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.5698318481445312,
      "rewards/margins": 0.717768669128418,
      "rewards/rejected": 1.8520629405975342,
      "step": 585
    },
    {
      "epoch": 0.5637167084676937,
      "grad_norm": 185.0,
      "learning_rate": 1.60860793357805e-07,
      "logits/chosen": -1.9403209686279297,
      "logits/rejected": -1.9107745885849,
      "logps/chosen": -0.30443838238716125,
      "logps/rejected": -0.3014160990715027,
      "loss": 0.7370032668113708,
      "loss/core": 0.7370032668113708,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.7498955726623535,
      "rewards/margins": 1.003064513206482,
      "rewards/rejected": 1.746830940246582,
      "step": 590
    },
    {
      "epoch": 0.5684939687089454,
      "grad_norm": 2.40625,
      "learning_rate": 1.5634835164602196e-07,
      "logits/chosen": -2.0419182777404785,
      "logits/rejected": -1.969354271888733,
      "logps/chosen": -0.2744390368461609,
      "logps/rejected": -0.2946432828903198,
      "loss": 0.6912421584129333,
      "loss/core": 0.6912421584129333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.8153032064437866,
      "rewards/margins": -0.3735305368900299,
      "rewards/rejected": 2.188833713531494,
      "step": 595
    },
    {
      "epoch": 0.5732712289501971,
      "grad_norm": 40.25,
      "learning_rate": 1.518711284891132e-07,
      "logits/chosen": -2.1388328075408936,
      "logits/rejected": -2.0539865493774414,
      "logps/chosen": -0.2972756028175354,
      "logps/rejected": -0.31171417236328125,
      "loss": 0.36328691244125366,
      "loss/core": 0.36328691244125366,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7741825580596924,
      "rewards/margins": 0.6642401814460754,
      "rewards/rejected": 1.1099423170089722,
      "step": 600
    },
    {
      "epoch": 0.5780484891914487,
      "grad_norm": 28.5,
      "learning_rate": 1.47430807587603e-07,
      "logits/chosen": -2.2820992469787598,
      "logits/rejected": -2.370114803314209,
      "logps/chosen": -0.29316216707229614,
      "logps/rejected": -0.3045360743999481,
      "loss": 0.418804794549942,
      "loss/core": 0.418804794549942,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.924261450767517,
      "rewards/margins": 1.342273473739624,
      "rewards/rejected": 0.5819882154464722,
      "step": 605
    },
    {
      "epoch": 0.5828257494327004,
      "grad_norm": 5.03125,
      "learning_rate": 1.430290587645865e-07,
      "logits/chosen": -2.0365357398986816,
      "logits/rejected": -2.1053943634033203,
      "logps/chosen": -0.30090898275375366,
      "logps/rejected": -0.30962470173835754,
      "loss": 0.7318702936172485,
      "loss/core": 0.7318702936172485,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.31716251373291,
      "rewards/margins": 0.7470144033432007,
      "rewards/rejected": 1.5701478719711304,
      "step": 610
    },
    {
      "epoch": 0.587603009673952,
      "grad_norm": 1.0390625,
      "learning_rate": 1.3866753733777765e-07,
      "logits/chosen": -2.0896778106689453,
      "logits/rejected": -2.0692594051361084,
      "logps/chosen": -0.3204464912414551,
      "logps/rejected": -0.31083038449287415,
      "loss": 0.15200874209403992,
      "loss/core": 0.15200874209403992,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.405200481414795,
      "rewards/margins": 0.6567575335502625,
      "rewards/rejected": 0.7484430074691772,
      "step": 615
    },
    {
      "epoch": 0.5923802699152036,
      "grad_norm": 10.25,
      "learning_rate": 1.343478834970121e-07,
      "logits/chosen": -2.2016444206237793,
      "logits/rejected": -2.162689447402954,
      "logps/chosen": -0.2618758976459503,
      "logps/rejected": -0.28418415784835815,
      "loss": 0.2642122209072113,
      "loss/core": 0.2642122209072113,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.8917810916900635,
      "rewards/margins": 0.6098601818084717,
      "rewards/rejected": 1.2819210290908813,
      "step": 620
    },
    {
      "epoch": 0.5971575301564552,
      "grad_norm": 13.5,
      "learning_rate": 1.3007172168743852e-07,
      "logits/chosen": -2.0167224407196045,
      "logits/rejected": -1.9906463623046875,
      "logps/chosen": -0.2946910262107849,
      "logps/rejected": -0.29906517267227173,
      "loss": 0.1281525194644928,
      "loss/core": 0.1281525194644928,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.5117716789245605,
      "rewards/margins": 0.6090092658996582,
      "rewards/rejected": 0.9027624130249023,
      "step": 625
    },
    {
      "epoch": 0.6019347903977069,
      "grad_norm": 48.0,
      "learning_rate": 1.25840659998631e-07,
      "logits/chosen": -2.24929141998291,
      "logits/rejected": -2.1729183197021484,
      "logps/chosen": -0.2794455885887146,
      "logps/rejected": -0.2963079810142517,
      "loss": 0.09536317735910416,
      "loss/core": 0.09536317735910416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.2493692636489868,
      "rewards/margins": 0.36890143156051636,
      "rewards/rejected": 0.8804677724838257,
      "step": 630
    },
    {
      "epoch": 0.6067120506389586,
      "grad_norm": 0.60546875,
      "learning_rate": 1.2165628955985313e-07,
      "logits/chosen": -2.070937395095825,
      "logits/rejected": -2.0991628170013428,
      "logps/chosen": -0.30978572368621826,
      "logps/rejected": -0.2937181890010834,
      "loss": 1.2708914279937744,
      "loss/core": 1.2708914279937744,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.1841537952423096,
      "rewards/margins": 2.332937240600586,
      "rewards/rejected": 0.851216197013855,
      "step": 635
    },
    {
      "epoch": 0.6114893108802102,
      "grad_norm": 255.0,
      "learning_rate": 1.175201839416988e-07,
      "logits/chosen": -1.9998000860214233,
      "logits/rejected": -2.1376776695251465,
      "logps/chosen": -0.2979816794395447,
      "logps/rejected": -0.2850627899169922,
      "loss": 0.48012876510620117,
      "loss/core": 0.48012876510620117,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.3821167945861816,
      "rewards/margins": 1.731583833694458,
      "rewards/rejected": 0.6505327820777893,
      "step": 640
    },
    {
      "epoch": 0.6162665711214619,
      "grad_norm": 13.0,
      "learning_rate": 1.1343389856433658e-07,
      "logits/chosen": -2.054605484008789,
      "logits/rejected": -2.1557681560516357,
      "logps/chosen": -0.2970808148384094,
      "logps/rejected": -0.26274365186691284,
      "loss": 2.3463776111602783,
      "loss/core": 2.3463776111602783,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 4.414872169494629,
      "rewards/margins": 2.85589337348938,
      "rewards/rejected": 1.5589792728424072,
      "step": 645
    },
    {
      "epoch": 0.6210438313627135,
      "grad_norm": 7.71875,
      "learning_rate": 1.0939897011258e-07,
      "logits/chosen": -2.2726993560791016,
      "logits/rejected": -2.288119077682495,
      "logps/chosen": -0.3080156445503235,
      "logps/rejected": -0.2879447340965271,
      "loss": 1.3906691074371338,
      "loss/core": 1.3906691074371338,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.284862995147705,
      "rewards/margins": 2.400153875350952,
      "rewards/rejected": 0.8847091794013977,
      "step": 650
    },
    {
      "epoch": 0.6258210916039652,
      "grad_norm": 17.75,
      "learning_rate": 1.0541691595800336e-07,
      "logits/chosen": -1.922623872756958,
      "logits/rejected": -1.9154341220855713,
      "logps/chosen": -0.28070202469825745,
      "logps/rejected": -0.3093103766441345,
      "loss": 0.5413604974746704,
      "loss/core": 0.5413604974746704,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.844754457473755,
      "rewards/margins": 1.1954736709594727,
      "rewards/rejected": 1.6492809057235718,
      "step": 655
    },
    {
      "epoch": 0.6305983518452167,
      "grad_norm": 16.75,
      "learning_rate": 1.0148923358832021e-07,
      "logits/chosen": -2.2555432319641113,
      "logits/rejected": -2.311061382293701,
      "logps/chosen": -0.27462202310562134,
      "logps/rejected": -0.2811489999294281,
      "loss": 0.3550433814525604,
      "loss/core": 0.3550433814525604,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0413260459899902,
      "rewards/margins": 0.9709696769714355,
      "rewards/rejected": 1.0703566074371338,
      "step": 660
    },
    {
      "epoch": 0.6353756120864684,
      "grad_norm": 26.875,
      "learning_rate": 9.761740004423926e-08,
      "logits/chosen": -2.12736439704895,
      "logits/rejected": -2.2038140296936035,
      "logps/chosen": -0.3001564145088196,
      "logps/rejected": -0.32446950674057007,
      "loss": 0.2872669994831085,
      "loss/core": 0.2872669994831085,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7709327936172485,
      "rewards/margins": 0.7759586572647095,
      "rewards/rejected": 0.9949741363525391,
      "step": 665
    },
    {
      "epoch": 0.64015287232772,
      "grad_norm": 14.125,
      "learning_rate": 9.380287136400999e-08,
      "logits/chosen": -2.269469738006592,
      "logits/rejected": -2.301842212677002,
      "logps/chosen": -0.285754531621933,
      "logps/rejected": -0.2945065200328827,
      "loss": 0.39818090200424194,
      "loss/core": 0.39818090200424194,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9868361949920654,
      "rewards/margins": 0.8125089406967163,
      "rewards/rejected": 1.1743274927139282,
      "step": 670
    },
    {
      "epoch": 0.6449301325689717,
      "grad_norm": 23.375,
      "learning_rate": 9.004708203586628e-08,
      "logits/chosen": -2.080825090408325,
      "logits/rejected": -2.2083404064178467,
      "logps/chosen": -0.2972838282585144,
      "logps/rejected": -0.28944870829582214,
      "loss": 0.3145357370376587,
      "loss/core": 0.3145357370376587,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.903146505355835,
      "rewards/margins": 0.8571847677230835,
      "rewards/rejected": 1.0459617376327515,
      "step": 675
    },
    {
      "epoch": 0.6497073928102234,
      "grad_norm": 704.0,
      "learning_rate": 8.635144445857407e-08,
      "logits/chosen": -2.1277308464050293,
      "logits/rejected": -2.093435525894165,
      "logps/chosen": -0.28518134355545044,
      "logps/rejected": -0.2920211851596832,
      "loss": 0.24321074783802032,
      "loss/core": 0.24321074783802032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.2132742404937744,
      "rewards/margins": -0.060915298759937286,
      "rewards/rejected": 1.2741894721984863,
      "step": 680
    },
    {
      "epoch": 0.654484653051475,
      "grad_norm": 392.0,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -2.1869044303894043,
      "logits/rejected": -2.1886041164398193,
      "logps/chosen": -0.2989751398563385,
      "logps/rejected": -0.30234605073928833,
      "loss": 0.5350499153137207,
      "loss/core": 0.5350499153137207,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.4281145334243774,
      "rewards/margins": -0.04558032751083374,
      "rewards/rejected": 1.4736945629119873,
      "step": 685
    },
    {
      "epoch": 0.6592619132927267,
      "grad_norm": 21.625,
      "learning_rate": 7.91461605259007e-08,
      "logits/chosen": -1.9535391330718994,
      "logits/rejected": -1.9515883922576904,
      "logps/chosen": -0.26703256368637085,
      "logps/rejected": -0.28917747735977173,
      "loss": 0.5689412355422974,
      "loss/core": 0.5689412355422974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.011037826538086,
      "rewards/margins": 1.4389708042144775,
      "rewards/rejected": 1.5720670223236084,
      "step": 690
    },
    {
      "epoch": 0.6640391735339782,
      "grad_norm": 6.96875,
      "learning_rate": 7.563922378313217e-08,
      "logits/chosen": -2.2548301219940186,
      "logits/rejected": -2.1238934993743896,
      "logps/chosen": -0.2951969504356384,
      "logps/rejected": -0.3159661591053009,
      "loss": 0.14107027649879456,
      "loss/core": 0.14107027649879456,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.4432185888290405,
      "rewards/margins": 0.7674707174301147,
      "rewards/rejected": 0.6757478713989258,
      "step": 695
    },
    {
      "epoch": 0.6688164337752299,
      "grad_norm": 183.0,
      "learning_rate": 7.219785699746572e-08,
      "logits/chosen": -2.0966005325317383,
      "logits/rejected": -2.1340537071228027,
      "logps/chosen": -0.2893856465816498,
      "logps/rejected": -0.2702532708644867,
      "loss": 0.855477511882782,
      "loss/core": 0.855477511882782,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 3.0240743160247803,
      "rewards/margins": 1.6322101354599,
      "rewards/rejected": 1.3918644189834595,
      "step": 700
    },
    {
      "epoch": 0.6735936940164815,
      "grad_norm": 148.0,
      "learning_rate": 6.882335432620779e-08,
      "logits/chosen": -2.008580446243286,
      "logits/rejected": -2.0907862186431885,
      "logps/chosen": -0.2771678864955902,
      "logps/rejected": -0.2853749692440033,
      "loss": 0.19750399887561798,
      "loss/core": 0.19750399887561798,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.4055383205413818,
      "rewards/margins": 0.3061255216598511,
      "rewards/rejected": 1.0994129180908203,
      "step": 705
    },
    {
      "epoch": 0.6783709542577332,
      "grad_norm": 5.15625,
      "learning_rate": 6.551698478180589e-08,
      "logits/chosen": -2.002742052078247,
      "logits/rejected": -2.040894031524658,
      "logps/chosen": -0.28560441732406616,
      "logps/rejected": -0.2906520962715149,
      "loss": 0.36187195777893066,
      "loss/core": 0.36187195777893066,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.5490121841430664,
      "rewards/margins": 1.2243046760559082,
      "rewards/rejected": 1.3247077465057373,
      "step": 710
    },
    {
      "epoch": 0.6831482144989849,
      "grad_norm": 2.125,
      "learning_rate": 6.22799917546252e-08,
      "logits/chosen": -2.0940136909484863,
      "logits/rejected": -2.1399710178375244,
      "logps/chosen": -0.28695395588874817,
      "logps/rejected": -0.2956109941005707,
      "loss": 0.47707152366638184,
      "loss/core": 0.47707152366638184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.8282418251037598,
      "rewards/margins": 1.4551374912261963,
      "rewards/rejected": 1.3731043338775635,
      "step": 715
    },
    {
      "epoch": 0.6879254747402365,
      "grad_norm": 1.9140625,
      "learning_rate": 5.9113592545359944e-08,
      "logits/chosen": -2.2517735958099365,
      "logits/rejected": -2.2257275581359863,
      "logps/chosen": -0.3003559112548828,
      "logps/rejected": -0.2943252921104431,
      "loss": 0.47043830156326294,
      "loss/core": 0.47043830156326294,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.214423418045044,
      "rewards/margins": 1.1652237176895142,
      "rewards/rejected": 1.0491998195648193,
      "step": 720
    },
    {
      "epoch": 0.6927027349814882,
      "grad_norm": 3.765625,
      "learning_rate": 5.601897790725643e-08,
      "logits/chosen": -2.103532314300537,
      "logits/rejected": -2.1393818855285645,
      "logps/chosen": -0.31979304552078247,
      "logps/rejected": -0.31625136733055115,
      "loss": 0.08710397779941559,
      "loss/core": 0.08710397779941559,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.1311936378479004,
      "rewards/margins": 0.3907800316810608,
      "rewards/rejected": 0.7404135465621948,
      "step": 725
    },
    {
      "epoch": 0.6974799952227397,
      "grad_norm": 19.875,
      "learning_rate": 5.299731159831952e-08,
      "logits/chosen": -2.185694456100464,
      "logits/rejected": -2.1851601600646973,
      "logps/chosen": -0.29845187067985535,
      "logps/rejected": -0.30084067583084106,
      "loss": 0.5585907697677612,
      "loss/core": 0.5585907697677612,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.5317673683166504,
      "rewards/margins": 1.6783374547958374,
      "rewards/rejected": 0.8534297943115234,
      "step": 730
    },
    {
      "epoch": 0.7022572554639914,
      "grad_norm": 7.65625,
      "learning_rate": 5.0049729943671013e-08,
      "logits/chosen": -2.2042665481567383,
      "logits/rejected": -2.212019681930542,
      "logps/chosen": -0.270427942276001,
      "logps/rejected": -0.29099661111831665,
      "loss": 0.2325257807970047,
      "loss/core": 0.2325257807970047,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.025036573410034,
      "rewards/margins": 0.7466187477111816,
      "rewards/rejected": 1.2784178256988525,
      "step": 735
    },
    {
      "epoch": 0.707034515705243,
      "grad_norm": 5.625,
      "learning_rate": 4.7177341408223994e-08,
      "logits/chosen": -2.023667812347412,
      "logits/rejected": -2.055938243865967,
      "logps/chosen": -0.267364501953125,
      "logps/rejected": -0.284996896982193,
      "loss": 0.3424833416938782,
      "loss/core": 0.3424833416938782,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.194387912750244,
      "rewards/margins": 1.0326035022735596,
      "rewards/rejected": 1.1617845296859741,
      "step": 740
    },
    {
      "epoch": 0.7118117759464947,
      "grad_norm": 62.25,
      "learning_rate": 4.438122617983442e-08,
      "logits/chosen": -1.9847673177719116,
      "logits/rejected": -2.012366771697998,
      "logps/chosen": -0.2840479910373688,
      "logps/rejected": -0.27548593282699585,
      "loss": 1.2490966320037842,
      "loss/core": 1.2490966320037842,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.142892360687256,
      "rewards/margins": 1.1425834894180298,
      "rewards/rejected": 2.0003087520599365,
      "step": 745
    },
    {
      "epoch": 0.7165890361877463,
      "grad_norm": 10.9375,
      "learning_rate": 4.1662435763087114e-08,
      "logits/chosen": -2.047241687774658,
      "logits/rejected": -2.079470634460449,
      "logps/chosen": -0.2526155114173889,
      "logps/rejected": -0.2926296293735504,
      "loss": 0.3922721743583679,
      "loss/core": 0.3922721743583679,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4976963996887207,
      "rewards/margins": 1.1639492511749268,
      "rewards/rejected": 1.333747148513794,
      "step": 750
    },
    {
      "epoch": 0.721366296428998,
      "grad_norm": 1.0078125,
      "learning_rate": 3.902199258386732e-08,
      "logits/chosen": -2.1882448196411133,
      "logits/rejected": -2.1855483055114746,
      "logps/chosen": -0.36757299304008484,
      "logps/rejected": -0.29461368918418884,
      "loss": 1.8846288919448853,
      "loss/core": 1.8846288919448853,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.939770221710205,
      "rewards/margins": 1.8737109899520874,
      "rewards/rejected": 2.06605863571167,
      "step": 755
    },
    {
      "epoch": 0.7261435566702497,
      "grad_norm": 6.875,
      "learning_rate": 3.646088960486862e-08,
      "logits/chosen": -2.087844133377075,
      "logits/rejected": -2.1318325996398926,
      "logps/chosen": -0.2983017563819885,
      "logps/rejected": -0.30041682720184326,
      "loss": 0.2357165366411209,
      "loss/core": 0.2357165366411209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.57404363155365,
      "rewards/margins": 0.8634489178657532,
      "rewards/rejected": 0.7105947136878967,
      "step": 760
    },
    {
      "epoch": 0.7309208169115012,
      "grad_norm": 4.3125,
      "learning_rate": 3.398008995217988e-08,
      "logits/chosen": -2.253822088241577,
      "logits/rejected": -2.278644323348999,
      "logps/chosen": -0.2737087607383728,
      "logps/rejected": -0.2841106057167053,
      "loss": 0.0988527461886406,
      "loss/core": 0.0988527461886406,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.4775121212005615,
      "rewards/margins": 0.5869768261909485,
      "rewards/rejected": 0.8905353546142578,
      "step": 765
    },
    {
      "epoch": 0.7356980771527529,
      "grad_norm": 69.0,
      "learning_rate": 3.1580526553093315e-08,
      "logits/chosen": -2.1181020736694336,
      "logits/rejected": -2.1448464393615723,
      "logps/chosen": -0.30012568831443787,
      "logps/rejected": -0.2899145185947418,
      "loss": 0.3107560873031616,
      "loss/core": 0.3107560873031616,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.013152599334717,
      "rewards/margins": 0.9449566006660461,
      "rewards/rejected": 1.0681959390640259,
      "step": 770
    },
    {
      "epoch": 0.7404753373940045,
      "grad_norm": 251.0,
      "learning_rate": 2.9263101785268252e-08,
      "logits/chosen": -2.248095750808716,
      "logits/rejected": -2.2297964096069336,
      "logps/chosen": -0.27153995633125305,
      "logps/rejected": -0.27207639813423157,
      "loss": 0.322858989238739,
      "loss/core": 0.322858989238739,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.0420784950256348,
      "rewards/margins": 1.176545262336731,
      "rewards/rejected": 0.8655333518981934,
      "step": 775
    },
    {
      "epoch": 0.7452525976352562,
      "grad_norm": 20.625,
      "learning_rate": 2.7028687137384264e-08,
      "logits/chosen": -2.028242826461792,
      "logits/rejected": -2.135806083679199,
      "logps/chosen": -0.2820461392402649,
      "logps/rejected": -0.2808576226234436,
      "loss": 0.5030468702316284,
      "loss/core": 0.5030468702316284,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.855433940887451,
      "rewards/margins": 1.1535121202468872,
      "rewards/rejected": 1.701921820640564,
      "step": 780
    },
    {
      "epoch": 0.7500298578765078,
      "grad_norm": 14.875,
      "learning_rate": 2.4878122881409447e-08,
      "logits/chosen": -1.9230458736419678,
      "logits/rejected": -1.8952453136444092,
      "logps/chosen": -0.28769412636756897,
      "logps/rejected": -0.2964576482772827,
      "loss": 0.5485445261001587,
      "loss/core": 0.5485445261001587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.40129017829895,
      "rewards/margins": 0.9055936932563782,
      "rewards/rejected": 1.4956964254379272,
      "step": 785
    },
    {
      "epoch": 0.7548071181177595,
      "grad_norm": 3.84375,
      "learning_rate": 2.2812217756608937e-08,
      "logits/chosen": -2.113828420639038,
      "logits/rejected": -2.184523344039917,
      "logps/chosen": -0.30777841806411743,
      "logps/rejected": -0.34668484330177307,
      "loss": 0.178707093000412,
      "loss/core": 0.178707093000412,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7738380432128906,
      "rewards/margins": 1.0590564012527466,
      "rewards/rejected": 0.7147814631462097,
      "step": 790
    },
    {
      "epoch": 0.7595843783590112,
      "grad_norm": 57.25,
      "learning_rate": 2.0831748665410763e-08,
      "logits/chosen": -2.0712850093841553,
      "logits/rejected": -2.047006368637085,
      "logps/chosen": -0.2615426182746887,
      "logps/rejected": -0.2844923138618469,
      "loss": 0.5626412630081177,
      "loss/core": 0.5626412630081177,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.4369277954101562,
      "rewards/margins": 0.5682427883148193,
      "rewards/rejected": 1.8686845302581787,
      "step": 795
    },
    {
      "epoch": 0.7643616386002627,
      "grad_norm": 9.0,
      "learning_rate": 1.8937460381244967e-08,
      "logits/chosen": -1.9848846197128296,
      "logits/rejected": -2.050729751586914,
      "logps/chosen": -0.3180027902126312,
      "logps/rejected": -0.2833881378173828,
      "loss": 1.3084042072296143,
      "loss/core": 1.3084042072296143,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 3.4368815422058105,
      "rewards/margins": 2.4918322563171387,
      "rewards/rejected": 0.9450491666793823,
      "step": 800
    },
    {
      "epoch": 0.7691388988415144,
      "grad_norm": 49.0,
      "learning_rate": 1.713006526846439e-08,
      "logits/chosen": -2.084895610809326,
      "logits/rejected": -2.2298989295959473,
      "logps/chosen": -0.2837081849575043,
      "logps/rejected": -0.28118696808815,
      "loss": 0.3433898687362671,
      "loss/core": 0.3433898687362671,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.079929828643799,
      "rewards/margins": 1.1563465595245361,
      "rewards/rejected": 0.9235833287239075,
      "step": 805
    },
    {
      "epoch": 0.773916159082766,
      "grad_norm": 10.9375,
      "learning_rate": 1.541024301445404e-08,
      "logits/chosen": -2.3721537590026855,
      "logits/rejected": -2.3594298362731934,
      "logps/chosen": -0.27915048599243164,
      "logps/rejected": -0.2653045058250427,
      "loss": 0.35306280851364136,
      "loss/core": 0.35306280851364136,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.984630823135376,
      "rewards/margins": 1.1519166231155396,
      "rewards/rejected": 0.8327140808105469,
      "step": 810
    },
    {
      "epoch": 0.7786934193240177,
      "grad_norm": 45.5,
      "learning_rate": 1.3778640374027983e-08,
      "logits/chosen": -2.0560569763183594,
      "logits/rejected": -2.0075690746307373,
      "logps/chosen": -0.29053401947021484,
      "logps/rejected": -0.3152034282684326,
      "loss": 0.1969846785068512,
      "loss/core": 0.1969846785068512,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.955904245376587,
      "rewards/margins": 0.8895227313041687,
      "rewards/rejected": 1.0663814544677734,
      "step": 815
    },
    {
      "epoch": 0.7834706795652693,
      "grad_norm": 30.75,
      "learning_rate": 1.2235870926211616e-08,
      "logits/chosen": -2.0074410438537598,
      "logits/rejected": -1.998299241065979,
      "logps/chosen": -0.2895830273628235,
      "logps/rejected": -0.2900421619415283,
      "loss": 0.6202723979949951,
      "loss/core": 0.6202723979949951,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.5479187965393066,
      "rewards/margins": 1.322956919670105,
      "rewards/rejected": 1.2249616384506226,
      "step": 820
    },
    {
      "epoch": 0.788247939806521,
      "grad_norm": 1.15625,
      "learning_rate": 1.0782514843499652e-08,
      "logits/chosen": -2.06480073928833,
      "logits/rejected": -2.2259457111358643,
      "logps/chosen": -0.301548570394516,
      "logps/rejected": -0.3503800928592682,
      "loss": 0.5075355768203735,
      "loss/core": 0.5075355768203735,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.2786221504211426,
      "rewards/margins": 1.6066519021987915,
      "rewards/rejected": 0.6719702482223511,
      "step": 825
    },
    {
      "epoch": 0.7930252000477725,
      "grad_norm": 55.0,
      "learning_rate": 9.419118673676924e-09,
      "logits/chosen": -2.1703529357910156,
      "logits/rejected": -2.184058666229248,
      "logps/chosen": -0.300544798374176,
      "logps/rejected": -0.2891280949115753,
      "loss": 0.2319040298461914,
      "loss/core": 0.2319040298461914,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8630399703979492,
      "rewards/margins": 1.0912376642227173,
      "rewards/rejected": 0.7718024849891663,
      "step": 830
    },
    {
      "epoch": 0.7978024602890242,
      "grad_norm": 7.75,
      "learning_rate": 8.14619513428405e-09,
      "logits/chosen": -2.2036571502685547,
      "logits/rejected": -2.212895631790161,
      "logps/chosen": -0.2536885142326355,
      "logps/rejected": -0.2815439701080322,
      "loss": 0.3949839472770691,
      "loss/core": 0.3949839472770691,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.112321138381958,
      "rewards/margins": 1.3582528829574585,
      "rewards/rejected": 0.7540683150291443,
      "step": 835
    },
    {
      "epoch": 0.8025797205302759,
      "grad_norm": 1256.0,
      "learning_rate": 6.96422291980539e-09,
      "logits/chosen": -2.035339832305908,
      "logits/rejected": -2.0912997722625732,
      "logps/chosen": -0.29334017634391785,
      "logps/rejected": -0.29095450043678284,
      "loss": 1.3885018825531006,
      "loss/core": 1.3885018825531006,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.5305423736572266,
      "rewards/margins": 2.387678861618042,
      "rewards/rejected": 1.1428637504577637,
      "step": 840
    },
    {
      "epoch": 0.8073569807715275,
      "grad_norm": 58.25,
      "learning_rate": 5.8736465216517594e-09,
      "logits/chosen": -1.8922698497772217,
      "logits/rejected": -1.9340620040893555,
      "logps/chosen": -0.30387482047080994,
      "logps/rejected": -0.31475311517715454,
      "loss": 1.267748236656189,
      "loss/core": 1.267748236656189,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.673053503036499,
      "rewards/margins": 0.4096958041191101,
      "rewards/rejected": 2.263357639312744,
      "step": 845
    },
    {
      "epoch": 0.8121342410127792,
      "grad_norm": 19.0,
      "learning_rate": 4.874876061005173e-09,
      "logits/chosen": -1.9448999166488647,
      "logits/rejected": -2.031862735748291,
      "logps/chosen": -0.2876799702644348,
      "logps/rejected": -0.29397642612457275,
      "loss": 0.3203374445438385,
      "loss/core": 0.3203374445438385,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.3861019611358643,
      "rewards/margins": 1.2526428699493408,
      "rewards/rejected": 1.1334589719772339,
      "step": 850
    },
    {
      "epoch": 0.8169115012540308,
      "grad_norm": 16.5,
      "learning_rate": 3.968287134589188e-09,
      "logits/chosen": -1.8941272497177124,
      "logits/rejected": -1.8450911045074463,
      "logps/chosen": -0.285461962223053,
      "logps/rejected": -0.28394120931625366,
      "loss": 0.4327033460140228,
      "loss/core": 0.4327033460140228,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.151184558868408,
      "rewards/margins": 0.9308632016181946,
      "rewards/rejected": 1.220321536064148,
      "step": 855
    },
    {
      "epoch": 0.8216887614952825,
      "grad_norm": 5.65625,
      "learning_rate": 3.154220673422192e-09,
      "logits/chosen": -1.9777624607086182,
      "logits/rejected": -2.0467350482940674,
      "logps/chosen": -0.28861913084983826,
      "logps/rejected": -0.3350909352302551,
      "loss": 0.501311182975769,
      "loss/core": 0.501311182975769,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.9023014307022095,
      "rewards/margins": 0.7318955659866333,
      "rewards/rejected": 1.1704059839248657,
      "step": 860
    },
    {
      "epoch": 0.826466021736534,
      "grad_norm": 105.5,
      "learning_rate": 2.4329828146074096e-09,
      "logits/chosen": -1.8944776058197021,
      "logits/rejected": -1.953665018081665,
      "logps/chosen": -0.2609250247478485,
      "logps/rejected": -0.29491904377937317,
      "loss": 0.4003438353538513,
      "loss/core": 0.4003438353538513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.2690205574035645,
      "rewards/margins": 1.027840256690979,
      "rewards/rejected": 1.241180181503296,
      "step": 865
    },
    {
      "epoch": 0.8312432819777857,
      "grad_norm": 924.0,
      "learning_rate": 1.8048447862070714e-09,
      "logits/chosen": -2.322716236114502,
      "logits/rejected": -2.29233980178833,
      "logps/chosen": -0.3064175248146057,
      "logps/rejected": -0.29991593956947327,
      "loss": 1.024073839187622,
      "loss/core": 1.024073839187622,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.370023012161255,
      "rewards/margins": 1.0268133878707886,
      "rewards/rejected": 1.343209981918335,
      "step": 870
    },
    {
      "epoch": 0.8360205422190374,
      "grad_norm": 35.5,
      "learning_rate": 1.2700428052447033e-09,
      "logits/chosen": -2.17930006980896,
      "logits/rejected": -2.034334897994995,
      "logps/chosen": -0.27654317021369934,
      "logps/rejected": -0.310249388217926,
      "loss": 0.34868985414505005,
      "loss/core": 0.34868985414505005,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.6901212930679321,
      "rewards/margins": -0.11936825513839722,
      "rewards/rejected": 1.8094894886016846,
      "step": 875
    },
    {
      "epoch": 0.840797802460289,
      "grad_norm": 12.25,
      "learning_rate": 8.287779888734858e-10,
      "logits/chosen": -2.0425121784210205,
      "logits/rejected": -2.0631158351898193,
      "logps/chosen": -0.31236395239830017,
      "logps/rejected": -0.32643812894821167,
      "loss": 0.7430340647697449,
      "loss/core": 0.7430340647697449,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.8382582664489746,
      "rewards/margins": 1.6181519031524658,
      "rewards/rejected": 1.2201061248779297,
      "step": 880
    },
    {
      "epoch": 0.8455750627015407,
      "grad_norm": 9.375,
      "learning_rate": 4.812162787445062e-10,
      "logits/chosen": -2.1633739471435547,
      "logits/rejected": -2.2033333778381348,
      "logps/chosen": -0.309976726770401,
      "logps/rejected": -0.3008884787559509,
      "loss": 0.05537397414445877,
      "loss/core": 0.05537397414445877,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.1624144315719604,
      "rewards/margins": 0.6319789290428162,
      "rewards/rejected": 0.5304354429244995,
      "step": 885
    },
    {
      "epoch": 0.8503523229427923,
      "grad_norm": 201.0,
      "learning_rate": 2.2748837860270265e-10,
      "logits/chosen": -2.046548366546631,
      "logits/rejected": -2.1142578125,
      "logps/chosen": -0.33651334047317505,
      "logps/rejected": -0.32960960268974304,
      "loss": 0.38854843378067017,
      "loss/core": 0.38854843378067017,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.0308008193969727,
      "rewards/margins": 1.3045727014541626,
      "rewards/rejected": 0.7262283563613892,
      "step": 890
    },
    {
      "epoch": 0.855129583184044,
      "grad_norm": 6.53125,
      "learning_rate": 6.768970513457151e-11,
      "logits/chosen": -2.0591623783111572,
      "logits/rejected": -2.0446290969848633,
      "logps/chosen": -0.2869151532649994,
      "logps/rejected": -0.2770235538482666,
      "loss": 0.31208789348602295,
      "loss/core": 0.31208789348602295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1741526126861572,
      "rewards/margins": 1.1323846578598022,
      "rewards/rejected": 1.0417680740356445,
      "step": 895
    },
    {
      "epoch": 0.8599068434252956,
      "grad_norm": 117.5,
      "learning_rate": 1.8803520859811406e-12,
      "logits/chosen": -2.1362462043762207,
      "logits/rejected": -2.0990567207336426,
      "logps/chosen": -0.3250855803489685,
      "logps/rejected": -0.332465261220932,
      "loss": 1.7968904972076416,
      "loss/core": 1.7968904972076416,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 4.322796821594238,
      "rewards/margins": 2.9066953659057617,
      "rewards/rejected": 1.4161012172698975,
      "step": 900
    },
    {
      "epoch": 0.8599068434252956,
      "step": 900,
      "total_flos": 0.0,
      "train_loss": 0.6663506550259061,
      "train_runtime": 7058.8291,
      "train_samples_per_second": 2.04,
      "train_steps_per_second": 0.127
    }
  ],
  "logging_steps": 5,
  "max_steps": 900,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 900,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}