{
  "best_metric": 0.800000011920929,
  "best_model_checkpoint": "./outputs/tinyllama-1.1b-dpo-pku-saferlhf/checkpoint-1400",
  "epoch": 0.9997600191984641,
  "eval_steps": 200,
  "global_step": 2083,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004799616030717543,
      "grad_norm": 56.5,
      "learning_rate": 2.3923444976076555e-07,
      "logits/chosen": -2.689218282699585,
      "logits/rejected": -2.554370880126953,
      "logps/chosen": -212.5878143310547,
      "logps/rejected": -186.63473510742188,
      "loss": 0.693,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": 4.648463800549507e-05,
      "rewards/margins": 0.0005794697208330035,
      "rewards/rejected": -0.0005329854902811348,
      "step": 10
    },
    {
      "epoch": 0.009599232061435085,
      "grad_norm": 59.75,
      "learning_rate": 4.784688995215311e-07,
      "logits/chosen": -2.7294280529022217,
      "logits/rejected": -2.6172096729278564,
      "logps/chosen": -223.5776824951172,
      "logps/rejected": -203.3684539794922,
      "loss": 0.6931,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0015927983913570642,
      "rewards/margins": 0.0005491179181262851,
      "rewards/rejected": 0.0010436807060614228,
      "step": 20
    },
    {
      "epoch": 0.014398848092152628,
      "grad_norm": 50.25,
      "learning_rate": 7.177033492822967e-07,
      "logits/chosen": -2.7166686058044434,
      "logits/rejected": -2.6399471759796143,
      "logps/chosen": -238.0247344970703,
      "logps/rejected": -219.1692352294922,
      "loss": 0.6949,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.003216738346964121,
      "rewards/margins": -0.0030289709102362394,
      "rewards/rejected": -0.0001877670583780855,
      "step": 30
    },
    {
      "epoch": 0.01919846412287017,
      "grad_norm": 57.0,
      "learning_rate": 9.569377990430622e-07,
      "logits/chosen": -2.7442615032196045,
      "logits/rejected": -2.5918688774108887,
      "logps/chosen": -251.1165771484375,
      "logps/rejected": -196.37649536132812,
      "loss": 0.6883,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.00424658227711916,
      "rewards/margins": 0.010354852303862572,
      "rewards/rejected": -0.006108270026743412,
      "step": 40
    },
    {
      "epoch": 0.023998080153587713,
      "grad_norm": 47.25,
      "learning_rate": 1.196172248803828e-06,
      "logits/chosen": -2.6663458347320557,
      "logits/rejected": -2.604515552520752,
      "logps/chosen": -234.8127899169922,
      "logps/rejected": -199.76278686523438,
      "loss": 0.6904,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0006293629994615912,
      "rewards/margins": 0.006024296395480633,
      "rewards/rejected": -0.00539493327960372,
      "step": 50
    },
    {
      "epoch": 0.028797696184305256,
      "grad_norm": 65.0,
      "learning_rate": 1.4354066985645934e-06,
      "logits/chosen": -2.7063660621643066,
      "logits/rejected": -2.6000542640686035,
      "logps/chosen": -223.295166015625,
      "logps/rejected": -215.13656616210938,
      "loss": 0.6887,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0032431543804705143,
      "rewards/margins": 0.009618126787245274,
      "rewards/rejected": -0.012861279770731926,
      "step": 60
    },
    {
      "epoch": 0.033597312215022795,
      "grad_norm": 50.0,
      "learning_rate": 1.6746411483253591e-06,
      "logits/chosen": -2.727038860321045,
      "logits/rejected": -2.585761070251465,
      "logps/chosen": -245.62393188476562,
      "logps/rejected": -205.93148803710938,
      "loss": 0.6877,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005830951035022736,
      "rewards/margins": 0.011942476034164429,
      "rewards/rejected": -0.006111525930464268,
      "step": 70
    },
    {
      "epoch": 0.03839692824574034,
      "grad_norm": 50.25,
      "learning_rate": 1.9138755980861244e-06,
      "logits/chosen": -2.7241501808166504,
      "logits/rejected": -2.6147332191467285,
      "logps/chosen": -235.19338989257812,
      "logps/rejected": -201.25424194335938,
      "loss": 0.6807,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.01283216755837202,
      "rewards/margins": 0.026413241401314735,
      "rewards/rejected": -0.01358107291162014,
      "step": 80
    },
    {
      "epoch": 0.04319654427645788,
      "grad_norm": 49.5,
      "learning_rate": 2.15311004784689e-06,
      "logits/chosen": -2.7387120723724365,
      "logits/rejected": -2.5572338104248047,
      "logps/chosen": -250.86380004882812,
      "logps/rejected": -189.31814575195312,
      "loss": 0.6712,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.03785385563969612,
      "rewards/margins": 0.04662873595952988,
      "rewards/rejected": -0.00877488125115633,
      "step": 90
    },
    {
      "epoch": 0.04799616030717543,
      "grad_norm": 49.0,
      "learning_rate": 2.392344497607656e-06,
      "logits/chosen": -2.7061264514923096,
      "logits/rejected": -2.5762696266174316,
      "logps/chosen": -227.9839630126953,
      "logps/rejected": -206.23196411132812,
      "loss": 0.6702,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06363014876842499,
      "rewards/margins": 0.05047706514596939,
      "rewards/rejected": 0.013153081759810448,
      "step": 100
    },
    {
      "epoch": 0.052795776337892966,
      "grad_norm": 49.5,
      "learning_rate": 2.631578947368421e-06,
      "logits/chosen": -2.7290475368499756,
      "logits/rejected": -2.548861026763916,
      "logps/chosen": -236.59375,
      "logps/rejected": -178.90164184570312,
      "loss": 0.6619,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06783770024776459,
      "rewards/margins": 0.070295050740242,
      "rewards/rejected": -0.0024573481641709805,
      "step": 110
    },
    {
      "epoch": 0.05759539236861051,
      "grad_norm": 52.0,
      "learning_rate": 2.870813397129187e-06,
      "logits/chosen": -2.7224419116973877,
      "logits/rejected": -2.5972402095794678,
      "logps/chosen": -233.290283203125,
      "logps/rejected": -204.4892578125,
      "loss": 0.6691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08400879800319672,
      "rewards/margins": 0.05939972400665283,
      "rewards/rejected": 0.024609070271253586,
      "step": 120
    },
    {
      "epoch": 0.06239500839932805,
      "grad_norm": 50.25,
      "learning_rate": 3.1100478468899525e-06,
      "logits/chosen": -2.7070722579956055,
      "logits/rejected": -2.5772013664245605,
      "logps/chosen": -250.78231811523438,
      "logps/rejected": -209.5062713623047,
      "loss": 0.6496,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.165785014629364,
      "rewards/margins": 0.10047496855258942,
      "rewards/rejected": 0.06531006097793579,
      "step": 130
    },
    {
      "epoch": 0.06719462443004559,
      "grad_norm": 49.5,
      "learning_rate": 3.3492822966507182e-06,
      "logits/chosen": -2.6727123260498047,
      "logits/rejected": -2.6064090728759766,
      "logps/chosen": -233.5751495361328,
      "logps/rejected": -217.31576538085938,
      "loss": 0.6574,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.20782318711280823,
      "rewards/margins": 0.0912146344780922,
      "rewards/rejected": 0.11660852283239365,
      "step": 140
    },
    {
      "epoch": 0.07199424046076314,
      "grad_norm": 45.75,
      "learning_rate": 3.5885167464114835e-06,
      "logits/chosen": -2.6911652088165283,
      "logits/rejected": -2.6086039543151855,
      "logps/chosen": -230.4497528076172,
      "logps/rejected": -244.7624053955078,
      "loss": 0.6583,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.19385293126106262,
      "rewards/margins": 0.0921817347407341,
      "rewards/rejected": 0.10167120397090912,
      "step": 150
    },
    {
      "epoch": 0.07679385649148068,
      "grad_norm": 51.5,
      "learning_rate": 3.827751196172249e-06,
      "logits/chosen": -2.65492844581604,
      "logits/rejected": -2.5312721729278564,
      "logps/chosen": -241.20199584960938,
      "logps/rejected": -200.1884765625,
      "loss": 0.6299,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.1793375313282013,
      "rewards/margins": 0.1596115678548813,
      "rewards/rejected": 0.019725963473320007,
      "step": 160
    },
    {
      "epoch": 0.08159347252219823,
      "grad_norm": 46.25,
      "learning_rate": 4.066985645933015e-06,
      "logits/chosen": -2.732844114303589,
      "logits/rejected": -2.5666396617889404,
      "logps/chosen": -255.0216827392578,
      "logps/rejected": -198.9121856689453,
      "loss": 0.6366,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.15441159904003143,
      "rewards/margins": 0.144814133644104,
      "rewards/rejected": 0.009597455151379108,
      "step": 170
    },
    {
      "epoch": 0.08639308855291576,
      "grad_norm": 43.25,
      "learning_rate": 4.30622009569378e-06,
      "logits/chosen": -2.7506613731384277,
      "logits/rejected": -2.6481566429138184,
      "logps/chosen": -240.16055297851562,
      "logps/rejected": -200.94822692871094,
      "loss": 0.6103,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.23929600417613983,
      "rewards/margins": 0.20658496022224426,
      "rewards/rejected": 0.032711055129766464,
      "step": 180
    },
    {
      "epoch": 0.09119270458363331,
      "grad_norm": 50.75,
      "learning_rate": 4.5454545454545455e-06,
      "logits/chosen": -2.675985097885132,
      "logits/rejected": -2.5708577632904053,
      "logps/chosen": -238.582763671875,
      "logps/rejected": -198.5998992919922,
      "loss": 0.6178,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.23324120044708252,
      "rewards/margins": 0.195271834731102,
      "rewards/rejected": 0.03796938806772232,
      "step": 190
    },
    {
      "epoch": 0.09599232061435085,
      "grad_norm": 37.25,
      "learning_rate": 4.784688995215312e-06,
      "logits/chosen": -2.7069146633148193,
      "logits/rejected": -2.5851969718933105,
      "logps/chosen": -225.49917602539062,
      "logps/rejected": -189.277587890625,
      "loss": 0.6075,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.3116636574268341,
      "rewards/margins": 0.22872300446033478,
      "rewards/rejected": 0.08294066041707993,
      "step": 200
    },
    {
      "epoch": 0.09599232061435085,
      "eval_logits/chosen": -2.6887047290802,
      "eval_logits/rejected": -2.5653347969055176,
      "eval_logps/chosen": -228.92459106445312,
      "eval_logps/rejected": -203.6246337890625,
      "eval_loss": 0.6051958799362183,
      "eval_rewards/accuracies": 0.7170000076293945,
      "eval_rewards/chosen": 0.4208393394947052,
      "eval_rewards/margins": 0.23022359609603882,
      "eval_rewards/rejected": 0.1906157284975052,
      "eval_runtime": 26.0524,
      "eval_samples_per_second": 38.384,
      "eval_steps_per_second": 9.596,
      "step": 200
    },
    {
      "epoch": 0.1007919366450684,
      "grad_norm": 45.75,
      "learning_rate": 4.999996487062011e-06,
      "logits/chosen": -2.645963668823242,
      "logits/rejected": -2.5576295852661133,
      "logps/chosen": -237.7938995361328,
      "logps/rejected": -211.9795684814453,
      "loss": 0.6059,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.4848661422729492,
      "rewards/margins": 0.2471170723438263,
      "rewards/rejected": 0.2377490997314453,
      "step": 210
    },
    {
      "epoch": 0.10559155267578593,
      "grad_norm": 41.5,
      "learning_rate": 4.999574946449064e-06,
      "logits/chosen": -2.6820342540740967,
      "logits/rejected": -2.550971269607544,
      "logps/chosen": -222.2080841064453,
      "logps/rejected": -184.314208984375,
      "loss": 0.6094,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.38701874017715454,
      "rewards/margins": 0.22917309403419495,
      "rewards/rejected": 0.1578456610441208,
      "step": 220
    },
    {
      "epoch": 0.11039116870650348,
      "grad_norm": 41.0,
      "learning_rate": 4.9984509539801644e-06,
      "logits/chosen": -2.6449620723724365,
      "logits/rejected": -2.532531976699829,
      "logps/chosen": -227.45291137695312,
      "logps/rejected": -221.8866729736328,
      "loss": 0.6227,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.3856458067893982,
      "rewards/margins": 0.22975265979766846,
      "rewards/rejected": 0.15589316189289093,
      "step": 230
    },
    {
      "epoch": 0.11519078473722102,
      "grad_norm": 48.0,
      "learning_rate": 4.996624825529257e-06,
      "logits/chosen": -2.7190098762512207,
      "logits/rejected": -2.6080780029296875,
      "logps/chosen": -212.9981231689453,
      "logps/rejected": -191.4314727783203,
      "loss": 0.5988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36744850873947144,
      "rewards/margins": 0.2741442620754242,
      "rewards/rejected": 0.09330429136753082,
      "step": 240
    },
    {
      "epoch": 0.11999040076793857,
      "grad_norm": 47.5,
      "learning_rate": 4.994097074290524e-06,
      "logits/chosen": -2.6760241985321045,
      "logits/rejected": -2.555823564529419,
      "logps/chosen": -225.98098754882812,
      "logps/rejected": -199.85360717773438,
      "loss": 0.6114,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.2855023443698883,
      "rewards/margins": 0.23584535717964172,
      "rewards/rejected": 0.04965699464082718,
      "step": 250
    },
    {
      "epoch": 0.1247900167986561,
      "grad_norm": 45.0,
      "learning_rate": 4.990868410634163e-06,
      "logits/chosen": -2.683492660522461,
      "logits/rejected": -2.59763240814209,
      "logps/chosen": -222.82955932617188,
      "logps/rejected": -192.34671020507812,
      "loss": 0.592,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.2924219071865082,
      "rewards/margins": 0.2935822010040283,
      "rewards/rejected": -0.0011602870654314756,
      "step": 260
    },
    {
      "epoch": 0.12958963282937366,
      "grad_norm": 42.0,
      "learning_rate": 4.9869397419067535e-06,
      "logits/chosen": -2.6904869079589844,
      "logits/rejected": -2.58687162399292,
      "logps/chosen": -210.76589965820312,
      "logps/rejected": -191.6869354248047,
      "loss": 0.5514,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.34391769766807556,
      "rewards/margins": 0.35880047082901,
      "rewards/rejected": -0.014882763847708702,
      "step": 270
    },
    {
      "epoch": 0.13438924886009118,
      "grad_norm": 46.75,
      "learning_rate": 4.982312172176264e-06,
      "logits/chosen": -2.7495617866516113,
      "logits/rejected": -2.549598217010498,
      "logps/chosen": -269.2088928222656,
      "logps/rejected": -204.53211975097656,
      "loss": 0.5697,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.4326564371585846,
      "rewards/margins": 0.35469716787338257,
      "rewards/rejected": 0.07795925438404083,
      "step": 280
    },
    {
      "epoch": 0.13918886489080873,
      "grad_norm": 57.0,
      "learning_rate": 4.976987001921787e-06,
      "logits/chosen": -2.669323444366455,
      "logits/rejected": -2.547853708267212,
      "logps/chosen": -232.5638885498047,
      "logps/rejected": -205.17001342773438,
      "loss": 0.5782,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.3145369589328766,
      "rewards/margins": 0.34478241205215454,
      "rewards/rejected": -0.030245428904891014,
      "step": 290
    },
    {
      "epoch": 0.14398848092152627,
      "grad_norm": 37.75,
      "learning_rate": 4.97096572766805e-06,
      "logits/chosen": -2.6843204498291016,
      "logits/rejected": -2.5420849323272705,
      "logps/chosen": -238.2855987548828,
      "logps/rejected": -192.5761260986328,
      "loss": 0.548,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.2724495232105255,
      "rewards/margins": 0.4326988756656647,
      "rewards/rejected": -0.16024938225746155,
      "step": 300
    },
    {
      "epoch": 0.14878809695224382,
      "grad_norm": 43.75,
      "learning_rate": 4.964250041564868e-06,
      "logits/chosen": -2.664036989212036,
      "logits/rejected": -2.533687114715576,
      "logps/chosen": -230.8778076171875,
      "logps/rejected": -200.39389038085938,
      "loss": 0.5758,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.21766161918640137,
      "rewards/margins": 0.3927158713340759,
      "rewards/rejected": -0.17505425214767456,
      "step": 310
    },
    {
      "epoch": 0.15358771298296137,
      "grad_norm": 38.25,
      "learning_rate": 4.956841830911588e-06,
      "logits/chosen": -2.6427550315856934,
      "logits/rejected": -2.5176196098327637,
      "logps/chosen": -241.25448608398438,
      "logps/rejected": -200.30337524414062,
      "loss": 0.5428,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.3211400508880615,
      "rewards/margins": 0.465925931930542,
      "rewards/rejected": -0.14478585124015808,
      "step": 320
    },
    {
      "epoch": 0.1583873290136789,
      "grad_norm": 53.5,
      "learning_rate": 4.9487431776267095e-06,
      "logits/chosen": -2.6581196784973145,
      "logits/rejected": -2.5447604656219482,
      "logps/chosen": -214.78573608398438,
      "logps/rejected": -197.250732421875,
      "loss": 0.5574,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.35328906774520874,
      "rewards/margins": 0.42138591408729553,
      "rewards/rejected": -0.0680968165397644,
      "step": 330
    },
    {
      "epoch": 0.16318694504439646,
      "grad_norm": 42.0,
      "learning_rate": 4.939956357662806e-06,
      "logits/chosen": -2.609428882598877,
      "logits/rejected": -2.4489550590515137,
      "logps/chosen": -230.47982788085938,
      "logps/rejected": -180.9293975830078,
      "loss": 0.5271,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.2847859263420105,
      "rewards/margins": 0.49130839109420776,
      "rewards/rejected": -0.20652246475219727,
      "step": 340
    },
    {
      "epoch": 0.16798656107511398,
      "grad_norm": 42.75,
      "learning_rate": 4.9304838403669155e-06,
      "logits/chosen": -2.586233139038086,
      "logits/rejected": -2.4430768489837646,
      "logps/chosen": -250.468017578125,
      "logps/rejected": -196.97486877441406,
      "loss": 0.535,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.3728100061416626,
      "rewards/margins": 0.4706306457519531,
      "rewards/rejected": -0.0978206992149353,
      "step": 350
    },
    {
      "epoch": 0.17278617710583152,
      "grad_norm": 46.5,
      "learning_rate": 4.920328287786587e-06,
      "logits/chosen": -2.603339910507202,
      "logits/rejected": -2.4968552589416504,
      "logps/chosen": -227.32632446289062,
      "logps/rejected": -196.89515686035156,
      "loss": 0.5242,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.26675719022750854,
      "rewards/margins": 0.5004665851593018,
      "rewards/rejected": -0.23370936512947083,
      "step": 360
    },
    {
      "epoch": 0.17758579313654907,
      "grad_norm": 44.5,
      "learning_rate": 4.909492553921761e-06,
      "logits/chosen": -2.6493752002716064,
      "logits/rejected": -2.486344337463379,
      "logps/chosen": -241.6817626953125,
      "logps/rejected": -204.93899536132812,
      "loss": 0.5103,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.33704155683517456,
      "rewards/margins": 0.5699586868286133,
      "rewards/rejected": -0.23291714489459991,
      "step": 370
    },
    {
      "epoch": 0.18238540916726662,
      "grad_norm": 39.75,
      "learning_rate": 4.897979683922728e-06,
      "logits/chosen": -2.670883893966675,
      "logits/rejected": -2.564422130584717,
      "logps/chosen": -215.97842407226562,
      "logps/rejected": -182.91036987304688,
      "loss": 0.5319,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.2965424954891205,
      "rewards/margins": 0.5261348485946655,
      "rewards/rejected": -0.22959236800670624,
      "step": 380
    },
    {
      "epoch": 0.18718502519798416,
      "grad_norm": 38.25,
      "learning_rate": 4.885792913234339e-06,
      "logits/chosen": -2.603323221206665,
      "logits/rejected": -2.5463268756866455,
      "logps/chosen": -219.7723388671875,
      "logps/rejected": -210.0283203125,
      "loss": 0.5334,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.3986433148384094,
      "rewards/margins": 0.5653668642044067,
      "rewards/rejected": -0.16672348976135254,
      "step": 390
    },
    {
      "epoch": 0.1919846412287017,
      "grad_norm": 41.0,
      "learning_rate": 4.872935666686767e-06,
      "logits/chosen": -2.6278882026672363,
      "logits/rejected": -2.5184621810913086,
      "logps/chosen": -229.7825164794922,
      "logps/rejected": -213.7990264892578,
      "loss": 0.5198,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.38563206791877747,
      "rewards/margins": 0.5534602999687195,
      "rewards/rejected": -0.16782823204994202,
      "step": 400
    },
    {
      "epoch": 0.1919846412287017,
      "eval_logits/chosen": -2.6478958129882812,
      "eval_logits/rejected": -2.5310354232788086,
      "eval_logps/chosen": -230.05001831054688,
      "eval_logps/rejected": -208.3632354736328,
      "eval_loss": 0.5145431160926819,
      "eval_rewards/accuracies": 0.7850000262260437,
      "eval_rewards/chosen": 0.3082956075668335,
      "eval_rewards/margins": 0.591540515422821,
      "eval_rewards/rejected": -0.28324493765830994,
      "eval_runtime": 26.9361,
      "eval_samples_per_second": 37.125,
      "eval_steps_per_second": 9.281,
      "step": 400
    },
    {
      "epoch": 0.19678425725941925,
      "grad_norm": 49.25,
      "learning_rate": 4.859411557533019e-06,
      "logits/chosen": -2.640899181365967,
      "logits/rejected": -2.532985210418701,
      "logps/chosen": -227.2956085205078,
      "logps/rejected": -199.454345703125,
      "loss": 0.5372,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.2837000787258148,
      "rewards/margins": 0.5689016580581665,
      "rewards/rejected": -0.2852015495300293,
      "step": 410
    },
    {
      "epoch": 0.2015838732901368,
      "grad_norm": 36.25,
      "learning_rate": 4.8452243864335216e-06,
      "logits/chosen": -2.6203932762145996,
      "logits/rejected": -2.546025037765503,
      "logps/chosen": -206.45437622070312,
      "logps/rejected": -212.8076171875,
      "loss": 0.5688,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.15797384083271027,
      "rewards/margins": 0.42366623878479004,
      "rewards/rejected": -0.2656923830509186,
      "step": 420
    },
    {
      "epoch": 0.20638348932085432,
      "grad_norm": 34.75,
      "learning_rate": 4.830378140388016e-06,
      "logits/chosen": -2.726120710372925,
      "logits/rejected": -2.5788116455078125,
      "logps/chosen": -236.8133544921875,
      "logps/rejected": -197.07266235351562,
      "loss": 0.5284,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.21513256430625916,
      "rewards/margins": 0.62417072057724,
      "rewards/rejected": -0.40903815627098083,
      "step": 430
    },
    {
      "epoch": 0.21118310535157186,
      "grad_norm": 37.5,
      "learning_rate": 4.814876991615104e-06,
      "logits/chosen": -2.612753391265869,
      "logits/rejected": -2.5211071968078613,
      "logps/chosen": -228.63931274414062,
      "logps/rejected": -203.56838989257812,
      "loss": 0.5862,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.15136560797691345,
      "rewards/margins": 0.43973368406295776,
      "rewards/rejected": -0.5910992622375488,
      "step": 440
    },
    {
      "epoch": 0.2159827213822894,
      "grad_norm": 53.5,
      "learning_rate": 4.798725296379736e-06,
      "logits/chosen": -2.6407535076141357,
      "logits/rejected": -2.5609383583068848,
      "logps/chosen": -220.7452392578125,
      "logps/rejected": -198.64051818847656,
      "loss": 0.5094,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.08102991431951523,
      "rewards/margins": 0.5998750329017639,
      "rewards/rejected": -0.5188450813293457,
      "step": 450
    },
    {
      "epoch": 0.22078233741300696,
      "grad_norm": 44.25,
      "learning_rate": 4.781927593768969e-06,
      "logits/chosen": -2.6856577396392822,
      "logits/rejected": -2.5593128204345703,
      "logps/chosen": -232.1822509765625,
      "logps/rejected": -205.1845703125,
      "loss": 0.5215,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.05589114502072334,
      "rewards/margins": 0.6284939050674438,
      "rewards/rejected": -0.5726026892662048,
      "step": 460
    },
    {
      "epoch": 0.2255819534437245,
      "grad_norm": 46.75,
      "learning_rate": 4.764488604416365e-06,
      "logits/chosen": -2.6717689037323,
      "logits/rejected": -2.506671667098999,
      "logps/chosen": -255.5167236328125,
      "logps/rejected": -221.23788452148438,
      "loss": 0.4516,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.2418864220380783,
      "rewards/margins": 0.806577205657959,
      "rewards/rejected": -0.5646907687187195,
      "step": 470
    },
    {
      "epoch": 0.23038156947444205,
      "grad_norm": 40.25,
      "learning_rate": 4.7464132291753464e-06,
      "logits/chosen": -2.613154888153076,
      "logits/rejected": -2.5245513916015625,
      "logps/chosen": -212.85330200195312,
      "logps/rejected": -193.86257934570312,
      "loss": 0.5092,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.12177534401416779,
      "rewards/margins": 0.6296752691268921,
      "rewards/rejected": -0.5078999996185303,
      "step": 480
    },
    {
      "epoch": 0.2351811855051596,
      "grad_norm": 42.0,
      "learning_rate": 4.727706547741924e-06,
      "logits/chosen": -2.6002917289733887,
      "logits/rejected": -2.469181537628174,
      "logps/chosen": -228.36746215820312,
      "logps/rejected": -184.94297790527344,
      "loss": 0.5446,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.18128438293933868,
      "rewards/margins": 0.4943477213382721,
      "rewards/rejected": -0.31306329369544983,
      "step": 490
    },
    {
      "epoch": 0.23998080153587714,
      "grad_norm": 41.75,
      "learning_rate": 4.708373817227158e-06,
      "logits/chosen": -2.5972156524658203,
      "logits/rejected": -2.468087673187256,
      "logps/chosen": -240.36544799804688,
      "logps/rejected": -206.00051879882812,
      "loss": 0.5061,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.16172266006469727,
      "rewards/margins": 0.651920735836029,
      "rewards/rejected": -0.49019813537597656,
      "step": 500
    },
    {
      "epoch": 0.24478041756659466,
      "grad_norm": 35.75,
      "learning_rate": 4.688420470679754e-06,
      "logits/chosen": -2.5911366939544678,
      "logits/rejected": -2.4612526893615723,
      "logps/chosen": -244.9521942138672,
      "logps/rejected": -197.1681365966797,
      "loss": 0.4955,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.15817375481128693,
      "rewards/margins": 0.7028775215148926,
      "rewards/rejected": -0.5447037816047668,
      "step": 510
    },
    {
      "epoch": 0.2495800335973122,
      "grad_norm": 37.75,
      "learning_rate": 4.667852115559227e-06,
      "logits/chosen": -2.6258175373077393,
      "logits/rejected": -2.492220401763916,
      "logps/chosen": -254.71044921875,
      "logps/rejected": -222.6087188720703,
      "loss": 0.4794,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.30519407987594604,
      "rewards/margins": 0.7252100706100464,
      "rewards/rejected": -0.4200161099433899,
      "step": 520
    },
    {
      "epoch": 0.2543796496280298,
      "grad_norm": 38.5,
      "learning_rate": 4.646674532160041e-06,
      "logits/chosen": -2.657778739929199,
      "logits/rejected": -2.5602223873138428,
      "logps/chosen": -233.77200317382812,
      "logps/rejected": -212.9705810546875,
      "loss": 0.5216,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1333063244819641,
      "rewards/margins": 0.635170042514801,
      "rewards/rejected": -0.5018636584281921,
      "step": 530
    },
    {
      "epoch": 0.2591792656587473,
      "grad_norm": 48.75,
      "learning_rate": 4.6248936719871855e-06,
      "logits/chosen": -2.6264724731445312,
      "logits/rejected": -2.5388011932373047,
      "logps/chosen": -220.25064086914062,
      "logps/rejected": -194.9637451171875,
      "loss": 0.4623,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.02713530696928501,
      "rewards/margins": 0.8179550170898438,
      "rewards/rejected": -0.7908197045326233,
      "step": 540
    },
    {
      "epoch": 0.2639788816894648,
      "grad_norm": 43.5,
      "learning_rate": 4.60251565608363e-06,
      "logits/chosen": -2.687124729156494,
      "logits/rejected": -2.6088709831237793,
      "logps/chosen": -234.5717010498047,
      "logps/rejected": -225.5623779296875,
      "loss": 0.4995,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.011210719123482704,
      "rewards/margins": 0.6683915257453918,
      "rewards/rejected": -0.6796022653579712,
      "step": 550
    },
    {
      "epoch": 0.26877849772018236,
      "grad_norm": 39.75,
      "learning_rate": 4.579546773310136e-06,
      "logits/chosen": -2.619655132293701,
      "logits/rejected": -2.446974754333496,
      "logps/chosen": -238.61904907226562,
      "logps/rejected": -216.04629516601562,
      "loss": 0.4966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06748590618371964,
      "rewards/margins": 0.7394402027130127,
      "rewards/rejected": -0.6719542741775513,
      "step": 560
    },
    {
      "epoch": 0.2735781137508999,
      "grad_norm": 44.0,
      "learning_rate": 4.5559934785779115e-06,
      "logits/chosen": -2.667036771774292,
      "logits/rejected": -2.469698190689087,
      "logps/chosen": -243.6424560546875,
      "logps/rejected": -192.75326538085938,
      "loss": 0.4458,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.23885676264762878,
      "rewards/margins": 0.8359629511833191,
      "rewards/rejected": -0.5971060991287231,
      "step": 570
    },
    {
      "epoch": 0.27837772978161746,
      "grad_norm": 39.75,
      "learning_rate": 4.531862391034591e-06,
      "logits/chosen": -2.58920955657959,
      "logits/rejected": -2.4982354640960693,
      "logps/chosen": -232.32406616210938,
      "logps/rejected": -205.0082244873047,
      "loss": 0.4441,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.24206867814064026,
      "rewards/margins": 0.9316803216934204,
      "rewards/rejected": -0.689611554145813,
      "step": 580
    },
    {
      "epoch": 0.283177345812335,
      "grad_norm": 52.25,
      "learning_rate": 4.507160292204074e-06,
      "logits/chosen": -2.675287961959839,
      "logits/rejected": -2.5451769828796387,
      "logps/chosen": -237.7034149169922,
      "logps/rejected": -209.1581573486328,
      "loss": 0.4854,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.012547047808766365,
      "rewards/margins": 0.7794401049613953,
      "rewards/rejected": -0.7919871807098389,
      "step": 590
    },
    {
      "epoch": 0.28797696184305255,
      "grad_norm": 45.25,
      "learning_rate": 4.481894124080714e-06,
      "logits/chosen": -2.6691808700561523,
      "logits/rejected": -2.557055950164795,
      "logps/chosen": -236.5981903076172,
      "logps/rejected": -220.3197021484375,
      "loss": 0.4703,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08622226119041443,
      "rewards/margins": 0.7651070952415466,
      "rewards/rejected": -0.8513293266296387,
      "step": 600
    },
    {
      "epoch": 0.28797696184305255,
      "eval_logits/chosen": -2.633901596069336,
      "eval_logits/rejected": -2.5215346813201904,
      "eval_logps/chosen": -233.1143798828125,
      "eval_logps/rejected": -213.52012634277344,
      "eval_loss": 0.4838341772556305,
      "eval_rewards/accuracies": 0.7940000295639038,
      "eval_rewards/chosen": 0.0018612403655424714,
      "eval_rewards/margins": 0.8007965683937073,
      "eval_rewards/rejected": -0.7989352941513062,
      "eval_runtime": 31.8483,
      "eval_samples_per_second": 31.399,
      "eval_steps_per_second": 7.85,
      "step": 600
    },
    {
      "epoch": 0.2927765778737701,
      "grad_norm": 37.0,
      "learning_rate": 4.456070987178427e-06,
      "logits/chosen": -2.636303424835205,
      "logits/rejected": -2.475978136062622,
      "logps/chosen": -220.087646484375,
      "logps/rejected": -184.07369995117188,
      "loss": 0.4705,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.108073391020298,
      "rewards/margins": 0.7934460639953613,
      "rewards/rejected": -0.9015194177627563,
      "step": 610
    },
    {
      "epoch": 0.29757619390448764,
      "grad_norm": 38.25,
      "learning_rate": 4.429698138535242e-06,
      "logits/chosen": -2.6021456718444824,
      "logits/rejected": -2.510624885559082,
      "logps/chosen": -237.7132110595703,
      "logps/rejected": -222.8521728515625,
      "loss": 0.5149,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.16316601634025574,
      "rewards/margins": 0.6969932317733765,
      "rewards/rejected": -0.5338272452354431,
      "step": 620
    },
    {
      "epoch": 0.3023758099352052,
      "grad_norm": 41.0,
      "learning_rate": 4.402782989673867e-06,
      "logits/chosen": -2.648524761199951,
      "logits/rejected": -2.49703311920166,
      "logps/chosen": -240.3001251220703,
      "logps/rejected": -206.89743041992188,
      "loss": 0.4332,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16457466781139374,
      "rewards/margins": 0.8773033022880554,
      "rewards/rejected": -0.7127286195755005,
      "step": 630
    },
    {
      "epoch": 0.30717542596592273,
      "grad_norm": 53.25,
      "learning_rate": 4.375333104518842e-06,
      "logits/chosen": -2.567253589630127,
      "logits/rejected": -2.5297513008117676,
      "logps/chosen": -224.3708038330078,
      "logps/rejected": -223.1370086669922,
      "loss": 0.5021,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.020184341818094254,
      "rewards/margins": 0.7216410636901855,
      "rewards/rejected": -0.7418254613876343,
      "step": 640
    },
    {
      "epoch": 0.3119750419966403,
      "grad_norm": 52.0,
      "learning_rate": 4.347356197270852e-06,
      "logits/chosen": -2.629487991333008,
      "logits/rejected": -2.4799911975860596,
      "logps/chosen": -232.6698455810547,
      "logps/rejected": -211.57406616210938,
      "loss": 0.4987,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.05573948472738266,
      "rewards/margins": 0.8003142476081848,
      "rewards/rejected": -0.744574785232544,
      "step": 650
    },
    {
      "epoch": 0.3167746580273578,
      "grad_norm": 51.25,
      "learning_rate": 4.318860130238828e-06,
      "logits/chosen": -2.564279794692993,
      "logits/rejected": -2.4956247806549072,
      "logps/chosen": -216.66201782226562,
      "logps/rejected": -223.7574005126953,
      "loss": 0.4958,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.15754355490207672,
      "rewards/margins": 0.775588870048523,
      "rewards/rejected": -0.618045449256897,
      "step": 660
    },
    {
      "epoch": 0.32157427405807537,
      "grad_norm": 35.25,
      "learning_rate": 4.289852911630407e-06,
      "logits/chosen": -2.656132221221924,
      "logits/rejected": -2.506979465484619,
      "logps/chosen": -259.60101318359375,
      "logps/rejected": -215.4107208251953,
      "loss": 0.4511,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.25082486867904663,
      "rewards/margins": 0.930018424987793,
      "rewards/rejected": -0.6791934967041016,
      "step": 670
    },
    {
      "epoch": 0.3263738900887929,
      "grad_norm": 65.5,
      "learning_rate": 4.260342693301396e-06,
      "logits/chosen": -2.630061626434326,
      "logits/rejected": -2.4976465702056885,
      "logps/chosen": -234.99032592773438,
      "logps/rejected": -197.80252075195312,
      "loss": 0.4984,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.04209694266319275,
      "rewards/margins": 0.7394791841506958,
      "rewards/rejected": -0.6973822116851807,
      "step": 680
    },
    {
      "epoch": 0.33117350611951046,
      "grad_norm": 36.25,
      "learning_rate": 4.2303377684648735e-06,
      "logits/chosen": -2.6006178855895996,
      "logits/rejected": -2.531825304031372,
      "logps/chosen": -228.2000732421875,
      "logps/rejected": -233.7799530029297,
      "loss": 0.4636,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.18744561076164246,
      "rewards/margins": 0.8298454284667969,
      "rewards/rejected": -0.6423999071121216,
      "step": 690
    },
    {
      "epoch": 0.33597312215022795,
      "grad_norm": 44.0,
      "learning_rate": 4.199846569360558e-06,
      "logits/chosen": -2.6074843406677246,
      "logits/rejected": -2.5071351528167725,
      "logps/chosen": -234.10958862304688,
      "logps/rejected": -215.6813507080078,
      "loss": 0.5146,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.24338316917419434,
      "rewards/margins": 0.7707773447036743,
      "rewards/rejected": -0.52739417552948,
      "step": 700
    },
    {
      "epoch": 0.3407727381809455,
      "grad_norm": 47.0,
      "learning_rate": 4.168877664885104e-06,
      "logits/chosen": -2.610435962677002,
      "logits/rejected": -2.462646961212158,
      "logps/chosen": -227.91342163085938,
      "logps/rejected": -188.88912963867188,
      "loss": 0.4715,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.12846776843070984,
      "rewards/margins": 0.8934639096260071,
      "rewards/rejected": -0.7649961709976196,
      "step": 710
    },
    {
      "epoch": 0.34557235421166305,
      "grad_norm": 33.5,
      "learning_rate": 4.1374397581840035e-06,
      "logits/chosen": -2.6382699012756348,
      "logits/rejected": -2.5071628093719482,
      "logps/chosen": -226.78402709960938,
      "logps/rejected": -190.30447387695312,
      "loss": 0.4784,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.12799586355686188,
      "rewards/margins": 0.7761613130569458,
      "rewards/rejected": -0.6481654047966003,
      "step": 720
    },
    {
      "epoch": 0.3503719702423806,
      "grad_norm": 55.75,
      "learning_rate": 4.105541684205752e-06,
      "logits/chosen": -2.590768337249756,
      "logits/rejected": -2.486171007156372,
      "logps/chosen": -217.5499725341797,
      "logps/rejected": -202.66116333007812,
      "loss": 0.497,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.11741193383932114,
      "rewards/margins": 0.8474688529968262,
      "rewards/rejected": -0.7300569415092468,
      "step": 730
    },
    {
      "epoch": 0.35517158627309814,
      "grad_norm": 39.75,
      "learning_rate": 4.073192407218972e-06,
      "logits/chosen": -2.633659839630127,
      "logits/rejected": -2.501005172729492,
      "logps/chosen": -239.923095703125,
      "logps/rejected": -199.56039428710938,
      "loss": 0.4329,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.1744803935289383,
      "rewards/margins": 1.0109570026397705,
      "rewards/rejected": -0.8364765048027039,
      "step": 740
    },
    {
      "epoch": 0.3599712023038157,
      "grad_norm": 47.25,
      "learning_rate": 4.040401018293204e-06,
      "logits/chosen": -2.5651516914367676,
      "logits/rejected": -2.5003182888031006,
      "logps/chosen": -222.2145538330078,
      "logps/rejected": -236.4008331298828,
      "loss": 0.4993,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06561844795942307,
      "rewards/margins": 0.7554530501365662,
      "rewards/rejected": -0.821071445941925,
      "step": 750
    },
    {
      "epoch": 0.36477081833453323,
      "grad_norm": 52.5,
      "learning_rate": 4.007176732744054e-06,
      "logits/chosen": -2.5720465183258057,
      "logits/rejected": -2.5482380390167236,
      "logps/chosen": -236.49081420898438,
      "logps/rejected": -238.8624725341797,
      "loss": 0.4684,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.019028931856155396,
      "rewards/margins": 0.9050389528274536,
      "rewards/rejected": -0.9240678548812866,
      "step": 760
    },
    {
      "epoch": 0.3695704343652508,
      "grad_norm": 41.5,
      "learning_rate": 3.9735288875434254e-06,
      "logits/chosen": -2.646091938018799,
      "logits/rejected": -2.4673545360565186,
      "logps/chosen": -244.1385040283203,
      "logps/rejected": -195.28207397460938,
      "loss": 0.4731,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.11497173458337784,
      "rewards/margins": 0.8432220220565796,
      "rewards/rejected": -0.9581937789916992,
      "step": 770
    },
    {
      "epoch": 0.3743700503959683,
      "grad_norm": 55.75,
      "learning_rate": 3.939466938695565e-06,
      "logits/chosen": -2.5559213161468506,
      "logits/rejected": -2.466019630432129,
      "logps/chosen": -253.15365600585938,
      "logps/rejected": -228.51220703125,
      "loss": 0.5308,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.10333450883626938,
      "rewards/margins": 0.7484658360481262,
      "rewards/rejected": -0.6451312303543091,
      "step": 780
    },
    {
      "epoch": 0.37916966642668587,
      "grad_norm": 36.25,
      "learning_rate": 3.905000458579657e-06,
      "logits/chosen": -2.570517063140869,
      "logits/rejected": -2.502270460128784,
      "logps/chosen": -209.8747100830078,
      "logps/rejected": -230.72579956054688,
      "loss": 0.5247,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.09132517874240875,
      "rewards/margins": 0.6891080141067505,
      "rewards/rejected": -0.5977829098701477,
      "step": 790
    },
    {
      "epoch": 0.3839692824574034,
      "grad_norm": 41.25,
      "learning_rate": 3.87013913325971e-06,
      "logits/chosen": -2.5875115394592285,
      "logits/rejected": -2.4476943016052246,
      "logps/chosen": -259.3215637207031,
      "logps/rejected": -214.0452423095703,
      "loss": 0.5223,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.05909506604075432,
      "rewards/margins": 0.7436100840568542,
      "rewards/rejected": -0.6845150589942932,
      "step": 800
    },
    {
      "epoch": 0.3839692824574034,
      "eval_logits/chosen": -2.6117820739746094,
      "eval_logits/rejected": -2.496819257736206,
      "eval_logps/chosen": -231.5717010498047,
      "eval_logps/rejected": -212.9447784423828,
      "eval_loss": 0.46309149265289307,
      "eval_rewards/accuracies": 0.7879999876022339,
      "eval_rewards/chosen": 0.15612684190273285,
      "eval_rewards/margins": 0.8975253701210022,
      "eval_rewards/rejected": -0.7413985729217529,
      "eval_runtime": 31.9562,
      "eval_samples_per_second": 31.293,
      "eval_steps_per_second": 7.823,
      "step": 800
    },
    {
      "epoch": 0.38876889848812096,
      "grad_norm": 41.75,
      "learning_rate": 3.8348927597624965e-06,
      "logits/chosen": -2.635223388671875,
      "logits/rejected": -2.5298709869384766,
      "logps/chosen": -231.2666473388672,
      "logps/rejected": -217.6613311767578,
      "loss": 0.4714,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.08412560075521469,
      "rewards/margins": 0.8177760243415833,
      "rewards/rejected": -0.7336505055427551,
      "step": 810
    },
    {
      "epoch": 0.3935685145188385,
      "grad_norm": 40.75,
      "learning_rate": 3.7992712433243117e-06,
      "logits/chosen": -2.6278603076934814,
      "logits/rejected": -2.470078945159912,
      "logps/chosen": -235.6140899658203,
      "logps/rejected": -188.5288848876953,
      "loss": 0.4796,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1967240571975708,
      "rewards/margins": 0.8465791940689087,
      "rewards/rejected": -1.0433032512664795,
      "step": 820
    },
    {
      "epoch": 0.39836813054955605,
      "grad_norm": 48.0,
      "learning_rate": 3.7632845946073136e-06,
      "logits/chosen": -2.6680498123168945,
      "logits/rejected": -2.513561964035034,
      "logps/chosen": -250.830810546875,
      "logps/rejected": -192.11917114257812,
      "loss": 0.4341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.35141512751579285,
      "rewards/margins": 0.932411789894104,
      "rewards/rejected": -1.2838269472122192,
      "step": 830
    },
    {
      "epoch": 0.4031677465802736,
      "grad_norm": 40.0,
      "learning_rate": 3.7269429268862513e-06,
      "logits/chosen": -2.630174398422241,
      "logits/rejected": -2.5625929832458496,
      "logps/chosen": -213.17184448242188,
      "logps/rejected": -208.76736450195312,
      "loss": 0.4875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3925122618675232,
      "rewards/margins": 0.8376442193984985,
      "rewards/rejected": -1.230156421661377,
      "step": 840
    },
    {
      "epoch": 0.40796736261099115,
      "grad_norm": 35.25,
      "learning_rate": 3.690256453206334e-06,
      "logits/chosen": -2.6113486289978027,
      "logits/rejected": -2.562234401702881,
      "logps/chosen": -217.27377319335938,
      "logps/rejected": -210.09207153320312,
      "loss": 0.4761,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.20575007796287537,
      "rewards/margins": 0.932608425617218,
      "rewards/rejected": -1.1383583545684814,
      "step": 850
    },
    {
      "epoch": 0.41276697864170864,
      "grad_norm": 54.5,
      "learning_rate": 3.6532354835130844e-06,
      "logits/chosen": -2.6446421146392822,
      "logits/rejected": -2.535243511199951,
      "logps/chosen": -249.38037109375,
      "logps/rejected": -225.61679077148438,
      "loss": 0.4788,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0025300472043454647,
      "rewards/margins": 0.9329078793525696,
      "rewards/rejected": -0.9354379773139954,
      "step": 860
    },
    {
      "epoch": 0.4175665946724262,
      "grad_norm": 43.5,
      "learning_rate": 3.6158904217549446e-06,
      "logits/chosen": -2.640524387359619,
      "logits/rejected": -2.5773513317108154,
      "logps/chosen": -225.6608123779297,
      "logps/rejected": -203.6904296875,
      "loss": 0.5129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13905248045921326,
      "rewards/margins": 0.7723478078842163,
      "rewards/rejected": -0.911400318145752,
      "step": 870
    },
    {
      "epoch": 0.42236621070314373,
      "grad_norm": 56.5,
      "learning_rate": 3.5782317629594708e-06,
      "logits/chosen": -2.6161234378814697,
      "logits/rejected": -2.5211544036865234,
      "logps/chosen": -240.87353515625,
      "logps/rejected": -220.1843719482422,
      "loss": 0.5035,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.07784248143434525,
      "rewards/margins": 0.8290297389030457,
      "rewards/rejected": -0.7511872053146362,
      "step": 880
    },
    {
      "epoch": 0.4271658267338613,
      "grad_norm": 41.5,
      "learning_rate": 3.5402700902839317e-06,
      "logits/chosen": -2.5109052658081055,
      "logits/rejected": -2.469113826751709,
      "logps/chosen": -206.3893280029297,
      "logps/rejected": -216.2916717529297,
      "loss": 0.481,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0023569464683532715,
      "rewards/margins": 0.8143336176872253,
      "rewards/rejected": -0.8166904449462891,
      "step": 890
    },
    {
      "epoch": 0.4319654427645788,
      "grad_norm": 38.0,
      "learning_rate": 3.5020160720411408e-06,
      "logits/chosen": -2.620961904525757,
      "logits/rejected": -2.4989562034606934,
      "logps/chosen": -232.7259979248047,
      "logps/rejected": -224.18222045898438,
      "loss": 0.457,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.1927359253168106,
      "rewards/margins": 0.9317037463188171,
      "rewards/rejected": -0.738967776298523,
      "step": 900
    },
    {
      "epoch": 0.43676505879529637,
      "grad_norm": 60.5,
      "learning_rate": 3.4634804587013505e-06,
      "logits/chosen": -2.5633127689361572,
      "logits/rejected": -2.51041841506958,
      "logps/chosen": -216.7091827392578,
      "logps/rejected": -217.47030639648438,
      "loss": 0.5084,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10815312713384628,
      "rewards/margins": 0.7729175090789795,
      "rewards/rejected": -0.8810704946517944,
      "step": 910
    },
    {
      "epoch": 0.4415646748260139,
      "grad_norm": 44.0,
      "learning_rate": 3.424674079871073e-06,
      "logits/chosen": -2.5757408142089844,
      "logits/rejected": -2.4900078773498535,
      "logps/chosen": -217.65444946289062,
      "logps/rejected": -204.16091918945312,
      "loss": 0.5214,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12332279980182648,
      "rewards/margins": 0.7060663104057312,
      "rewards/rejected": -0.8293890953063965,
      "step": 920
    },
    {
      "epoch": 0.44636429085673146,
      "grad_norm": 36.25,
      "learning_rate": 3.3856078412496424e-06,
      "logits/chosen": -2.6325020790100098,
      "logits/rejected": -2.5013327598571777,
      "logps/chosen": -240.5934295654297,
      "logps/rejected": -201.51132202148438,
      "loss": 0.4388,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12551036477088928,
      "rewards/margins": 0.971343994140625,
      "rewards/rejected": -1.0968544483184814,
      "step": 930
    },
    {
      "epoch": 0.451163906887449,
      "grad_norm": 50.5,
      "learning_rate": 3.346292721564407e-06,
      "logits/chosen": -2.654001474380493,
      "logits/rejected": -2.5543465614318848,
      "logps/chosen": -265.34912109375,
      "logps/rejected": -228.43905639648438,
      "loss": 0.4906,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.23702020943164825,
      "rewards/margins": 0.903741717338562,
      "rewards/rejected": -1.1407619714736938,
      "step": 940
    },
    {
      "epoch": 0.45596352291816655,
      "grad_norm": 43.75,
      "learning_rate": 3.306739769485394e-06,
      "logits/chosen": -2.593740940093994,
      "logits/rejected": -2.4713730812072754,
      "logps/chosen": -235.24942016601562,
      "logps/rejected": -202.47262573242188,
      "loss": 0.446,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.12331932783126831,
      "rewards/margins": 0.9522517919540405,
      "rewards/rejected": -1.075571060180664,
      "step": 950
    },
    {
      "epoch": 0.4607631389488841,
      "grad_norm": 49.75,
      "learning_rate": 3.266960100520316e-06,
      "logits/chosen": -2.6186511516571045,
      "logits/rejected": -2.5293595790863037,
      "logps/chosen": -207.3675537109375,
      "logps/rejected": -198.95132446289062,
      "loss": 0.445,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.22513869404792786,
      "rewards/margins": 1.0883581638336182,
      "rewards/rejected": -1.3134969472885132,
      "step": 960
    },
    {
      "epoch": 0.46556275497960165,
      "grad_norm": 35.75,
      "learning_rate": 3.2269648938907977e-06,
      "logits/chosen": -2.5796661376953125,
      "logits/rejected": -2.469088315963745,
      "logps/chosen": -219.3197479248047,
      "logps/rejected": -195.90982055664062,
      "loss": 0.4861,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.38874492049217224,
      "rewards/margins": 0.9575654864311218,
      "rewards/rejected": -1.3463106155395508,
      "step": 970
    },
    {
      "epoch": 0.4703623710103192,
      "grad_norm": 35.75,
      "learning_rate": 3.186765389390696e-06,
      "logits/chosen": -2.6649694442749023,
      "logits/rejected": -2.528609037399292,
      "logps/chosen": -255.8763885498047,
      "logps/rejected": -208.0857391357422,
      "loss": 0.4569,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.3541041314601898,
      "rewards/margins": 0.9702304005622864,
      "rewards/rejected": -1.3243346214294434,
      "step": 980
    },
    {
      "epoch": 0.47516198704103674,
      "grad_norm": 53.0,
      "learning_rate": 3.146372884227393e-06,
      "logits/chosen": -2.630551815032959,
      "logits/rejected": -2.5320029258728027,
      "logps/chosen": -253.5113983154297,
      "logps/rejected": -226.9222412109375,
      "loss": 0.5222,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3444979786872864,
      "rewards/margins": 0.7771540880203247,
      "rewards/rejected": -1.1216518878936768,
      "step": 990
    },
    {
      "epoch": 0.4799616030717543,
      "grad_norm": 35.0,
      "learning_rate": 3.1057987298469693e-06,
      "logits/chosen": -2.55789852142334,
      "logits/rejected": -2.4452617168426514,
      "logps/chosen": -217.3821258544922,
      "logps/rejected": -195.25726318359375,
      "loss": 0.4335,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.20461265742778778,
      "rewards/margins": 1.0646028518676758,
      "rewards/rejected": -1.26921546459198,
      "step": 1000
    },
    {
      "epoch": 0.4799616030717543,
      "eval_logits/chosen": -2.6097192764282227,
      "eval_logits/rejected": -2.497605085372925,
      "eval_logps/chosen": -234.7398681640625,
      "eval_logps/rejected": -217.08157348632812,
      "eval_loss": 0.45548132061958313,
      "eval_rewards/accuracies": 0.7950000166893005,
      "eval_rewards/chosen": -0.16069155931472778,
      "eval_rewards/margins": 0.9943889379501343,
      "eval_rewards/rejected": -1.1550804376602173,
      "eval_runtime": 32.012,
      "eval_samples_per_second": 31.238,
      "eval_steps_per_second": 7.81,
      "step": 1000
    },
    {
      "epoch": 0.48476121910247183,
      "grad_norm": 35.75,
      "learning_rate": 3.06505432874411e-06,
      "logits/chosen": -2.5709657669067383,
      "logits/rejected": -2.432443141937256,
      "logps/chosen": -251.8423614501953,
      "logps/rejected": -219.88711547851562,
      "loss": 0.469,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.1605864316225052,
      "rewards/margins": 0.9071685671806335,
      "rewards/rejected": -1.0677549839019775,
      "step": 1010
    },
    {
      "epoch": 0.4895608351331893,
      "grad_norm": 44.0,
      "learning_rate": 3.024151131257688e-06,
      "logits/chosen": -2.601938009262085,
      "logits/rejected": -2.4874444007873535,
      "logps/chosen": -249.327880859375,
      "logps/rejected": -204.016357421875,
      "loss": 0.4554,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.34456175565719604,
      "rewards/margins": 0.9197514653205872,
      "rewards/rejected": -1.2643131017684937,
      "step": 1020
    },
    {
      "epoch": 0.49436045116390687,
      "grad_norm": 52.25,
      "learning_rate": 2.983100632352889e-06,
      "logits/chosen": -2.67118501663208,
      "logits/rejected": -2.495823860168457,
      "logps/chosen": -256.9388732910156,
      "logps/rejected": -210.40402221679688,
      "loss": 0.4574,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.21712104976177216,
      "rewards/margins": 1.0202686786651611,
      "rewards/rejected": -1.2373896837234497,
      "step": 1030
    },
    {
      "epoch": 0.4991600671946244,
      "grad_norm": 48.0,
      "learning_rate": 2.9419143683907987e-06,
      "logits/chosen": -2.5967347621917725,
      "logits/rejected": -2.5237019062042236,
      "logps/chosen": -223.098876953125,
      "logps/rejected": -219.5789031982422,
      "loss": 0.4829,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4448261260986328,
      "rewards/margins": 0.9350289106369019,
      "rewards/rejected": -1.3798550367355347,
      "step": 1040
    },
    {
      "epoch": 0.503959683225342,
      "grad_norm": 44.25,
      "learning_rate": 2.9006039138863572e-06,
      "logits/chosen": -2.5805346965789795,
      "logits/rejected": -2.4660849571228027,
      "logps/chosen": -247.05136108398438,
      "logps/rejected": -224.231201171875,
      "loss": 0.415,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.20926351845264435,
      "rewards/margins": 1.0519158840179443,
      "rewards/rejected": -1.2611795663833618,
      "step": 1050
    },
    {
      "epoch": 0.5087592992560596,
      "grad_norm": 36.0,
      "learning_rate": 2.8591808782555883e-06,
      "logits/chosen": -2.58505916595459,
      "logits/rejected": -2.5212624073028564,
      "logps/chosen": -233.4881134033203,
      "logps/rejected": -227.78945922851562,
      "loss": 0.4266,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.09728523343801498,
      "rewards/margins": 1.1170459985733032,
      "rewards/rejected": -1.2143312692642212,
      "step": 1060
    },
    {
      "epoch": 0.5135589152867771,
      "grad_norm": 52.75,
      "learning_rate": 2.817656902553024e-06,
      "logits/chosen": -2.6064798831939697,
      "logits/rejected": -2.511019468307495,
      "logps/chosen": -234.0939483642578,
      "logps/rejected": -214.7052764892578,
      "loss": 0.4975,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.10685852915048599,
      "rewards/margins": 0.8924004435539246,
      "rewards/rejected": -0.9992589950561523,
      "step": 1070
    },
    {
      "epoch": 0.5183585313174947,
      "grad_norm": 52.75,
      "learning_rate": 2.7760436562002354e-06,
      "logits/chosen": -2.5962636470794678,
      "logits/rejected": -2.428013324737549,
      "logps/chosen": -267.91326904296875,
      "logps/rejected": -191.6916046142578,
      "loss": 0.5203,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1610618531703949,
      "rewards/margins": 0.8274253606796265,
      "rewards/rejected": -0.9884872436523438,
      "step": 1080
    },
    {
      "epoch": 0.5231581473482121,
      "grad_norm": 39.0,
      "learning_rate": 2.7343528337063924e-06,
      "logits/chosen": -2.6854658126831055,
      "logits/rejected": -2.561249256134033,
      "logps/chosen": -250.7506866455078,
      "logps/rejected": -225.0154266357422,
      "loss": 0.4171,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.04954652860760689,
      "rewards/margins": 1.022822618484497,
      "rewards/rejected": -1.07236909866333,
      "step": 1090
    },
    {
      "epoch": 0.5279577633789296,
      "grad_norm": 28.75,
      "learning_rate": 2.692596151381774e-06,
      "logits/chosen": -2.5916976928710938,
      "logits/rejected": -2.5371453762054443,
      "logps/chosen": -202.2080078125,
      "logps/rejected": -212.92587280273438,
      "loss": 0.4372,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.23717375099658966,
      "rewards/margins": 0.9503253698348999,
      "rewards/rejected": -1.1874991655349731,
      "step": 1100
    },
    {
      "epoch": 0.5327573794096472,
      "grad_norm": 43.5,
      "learning_rate": 2.650785344045149e-06,
      "logits/chosen": -2.6039624214172363,
      "logits/rejected": -2.515532970428467,
      "logps/chosen": -228.81729125976562,
      "logps/rejected": -217.0961151123047,
      "loss": 0.4695,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.10938185453414917,
      "rewards/margins": 1.068205714225769,
      "rewards/rejected": -1.177587628364563,
      "step": 1110
    },
    {
      "epoch": 0.5375569954403647,
      "grad_norm": 33.75,
      "learning_rate": 2.6089321617259583e-06,
      "logits/chosen": -2.6028213500976562,
      "logits/rejected": -2.479320764541626,
      "logps/chosen": -232.82186889648438,
      "logps/rejected": -216.1719207763672,
      "loss": 0.4028,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.030288416892290115,
      "rewards/margins": 1.2107077836990356,
      "rewards/rejected": -1.2409961223602295,
      "step": 1120
    },
    {
      "epoch": 0.5423566114710823,
      "grad_norm": 58.75,
      "learning_rate": 2.567048366362225e-06,
      "logits/chosen": -2.5917420387268066,
      "logits/rejected": -2.491093873977661,
      "logps/chosen": -242.26766967773438,
      "logps/rejected": -212.3538055419922,
      "loss": 0.4948,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2735952138900757,
      "rewards/margins": 0.8888144493103027,
      "rewards/rejected": -1.1624095439910889,
      "step": 1130
    },
    {
      "epoch": 0.5471562275017998,
      "grad_norm": 33.5,
      "learning_rate": 2.525145728495106e-06,
      "logits/chosen": -2.608853816986084,
      "logits/rejected": -2.5186550617218018,
      "logps/chosen": -230.33004760742188,
      "logps/rejected": -204.28524780273438,
      "loss": 0.4649,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2685418426990509,
      "rewards/margins": 1.0452083349227905,
      "rewards/rejected": -1.313750147819519,
      "step": 1140
    },
    {
      "epoch": 0.5519558435325174,
      "grad_norm": 38.0,
      "learning_rate": 2.4832360239610416e-06,
      "logits/chosen": -2.6019065380096436,
      "logits/rejected": -2.4967987537384033,
      "logps/chosen": -231.5498809814453,
      "logps/rejected": -213.8773651123047,
      "loss": 0.4629,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.29873836040496826,
      "rewards/margins": 0.9136406183242798,
      "rewards/rejected": -1.212378978729248,
      "step": 1150
    },
    {
      "epoch": 0.5567554595632349,
      "grad_norm": 49.25,
      "learning_rate": 2.441331030582407e-06,
      "logits/chosen": -2.615382671356201,
      "logits/rejected": -2.5341572761535645,
      "logps/chosen": -226.35079956054688,
      "logps/rejected": -216.3585662841797,
      "loss": 0.4676,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.21219320595264435,
      "rewards/margins": 0.8832317590713501,
      "rewards/rejected": -1.0954248905181885,
      "step": 1160
    },
    {
      "epoch": 0.5615550755939525,
      "grad_norm": 52.0,
      "learning_rate": 2.3994425248576102e-06,
      "logits/chosen": -2.6414036750793457,
      "logits/rejected": -2.504991054534912,
      "logps/chosen": -248.39572143554688,
      "logps/rejected": -202.54635620117188,
      "loss": 0.5142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.26081639528274536,
      "rewards/margins": 0.7797340154647827,
      "rewards/rejected": -1.0405504703521729,
      "step": 1170
    },
    {
      "epoch": 0.56635469162467,
      "grad_norm": 56.0,
      "learning_rate": 2.357582278651553e-06,
      "logits/chosen": -2.580275535583496,
      "logits/rejected": -2.4745254516601562,
      "logps/chosen": -234.48818969726562,
      "logps/rejected": -222.6685028076172,
      "loss": 0.4538,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.2316078245639801,
      "rewards/margins": 1.032219409942627,
      "rewards/rejected": -1.2638272047042847,
      "step": 1180
    },
    {
      "epoch": 0.5711543076553875,
      "grad_norm": 44.5,
      "learning_rate": 2.315762055887411e-06,
      "logits/chosen": -2.6241419315338135,
      "logits/rejected": -2.4953231811523438,
      "logps/chosen": -246.7222442626953,
      "logps/rejected": -206.4970703125,
      "loss": 0.5141,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.31947702169418335,
      "rewards/margins": 0.9516233205795288,
      "rewards/rejected": -1.271100401878357,
      "step": 1190
    },
    {
      "epoch": 0.5759539236861051,
      "grad_norm": 37.5,
      "learning_rate": 2.273993609240629e-06,
      "logits/chosen": -2.5731539726257324,
      "logits/rejected": -2.483686685562134,
      "logps/chosen": -229.90762329101562,
      "logps/rejected": -225.7178955078125,
      "loss": 0.5214,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3317897915840149,
      "rewards/margins": 0.849661648273468,
      "rewards/rejected": -1.1814515590667725,
      "step": 1200
    },
    {
      "epoch": 0.5759539236861051,
      "eval_logits/chosen": -2.6119532585144043,
      "eval_logits/rejected": -2.5000758171081543,
      "eval_logps/chosen": -234.451904296875,
      "eval_logps/rejected": -217.00180053710938,
      "eval_loss": 0.45114314556121826,
      "eval_rewards/accuracies": 0.7990000247955322,
      "eval_rewards/chosen": -0.13189174234867096,
      "eval_rewards/margins": 1.0152093172073364,
      "eval_rewards/rejected": -1.1471011638641357,
      "eval_runtime": 31.9635,
      "eval_samples_per_second": 31.286,
      "eval_steps_per_second": 7.821,
      "step": 1200
    },
    {
      "epoch": 0.5807535397168226,
      "grad_norm": 59.0,
      "learning_rate": 2.2322886768360874e-06,
      "logits/chosen": -2.513803005218506,
      "logits/rejected": -2.4552152156829834,
      "logps/chosen": -239.845703125,
      "logps/rejected": -215.5902099609375,
      "loss": 0.4561,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0830584168434143,
      "rewards/margins": 1.117357850074768,
      "rewards/rejected": -1.2004162073135376,
      "step": 1210
    },
    {
      "epoch": 0.5855531557475402,
      "grad_norm": 49.75,
      "learning_rate": 2.190658978949352e-06,
      "logits/chosen": -2.607837438583374,
      "logits/rejected": -2.4749207496643066,
      "logps/chosen": -220.0395050048828,
      "logps/rejected": -194.2960968017578,
      "loss": 0.4986,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1924716681241989,
      "rewards/margins": 0.8406950235366821,
      "rewards/rejected": -1.0331666469573975,
      "step": 1220
    },
    {
      "epoch": 0.5903527717782577,
      "grad_norm": 57.25,
      "learning_rate": 2.149116214712943e-06,
      "logits/chosen": -2.6154885292053223,
      "logits/rejected": -2.509730577468872,
      "logps/chosen": -234.84402465820312,
      "logps/rejected": -219.07339477539062,
      "loss": 0.4841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1697658747434616,
      "rewards/margins": 0.8840200304985046,
      "rewards/rejected": -1.053786039352417,
      "step": 1230
    },
    {
      "epoch": 0.5951523878089753,
      "grad_norm": 55.0,
      "learning_rate": 2.107672058828544e-06,
      "logits/chosen": -2.6242995262145996,
      "logits/rejected": -2.51531720161438,
      "logps/chosen": -228.4006805419922,
      "logps/rejected": -202.6013641357422,
      "loss": 0.474,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.13985280692577362,
      "rewards/margins": 0.875139594078064,
      "rewards/rejected": -1.0149924755096436,
      "step": 1240
    },
    {
      "epoch": 0.5999520038396928,
      "grad_norm": 60.5,
      "learning_rate": 2.066338158286083e-06,
      "logits/chosen": -2.6175296306610107,
      "logits/rejected": -2.549783229827881,
      "logps/chosen": -227.7768096923828,
      "logps/rejected": -222.3975830078125,
      "loss": 0.3924,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06711763888597488,
      "rewards/margins": 1.2078065872192383,
      "rewards/rejected": -1.2749242782592773,
      "step": 1250
    },
    {
      "epoch": 0.6047516198704104,
      "grad_norm": 43.5,
      "learning_rate": 2.025126129090588e-06,
      "logits/chosen": -2.673826217651367,
      "logits/rejected": -2.53361177444458,
      "logps/chosen": -222.8029327392578,
      "logps/rejected": -191.4785919189453,
      "loss": 0.4424,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0933801457285881,
      "rewards/margins": 1.0600812435150146,
      "rewards/rejected": -1.1534613370895386,
      "step": 1260
    },
    {
      "epoch": 0.6095512359011279,
      "grad_norm": 49.75,
      "learning_rate": 1.9840475529977655e-06,
      "logits/chosen": -2.6106059551239014,
      "logits/rejected": -2.5068392753601074,
      "logps/chosen": -230.1819610595703,
      "logps/rejected": -208.9829559326172,
      "loss": 0.4371,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.1759682148694992,
      "rewards/margins": 1.1224491596221924,
      "rewards/rejected": -1.298417329788208,
      "step": 1270
    },
    {
      "epoch": 0.6143508519318455,
      "grad_norm": 40.0,
      "learning_rate": 1.9431139742591897e-06,
      "logits/chosen": -2.594632625579834,
      "logits/rejected": -2.489180088043213,
      "logps/chosen": -209.05859375,
      "logps/rejected": -198.956298828125,
      "loss": 0.4101,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.08843652904033661,
      "rewards/margins": 1.0289084911346436,
      "rewards/rejected": -1.117344856262207,
      "step": 1280
    },
    {
      "epoch": 0.619150467962563,
      "grad_norm": 49.0,
      "learning_rate": 1.9023368963780458e-06,
      "logits/chosen": -2.6134049892425537,
      "logits/rejected": -2.5106282234191895,
      "logps/chosen": -234.5142059326172,
      "logps/rejected": -207.1331329345703,
      "loss": 0.4701,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.14769946038722992,
      "rewards/margins": 0.9224032163619995,
      "rewards/rejected": -1.0701026916503906,
      "step": 1290
    },
    {
      "epoch": 0.6239500839932806,
      "grad_norm": 42.0,
      "learning_rate": 1.861727778876314e-06,
      "logits/chosen": -2.5988595485687256,
      "logits/rejected": -2.4945011138916016,
      "logps/chosen": -209.90835571289062,
      "logps/rejected": -185.820068359375,
      "loss": 0.4437,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.21710054576396942,
      "rewards/margins": 1.0192363262176514,
      "rewards/rejected": -1.2363369464874268,
      "step": 1300
    },
    {
      "epoch": 0.6287497000239981,
      "grad_norm": 37.0,
      "learning_rate": 1.8212980340743152e-06,
      "logits/chosen": -2.6286704540252686,
      "logits/rejected": -2.5620837211608887,
      "logps/chosen": -225.5823974609375,
      "logps/rejected": -215.69375610351562,
      "loss": 0.4882,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.22902747988700867,
      "rewards/margins": 0.9663082957267761,
      "rewards/rejected": -1.195335865020752,
      "step": 1310
    },
    {
      "epoch": 0.6335493160547156,
      "grad_norm": 39.75,
      "learning_rate": 1.7810590238835279e-06,
      "logits/chosen": -2.5562384128570557,
      "logits/rejected": -2.534426212310791,
      "logps/chosen": -226.40902709960938,
      "logps/rejected": -251.94418334960938,
      "loss": 0.4495,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.19038750231266022,
      "rewards/margins": 1.0307300090789795,
      "rewards/rejected": -1.2211174964904785,
      "step": 1320
    },
    {
      "epoch": 0.6383489320854332,
      "grad_norm": 43.0,
      "learning_rate": 1.7410220566135605e-06,
      "logits/chosen": -2.6242737770080566,
      "logits/rejected": -2.5161290168762207,
      "logps/chosen": -228.24398803710938,
      "logps/rejected": -207.40316772460938,
      "loss": 0.4474,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06271727383136749,
      "rewards/margins": 1.0257116556167603,
      "rewards/rejected": -1.0884288549423218,
      "step": 1330
    },
    {
      "epoch": 0.6431485481161507,
      "grad_norm": 58.5,
      "learning_rate": 1.7011983837942023e-06,
      "logits/chosen": -2.6005454063415527,
      "logits/rejected": -2.4911046028137207,
      "logps/chosen": -235.7122344970703,
      "logps/rejected": -217.5489044189453,
      "loss": 0.4614,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.12799616158008575,
      "rewards/margins": 0.9990032315254211,
      "rewards/rejected": -1.1269992589950562,
      "step": 1340
    },
    {
      "epoch": 0.6479481641468683,
      "grad_norm": 45.25,
      "learning_rate": 1.661599197013416e-06,
      "logits/chosen": -2.633547306060791,
      "logits/rejected": -2.5312139987945557,
      "logps/chosen": -223.9100341796875,
      "logps/rejected": -203.5366973876953,
      "loss": 0.4765,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2294052541255951,
      "rewards/margins": 0.95598304271698,
      "rewards/rejected": -1.185388207435608,
      "step": 1350
    },
    {
      "epoch": 0.6527477801775858,
      "grad_norm": 46.75,
      "learning_rate": 1.6222356247721831e-06,
      "logits/chosen": -2.5966320037841797,
      "logits/rejected": -2.4941086769104004,
      "logps/chosen": -234.2575225830078,
      "logps/rejected": -221.3917999267578,
      "loss": 0.4476,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.12550556659698486,
      "rewards/margins": 1.0439780950546265,
      "rewards/rejected": -1.1694835424423218,
      "step": 1360
    },
    {
      "epoch": 0.6575473962083034,
      "grad_norm": 46.0,
      "learning_rate": 1.5831187293570826e-06,
      "logits/chosen": -2.616199016571045,
      "logits/rejected": -2.5082523822784424,
      "logps/chosen": -275.43511962890625,
      "logps/rejected": -229.6664276123047,
      "loss": 0.4631,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2143980711698532,
      "rewards/margins": 1.0123686790466309,
      "rewards/rejected": -1.22676682472229,
      "step": 1370
    },
    {
      "epoch": 0.6623470122390209,
      "grad_norm": 48.5,
      "learning_rate": 1.544259503731465e-06,
      "logits/chosen": -2.612053632736206,
      "logits/rejected": -2.4867258071899414,
      "logps/chosen": -240.70059204101562,
      "logps/rejected": -198.64918518066406,
      "loss": 0.3862,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.0024546384811401367,
      "rewards/margins": 1.1859334707260132,
      "rewards/rejected": -1.1883881092071533,
      "step": 1380
    },
    {
      "epoch": 0.6671466282697385,
      "grad_norm": 36.0,
      "learning_rate": 1.5056688684461235e-06,
      "logits/chosen": -2.6124138832092285,
      "logits/rejected": -2.491760730743408,
      "logps/chosen": -245.6887969970703,
      "logps/rejected": -216.34268188476562,
      "loss": 0.4459,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.3357298672199249,
      "rewards/margins": 1.0037829875946045,
      "rewards/rejected": -1.3395130634307861,
      "step": 1390
    },
    {
      "epoch": 0.6719462443004559,
      "grad_norm": 54.0,
      "learning_rate": 1.4673576685703027e-06,
      "logits/chosen": -2.6035733222961426,
      "logits/rejected": -2.521059513092041,
      "logps/chosen": -243.61123657226562,
      "logps/rejected": -220.312744140625,
      "loss": 0.4784,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.27499574422836304,
      "rewards/margins": 0.896142840385437,
      "rewards/rejected": -1.1711386442184448,
      "step": 1400
    },
    {
      "epoch": 0.6719462443004559,
      "eval_logits/chosen": -2.6097218990325928,
      "eval_logits/rejected": -2.4986746311187744,
      "eval_logps/chosen": -234.91114807128906,
      "eval_logps/rejected": -217.71800231933594,
      "eval_loss": 0.4486246407032013,
      "eval_rewards/accuracies": 0.800000011920929,
      "eval_rewards/chosen": -0.17781423032283783,
      "eval_rewards/margins": 1.0409064292907715,
      "eval_rewards/rejected": -1.218720555305481,
      "eval_runtime": 31.822,
      "eval_samples_per_second": 31.425,
      "eval_steps_per_second": 7.856,
      "step": 1400
    },
    {
      "epoch": 0.6767458603311735,
      "grad_norm": 28.0,
      "learning_rate": 1.4293366706439293e-06,
      "logits/chosen": -2.5871994495391846,
      "logits/rejected": -2.4666335582733154,
      "logps/chosen": -221.1563262939453,
      "logps/rejected": -207.7967529296875,
      "loss": 0.4466,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.30613771080970764,
      "rewards/margins": 1.0556987524032593,
      "rewards/rejected": -1.361836314201355,
      "step": 1410
    },
    {
      "epoch": 0.681545476361891,
      "grad_norm": 41.25,
      "learning_rate": 1.3916165596519015e-06,
      "logits/chosen": -2.6185154914855957,
      "logits/rejected": -2.452584981918335,
      "logps/chosen": -233.75131225585938,
      "logps/rejected": -198.76773071289062,
      "loss": 0.4315,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.31702926754951477,
      "rewards/margins": 0.9871221780776978,
      "rewards/rejected": -1.3041512966156006,
      "step": 1420
    },
    {
      "epoch": 0.6863450923926085,
      "grad_norm": 47.0,
      "learning_rate": 1.3542079360213089e-06,
      "logits/chosen": -2.6181461811065674,
      "logits/rejected": -2.469588041305542,
      "logps/chosen": -224.7504425048828,
      "logps/rejected": -193.1361541748047,
      "loss": 0.4592,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.17981493473052979,
      "rewards/margins": 0.9263655543327332,
      "rewards/rejected": -1.1061805486679077,
      "step": 1430
    },
    {
      "epoch": 0.6911447084233261,
      "grad_norm": 52.75,
      "learning_rate": 1.317121312642406e-06,
      "logits/chosen": -2.6066277027130127,
      "logits/rejected": -2.465261936187744,
      "logps/chosen": -233.60720825195312,
      "logps/rejected": -211.9008026123047,
      "loss": 0.4429,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.18064916133880615,
      "rewards/margins": 1.064951777458191,
      "rewards/rejected": -1.245600938796997,
      "step": 1440
    },
    {
      "epoch": 0.6959443244540436,
      "grad_norm": 45.0,
      "learning_rate": 1.2803671119141953e-06,
      "logits/chosen": -2.5287094116210938,
      "logits/rejected": -2.4450387954711914,
      "logps/chosen": -248.5905303955078,
      "logps/rejected": -240.5746307373047,
      "loss": 0.478,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2681611478328705,
      "rewards/margins": 1.0177576541900635,
      "rewards/rejected": -1.285918951034546,
      "step": 1450
    },
    {
      "epoch": 0.7007439404847612,
      "grad_norm": 34.5,
      "learning_rate": 1.2439556628154293e-06,
      "logits/chosen": -2.612083911895752,
      "logits/rejected": -2.450336456298828,
      "logps/chosen": -250.229248046875,
      "logps/rejected": -219.356689453125,
      "loss": 0.4872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24559959769248962,
      "rewards/margins": 0.9850479960441589,
      "rewards/rejected": -1.2306474447250366,
      "step": 1460
    },
    {
      "epoch": 0.7055435565154787,
      "grad_norm": 41.5,
      "learning_rate": 1.207897198001878e-06,
      "logits/chosen": -2.644726037979126,
      "logits/rejected": -2.5499677658081055,
      "logps/chosen": -232.20126342773438,
      "logps/rejected": -208.1292724609375,
      "loss": 0.4442,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.15714068710803986,
      "rewards/margins": 1.035380482673645,
      "rewards/rejected": -1.192521095275879,
      "step": 1470
    },
    {
      "epoch": 0.7103431725461963,
      "grad_norm": 42.5,
      "learning_rate": 1.1722018509306587e-06,
      "logits/chosen": -2.6035397052764893,
      "logits/rejected": -2.4596409797668457,
      "logps/chosen": -250.045166015625,
      "logps/rejected": -202.33425903320312,
      "loss": 0.4371,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.22928068041801453,
      "rewards/margins": 1.1220018863677979,
      "rewards/rejected": -1.3512827157974243,
      "step": 1480
    },
    {
      "epoch": 0.7151427885769138,
      "grad_norm": 38.5,
      "learning_rate": 1.1368796530124442e-06,
      "logits/chosen": -2.5670337677001953,
      "logits/rejected": -2.446946620941162,
      "logps/chosen": -252.01260375976562,
      "logps/rejected": -207.9329071044922,
      "loss": 0.4135,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.18759331107139587,
      "rewards/margins": 1.0254666805267334,
      "rewards/rejected": -1.213059902191162,
      "step": 1490
    },
    {
      "epoch": 0.7199424046076314,
      "grad_norm": 51.25,
      "learning_rate": 1.101940530792356e-06,
      "logits/chosen": -2.6158971786499023,
      "logits/rejected": -2.5017120838165283,
      "logps/chosen": -245.5004425048828,
      "logps/rejected": -214.7168731689453,
      "loss": 0.4562,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.01713145337998867,
      "rewards/margins": 0.9989277124404907,
      "rewards/rejected": -1.016059160232544,
      "step": 1500
    },
    {
      "epoch": 0.7247420206383489,
      "grad_norm": 30.5,
      "learning_rate": 1.0673943031603134e-06,
      "logits/chosen": -2.605909824371338,
      "logits/rejected": -2.5186052322387695,
      "logps/chosen": -225.20675659179688,
      "logps/rejected": -219.88949584960938,
      "loss": 0.4336,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.2961525321006775,
      "rewards/margins": 1.0499502420425415,
      "rewards/rejected": -1.3461029529571533,
      "step": 1510
    },
    {
      "epoch": 0.7295416366690665,
      "grad_norm": 42.75,
      "learning_rate": 1.0332506785916524e-06,
      "logits/chosen": -2.590073823928833,
      "logits/rejected": -2.493535041809082,
      "logps/chosen": -241.81918334960938,
      "logps/rejected": -221.04598999023438,
      "loss": 0.4794,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2541709542274475,
      "rewards/margins": 0.9314506649971008,
      "rewards/rejected": -1.1856216192245483,
      "step": 1520
    },
    {
      "epoch": 0.734341252699784,
      "grad_norm": 44.5,
      "learning_rate": 9.995192524187639e-07,
      "logits/chosen": -2.5296549797058105,
      "logits/rejected": -2.480565071105957,
      "logps/chosen": -224.52774047851562,
      "logps/rejected": -220.7323760986328,
      "loss": 0.506,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.37830549478530884,
      "rewards/margins": 0.9571624994277954,
      "rewards/rejected": -1.335468053817749,
      "step": 1530
    },
    {
      "epoch": 0.7391408687305016,
      "grad_norm": 48.0,
      "learning_rate": 9.662095041345318e-07,
      "logits/chosen": -2.569598436355591,
      "logits/rejected": -2.4485225677490234,
      "logps/chosen": -247.4419708251953,
      "logps/rejected": -228.1896209716797,
      "loss": 0.4693,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.18139569461345673,
      "rewards/margins": 1.0061061382293701,
      "rewards/rejected": -1.187502145767212,
      "step": 1540
    },
    {
      "epoch": 0.7439404847612191,
      "grad_norm": 46.75,
      "learning_rate": 9.333307947283258e-07,
      "logits/chosen": -2.628610610961914,
      "logits/rejected": -2.5239391326904297,
      "logps/chosen": -243.1181640625,
      "logps/rejected": -224.0470733642578,
      "loss": 0.448,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.21100196242332458,
      "rewards/margins": 0.9517234563827515,
      "rewards/rejected": -1.1627254486083984,
      "step": 1550
    },
    {
      "epoch": 0.7487401007919366,
      "grad_norm": 34.75,
      "learning_rate": 9.00892364055298e-07,
      "logits/chosen": -2.587705135345459,
      "logits/rejected": -2.5059056282043457,
      "logps/chosen": -214.00631713867188,
      "logps/rejected": -191.03036499023438,
      "loss": 0.4244,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.27500849962234497,
      "rewards/margins": 1.0308496952056885,
      "rewards/rejected": -1.3058582544326782,
      "step": 1560
    },
    {
      "epoch": 0.7535397168226542,
      "grad_norm": 47.5,
      "learning_rate": 8.689033282397166e-07,
      "logits/chosen": -2.6136879920959473,
      "logits/rejected": -2.5038022994995117,
      "logps/chosen": -231.7812957763672,
      "logps/rejected": -214.75051879882812,
      "loss": 0.4818,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.29626065492630005,
      "rewards/margins": 0.9530956149101257,
      "rewards/rejected": -1.2493562698364258,
      "step": 1570
    },
    {
      "epoch": 0.7583393328533717,
      "grad_norm": 30.125,
      "learning_rate": 8.373726771130769e-07,
      "logits/chosen": -2.607466220855713,
      "logits/rejected": -2.4886395931243896,
      "logps/chosen": -242.94442749023438,
      "logps/rejected": -211.0756072998047,
      "loss": 0.4417,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.22200937569141388,
      "rewards/margins": 1.0732426643371582,
      "rewards/rejected": -1.2952520847320557,
      "step": 1580
    },
    {
      "epoch": 0.7631389488840893,
      "grad_norm": 54.5,
      "learning_rate": 8.063092716877016e-07,
      "logits/chosen": -2.549015998840332,
      "logits/rejected": -2.454470157623291,
      "logps/chosen": -255.9425811767578,
      "logps/rejected": -221.3877410888672,
      "loss": 0.4876,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1736731380224228,
      "rewards/margins": 0.8451417088508606,
      "rewards/rejected": -1.0188149213790894,
      "step": 1590
    },
    {
      "epoch": 0.7679385649148068,
      "grad_norm": 37.75,
      "learning_rate": 7.757218416665446e-07,
      "logits/chosen": -2.6379103660583496,
      "logits/rejected": -2.488036632537842,
      "logps/chosen": -232.8388214111328,
      "logps/rejected": -202.93429565429688,
      "loss": 0.4223,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2727198600769043,
      "rewards/margins": 1.1647435426712036,
      "rewards/rejected": -1.4374632835388184,
      "step": 1600
    },
    {
      "epoch": 0.7679385649148068,
      "eval_logits/chosen": -2.609999656677246,
      "eval_logits/rejected": -2.498737096786499,
      "eval_logps/chosen": -234.40658569335938,
      "eval_logps/rejected": -217.20668029785156,
      "eval_loss": 0.44873249530792236,
      "eval_rewards/accuracies": 0.7990000247955322,
      "eval_rewards/chosen": -0.12736062705516815,
      "eval_rewards/margins": 1.040230393409729,
      "eval_rewards/rejected": -1.1675910949707031,
      "eval_runtime": 31.9547,
      "eval_samples_per_second": 31.294,
      "eval_steps_per_second": 7.824,
      "step": 1600
    },
    {
      "epoch": 0.7727381809455244,
      "grad_norm": 40.5,
      "learning_rate": 7.456189829898955e-07,
      "logits/chosen": -2.617159366607666,
      "logits/rejected": -2.4671359062194824,
      "logps/chosen": -239.3896484375,
      "logps/rejected": -200.057861328125,
      "loss": 0.4496,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.19172294437885284,
      "rewards/margins": 1.0450642108917236,
      "rewards/rejected": -1.23678719997406,
      "step": 1610
    },
    {
      "epoch": 0.7775377969762419,
      "grad_norm": 37.0,
      "learning_rate": 7.160091554196732e-07,
      "logits/chosen": -2.6596245765686035,
      "logits/rejected": -2.534268617630005,
      "logps/chosen": -236.6833953857422,
      "logps/rejected": -209.33248901367188,
      "loss": 0.4434,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.2100035846233368,
      "rewards/margins": 1.0765715837478638,
      "rewards/rejected": -1.2865750789642334,
      "step": 1620
    },
    {
      "epoch": 0.7823374130069595,
      "grad_norm": 48.5,
      "learning_rate": 6.869006801619941e-07,
      "logits/chosen": -2.57913875579834,
      "logits/rejected": -2.5170235633850098,
      "logps/chosen": -243.2036895751953,
      "logps/rejected": -240.40792846679688,
      "loss": 0.4967,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.19071198999881744,
      "rewards/margins": 1.0176457166671753,
      "rewards/rejected": -1.208357810974121,
      "step": 1630
    },
    {
      "epoch": 0.787137029037677,
      "grad_norm": 54.5,
      "learning_rate": 6.583017375286726e-07,
      "logits/chosen": -2.5934314727783203,
      "logits/rejected": -2.4739251136779785,
      "logps/chosen": -231.6185760498047,
      "logps/rejected": -208.304443359375,
      "loss": 0.4574,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.2090485543012619,
      "rewards/margins": 1.0793238878250122,
      "rewards/rejected": -1.2883723974227905,
      "step": 1640
    },
    {
      "epoch": 0.7919366450683946,
      "grad_norm": 41.75,
      "learning_rate": 6.30220364638324e-07,
      "logits/chosen": -2.6110548973083496,
      "logits/rejected": -2.519888401031494,
      "logps/chosen": -244.562255859375,
      "logps/rejected": -207.3623504638672,
      "loss": 0.4747,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.21555647253990173,
      "rewards/margins": 0.8874263763427734,
      "rewards/rejected": -1.1029828786849976,
      "step": 1650
    },
    {
      "epoch": 0.7967362610991121,
      "grad_norm": 45.25,
      "learning_rate": 6.02664453157703e-07,
      "logits/chosen": -2.6588504314422607,
      "logits/rejected": -2.5547537803649902,
      "logps/chosen": -235.1988983154297,
      "logps/rejected": -226.4233856201172,
      "loss": 0.4643,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2098228931427002,
      "rewards/margins": 0.9475749731063843,
      "rewards/rejected": -1.157397985458374,
      "step": 1660
    },
    {
      "epoch": 0.8015358771298297,
      "grad_norm": 38.75,
      "learning_rate": 5.756417470839195e-07,
      "logits/chosen": -2.6417441368103027,
      "logits/rejected": -2.549515724182129,
      "logps/chosen": -229.06411743164062,
      "logps/rejected": -208.6210174560547,
      "loss": 0.445,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.18316276371479034,
      "rewards/margins": 0.9890239834785461,
      "rewards/rejected": -1.1721866130828857,
      "step": 1670
    },
    {
      "epoch": 0.8063354931605472,
      "grad_norm": 46.0,
      "learning_rate": 5.491598405681559e-07,
      "logits/chosen": -2.673280954360962,
      "logits/rejected": -2.498465061187744,
      "logps/chosen": -247.2686767578125,
      "logps/rejected": -202.4930419921875,
      "loss": 0.4942,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.17769768834114075,
      "rewards/margins": 0.8722941279411316,
      "rewards/rejected": -1.0499918460845947,
      "step": 1680
    },
    {
      "epoch": 0.8111351091912647,
      "grad_norm": 35.25,
      "learning_rate": 5.232261757814924e-07,
      "logits/chosen": -2.571895122528076,
      "logits/rejected": -2.438910722732544,
      "logps/chosen": -241.5240936279297,
      "logps/rejected": -213.72073364257812,
      "loss": 0.4317,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.1457543671131134,
      "rewards/margins": 1.0501019954681396,
      "rewards/rejected": -1.1958563327789307,
      "step": 1690
    },
    {
      "epoch": 0.8159347252219823,
      "grad_norm": 50.0,
      "learning_rate": 4.978480408234465e-07,
      "logits/chosen": -2.5237784385681152,
      "logits/rejected": -2.4956631660461426,
      "logps/chosen": -216.0598602294922,
      "logps/rejected": -214.1880340576172,
      "loss": 0.5087,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20147447288036346,
      "rewards/margins": 0.8618594408035278,
      "rewards/rejected": -1.0633338689804077,
      "step": 1700
    },
    {
      "epoch": 0.8207343412526998,
      "grad_norm": 51.75,
      "learning_rate": 4.73032567673809e-07,
      "logits/chosen": -2.6058714389801025,
      "logits/rejected": -2.504986047744751,
      "logps/chosen": -230.72103881835938,
      "logps/rejected": -206.2929229736328,
      "loss": 0.4769,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.30534833669662476,
      "rewards/margins": 0.8801034092903137,
      "rewards/rejected": -1.1854515075683594,
      "step": 1710
    },
    {
      "epoch": 0.8255339572834173,
      "grad_norm": 46.75,
      "learning_rate": 4.487867301883528e-07,
      "logits/chosen": -2.5329365730285645,
      "logits/rejected": -2.43863844871521,
      "logps/chosen": -225.4794921875,
      "logps/rejected": -214.2443389892578,
      "loss": 0.5013,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2076941281557083,
      "rewards/margins": 0.8862035870552063,
      "rewards/rejected": -1.0938977003097534,
      "step": 1720
    },
    {
      "epoch": 0.8303335733141348,
      "grad_norm": 42.25,
      "learning_rate": 4.2511734213898093e-07,
      "logits/chosen": -2.64858341217041,
      "logits/rejected": -2.5094025135040283,
      "logps/chosen": -254.30532836914062,
      "logps/rejected": -218.06900024414062,
      "loss": 0.5122,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.28009095788002014,
      "rewards/margins": 0.8344430923461914,
      "rewards/rejected": -1.1145341396331787,
      "step": 1730
    },
    {
      "epoch": 0.8351331893448524,
      "grad_norm": 69.5,
      "learning_rate": 4.020310552988632e-07,
      "logits/chosen": -2.620943546295166,
      "logits/rejected": -2.456711530685425,
      "logps/chosen": -250.24374389648438,
      "logps/rejected": -217.15219116210938,
      "loss": 0.4241,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.21671004593372345,
      "rewards/margins": 1.0417428016662598,
      "rewards/rejected": -1.2584527730941772,
      "step": 1740
    },
    {
      "epoch": 0.8399328053755699,
      "grad_norm": 51.5,
      "learning_rate": 3.7953435757309756e-07,
      "logits/chosen": -2.628960132598877,
      "logits/rejected": -2.504699468612671,
      "logps/chosen": -253.9635467529297,
      "logps/rejected": -227.1519317626953,
      "loss": 0.5159,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.17710088193416595,
      "rewards/margins": 0.8642765879631042,
      "rewards/rejected": -1.0413774251937866,
      "step": 1750
    },
    {
      "epoch": 0.8447324214062875,
      "grad_norm": 52.25,
      "learning_rate": 3.5763357117542364e-07,
      "logits/chosen": -2.6271114349365234,
      "logits/rejected": -2.5810532569885254,
      "logps/chosen": -237.49758911132812,
      "logps/rejected": -229.40185546875,
      "loss": 0.4646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2700980305671692,
      "rewards/margins": 1.0334694385528564,
      "rewards/rejected": -1.3035674095153809,
      "step": 1760
    },
    {
      "epoch": 0.849532037437005,
      "grad_norm": 63.75,
      "learning_rate": 3.363348508515016e-07,
      "logits/chosen": -2.6458356380462646,
      "logits/rejected": -2.519512176513672,
      "logps/chosen": -238.7860565185547,
      "logps/rejected": -220.2080078125,
      "loss": 0.5092,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.2170904129743576,
      "rewards/margins": 0.8388406038284302,
      "rewards/rejected": -1.0559309720993042,
      "step": 1770
    },
    {
      "epoch": 0.8543316534677226,
      "grad_norm": 45.25,
      "learning_rate": 3.156441821492506e-07,
      "logits/chosen": -2.602996349334717,
      "logits/rejected": -2.490618944168091,
      "logps/chosen": -237.382568359375,
      "logps/rejected": -219.4293975830078,
      "loss": 0.4473,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.2120467871427536,
      "rewards/margins": 1.101704478263855,
      "rewards/rejected": -1.313751220703125,
      "step": 1780
    },
    {
      "epoch": 0.8591312694984401,
      "grad_norm": 38.75,
      "learning_rate": 2.9556737973674117e-07,
      "logits/chosen": -2.609147310256958,
      "logits/rejected": -2.473086357116699,
      "logps/chosen": -242.01962280273438,
      "logps/rejected": -202.3125762939453,
      "loss": 0.4658,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.25016945600509644,
      "rewards/margins": 1.0174734592437744,
      "rewards/rejected": -1.267642855644226,
      "step": 1790
    },
    {
      "epoch": 0.8639308855291576,
      "grad_norm": 51.25,
      "learning_rate": 2.761100857681068e-07,
      "logits/chosen": -2.625156879425049,
      "logits/rejected": -2.513139486312866,
      "logps/chosen": -219.42385864257812,
      "logps/rejected": -200.65432739257812,
      "loss": 0.5114,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.38931506872177124,
      "rewards/margins": 0.8760172724723816,
      "rewards/rejected": -1.2653322219848633,
      "step": 1800
    },
    {
      "epoch": 0.8639308855291576,
      "eval_logits/chosen": -2.6100218296051025,
      "eval_logits/rejected": -2.4987967014312744,
      "eval_logps/chosen": -234.25213623046875,
      "eval_logps/rejected": -217.06610107421875,
      "eval_loss": 0.44831663370132446,
      "eval_rewards/accuracies": 0.7990000247955322,
      "eval_rewards/chosen": -0.11191659420728683,
      "eval_rewards/margins": 1.0416151285171509,
      "eval_rewards/rejected": -1.1535316705703735,
      "eval_runtime": 31.9766,
      "eval_samples_per_second": 31.273,
      "eval_steps_per_second": 7.818,
      "step": 1800
    },
    {
      "epoch": 0.8687305015598752,
      "grad_norm": 40.75,
      "learning_rate": 2.5727776829793774e-07,
      "logits/chosen": -2.6402649879455566,
      "logits/rejected": -2.48222017288208,
      "logps/chosen": -238.41708374023438,
      "logps/rejected": -186.76150512695312,
      "loss": 0.4385,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.12383349239826202,
      "rewards/margins": 1.0703352689743042,
      "rewards/rejected": -1.1941686868667603,
      "step": 1810
    },
    {
      "epoch": 0.8735301175905927,
      "grad_norm": 64.5,
      "learning_rate": 2.3907571974460255e-07,
      "logits/chosen": -2.614182949066162,
      "logits/rejected": -2.4683122634887695,
      "logps/chosen": -242.4246063232422,
      "logps/rejected": -199.64266967773438,
      "loss": 0.4524,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18747808039188385,
      "rewards/margins": 0.9742029905319214,
      "rewards/rejected": -1.1616809368133545,
      "step": 1820
    },
    {
      "epoch": 0.8783297336213103,
      "grad_norm": 52.5,
      "learning_rate": 2.2150905540292589e-07,
      "logits/chosen": -2.6297433376312256,
      "logits/rejected": -2.4974730014801025,
      "logps/chosen": -229.2626953125,
      "logps/rejected": -213.91567993164062,
      "loss": 0.4491,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.12955379486083984,
      "rewards/margins": 0.960136890411377,
      "rewards/rejected": -1.0896905660629272,
      "step": 1830
    },
    {
      "epoch": 0.8831293496520278,
      "grad_norm": 39.5,
      "learning_rate": 2.0458271200664626e-07,
      "logits/chosen": -2.5526020526885986,
      "logits/rejected": -2.517185688018799,
      "logps/chosen": -214.2257843017578,
      "logps/rejected": -210.78115844726562,
      "loss": 0.4151,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.33652427792549133,
      "rewards/margins": 1.0866222381591797,
      "rewards/rejected": -1.4231464862823486,
      "step": 1840
    },
    {
      "epoch": 0.8879289656827454,
      "grad_norm": 32.5,
      "learning_rate": 1.8830144634105206e-07,
      "logits/chosen": -2.5996253490448,
      "logits/rejected": -2.453141927719116,
      "logps/chosen": -247.02804565429688,
      "logps/rejected": -199.14678955078125,
      "loss": 0.4273,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.08400492370128632,
      "rewards/margins": 1.1507220268249512,
      "rewards/rejected": -1.234726905822754,
      "step": 1850
    },
    {
      "epoch": 0.8927285817134629,
      "grad_norm": 42.25,
      "learning_rate": 1.7266983390618997e-07,
      "logits/chosen": -2.5644283294677734,
      "logits/rejected": -2.453029155731201,
      "logps/chosen": -228.55294799804688,
      "logps/rejected": -198.43435668945312,
      "loss": 0.4349,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06119618937373161,
      "rewards/margins": 1.1090104579925537,
      "rewards/rejected": -1.1702066659927368,
      "step": 1860
    },
    {
      "epoch": 0.8975281977441805,
      "grad_norm": 49.0,
      "learning_rate": 1.5769226763101887e-07,
      "logits/chosen": -2.5068180561065674,
      "logits/rejected": -2.47198748588562,
      "logps/chosen": -231.07266235351562,
      "logps/rejected": -219.62387084960938,
      "loss": 0.4895,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2498408854007721,
      "rewards/margins": 0.9979842901229858,
      "rewards/rejected": -1.2478251457214355,
      "step": 1870
    },
    {
      "epoch": 0.902327813774898,
      "grad_norm": 54.0,
      "learning_rate": 1.4337295663887086e-07,
      "logits/chosen": -2.659472942352295,
      "logits/rejected": -2.517972946166992,
      "logps/chosen": -239.847900390625,
      "logps/rejected": -198.91372680664062,
      "loss": 0.4644,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.11448581516742706,
      "rewards/margins": 1.0199806690216064,
      "rewards/rejected": -1.134466528892517,
      "step": 1880
    },
    {
      "epoch": 0.9071274298056156,
      "grad_norm": 63.0,
      "learning_rate": 1.2971592506456799e-07,
      "logits/chosen": -2.5675461292266846,
      "logits/rejected": -2.4976906776428223,
      "logps/chosen": -206.1525421142578,
      "logps/rejected": -200.3158721923828,
      "loss": 0.4568,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.2682192623615265,
      "rewards/margins": 1.0316725969314575,
      "rewards/rejected": -1.2998919486999512,
      "step": 1890
    },
    {
      "epoch": 0.9119270458363331,
      "grad_norm": 41.25,
      "learning_rate": 1.1672501092352545e-07,
      "logits/chosen": -2.613196611404419,
      "logits/rejected": -2.4825501441955566,
      "logps/chosen": -241.4010772705078,
      "logps/rejected": -214.26339721679688,
      "loss": 0.4681,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.11850067228078842,
      "rewards/margins": 0.9811543226242065,
      "rewards/rejected": -1.0996549129486084,
      "step": 1900
    },
    {
      "epoch": 0.9167266618670507,
      "grad_norm": 56.0,
      "learning_rate": 1.0440386503315969e-07,
      "logits/chosen": -2.5355827808380127,
      "logits/rejected": -2.4754626750946045,
      "logps/chosen": -224.82363891601562,
      "logps/rejected": -254.0577850341797,
      "loss": 0.4594,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.16619113087654114,
      "rewards/margins": 1.0043249130249023,
      "rewards/rejected": -1.1705158948898315,
      "step": 1910
    },
    {
      "epoch": 0.9215262778977682,
      "grad_norm": 47.0,
      "learning_rate": 9.275594998690574e-08,
      "logits/chosen": -2.5817465782165527,
      "logits/rejected": -2.4193766117095947,
      "logps/chosen": -251.2194061279297,
      "logps/rejected": -203.05404663085938,
      "loss": 0.4609,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.17528951168060303,
      "rewards/margins": 0.9491412043571472,
      "rewards/rejected": -1.1244306564331055,
      "step": 1920
    },
    {
      "epoch": 0.9263258939284857,
      "grad_norm": 44.75,
      "learning_rate": 8.178453918112783e-08,
      "logits/chosen": -2.5946240425109863,
      "logits/rejected": -2.4726099967956543,
      "logps/chosen": -223.893798828125,
      "logps/rejected": -191.31539916992188,
      "loss": 0.4227,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.033659275621175766,
      "rewards/margins": 1.185791015625,
      "rewards/rejected": -1.2194502353668213,
      "step": 1930
    },
    {
      "epoch": 0.9311255099592033,
      "grad_norm": 72.0,
      "learning_rate": 7.149271589520167e-08,
      "logits/chosen": -2.551396131515503,
      "logits/rejected": -2.4351983070373535,
      "logps/chosen": -212.9639129638672,
      "logps/rejected": -204.0959930419922,
      "loss": 0.4853,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.29301124811172485,
      "rewards/margins": 0.9584601521492004,
      "rewards/rejected": -1.2514712810516357,
      "step": 1940
    },
    {
      "epoch": 0.9359251259899208,
      "grad_norm": 42.5,
      "learning_rate": 6.188337242502784e-08,
      "logits/chosen": -2.592571496963501,
      "logits/rejected": -2.4578793048858643,
      "logps/chosen": -238.7187042236328,
      "logps/rejected": -204.2020263671875,
      "loss": 0.4352,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.20334915816783905,
      "rewards/margins": 1.0652612447738647,
      "rewards/rejected": -1.2686102390289307,
      "step": 1950
    },
    {
      "epoch": 0.9407247420206384,
      "grad_norm": 34.25,
      "learning_rate": 5.295920927021109e-08,
      "logits/chosen": -2.6256229877471924,
      "logits/rejected": -2.5241026878356934,
      "logps/chosen": -236.45401000976562,
      "logps/rejected": -210.2203369140625,
      "loss": 0.4396,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.19231542944908142,
      "rewards/margins": 1.1057020425796509,
      "rewards/rejected": -1.2980175018310547,
      "step": 1960
    },
    {
      "epoch": 0.9455243580513559,
      "grad_norm": 44.5,
      "learning_rate": 4.472273437514357e-08,
      "logits/chosen": -2.644946336746216,
      "logits/rejected": -2.526045083999634,
      "logps/chosen": -258.8535461425781,
      "logps/rejected": -222.54019165039062,
      "loss": 0.4178,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06698047369718552,
      "rewards/margins": 1.1277925968170166,
      "rewards/rejected": -1.1947730779647827,
      "step": 1970
    },
    {
      "epoch": 0.9503239740820735,
      "grad_norm": 36.0,
      "learning_rate": 3.717626242420252e-08,
      "logits/chosen": -2.60685658454895,
      "logits/rejected": -2.5159687995910645,
      "logps/chosen": -223.88827514648438,
      "logps/rejected": -212.7635498046875,
      "loss": 0.5017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22438673675060272,
      "rewards/margins": 0.8267514109611511,
      "rewards/rejected": -1.0511382818222046,
      "step": 1980
    },
    {
      "epoch": 0.955123590112791,
      "grad_norm": 43.75,
      "learning_rate": 3.03219141912553e-08,
      "logits/chosen": -2.586005687713623,
      "logits/rejected": -2.484550714492798,
      "logps/chosen": -244.2949676513672,
      "logps/rejected": -225.6749725341797,
      "loss": 0.4699,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.22158965468406677,
      "rewards/margins": 0.8752668499946594,
      "rewards/rejected": -1.0968565940856934,
      "step": 1990
    },
    {
      "epoch": 0.9599232061435086,
      "grad_norm": 49.75,
      "learning_rate": 2.4161615943664174e-08,
      "logits/chosen": -2.6366090774536133,
      "logits/rejected": -2.5460915565490723,
      "logps/chosen": -219.8328094482422,
      "logps/rejected": -214.83737182617188,
      "loss": 0.4763,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11922893673181534,
      "rewards/margins": 0.9408013224601746,
      "rewards/rejected": -1.060030221939087,
      "step": 2000
    },
    {
      "epoch": 0.9599232061435086,
      "eval_logits/chosen": -2.610217571258545,
      "eval_logits/rejected": -2.4989378452301025,
      "eval_logps/chosen": -234.27659606933594,
      "eval_logps/rejected": -217.0829620361328,
      "eval_loss": 0.4485087990760803,
      "eval_rewards/accuracies": 0.7950000166893005,
      "eval_rewards/chosen": -0.11436203867197037,
      "eval_rewards/margins": 1.0408560037612915,
      "eval_rewards/rejected": -1.1552180051803589,
      "eval_runtime": 21.4702,
      "eval_samples_per_second": 46.576,
      "eval_steps_per_second": 11.644,
      "step": 2000
    },
    {
      "epoch": 0.9647228221742261,
      "grad_norm": 34.5,
      "learning_rate": 1.8697098900948285e-08,
      "logits/chosen": -2.5669498443603516,
      "logits/rejected": -2.4752142429351807,
      "logps/chosen": -227.6999053955078,
      "logps/rejected": -230.0476531982422,
      "loss": 0.435,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.15688031911849976,
      "rewards/margins": 1.0234278440475464,
      "rewards/rejected": -1.180308222770691,
      "step": 2010
    },
    {
      "epoch": 0.9695224382049437,
      "grad_norm": 48.25,
      "learning_rate": 1.392989874826195e-08,
      "logits/chosen": -2.6464297771453857,
      "logits/rejected": -2.513470411300659,
      "logps/chosen": -231.412353515625,
      "logps/rejected": -216.2265167236328,
      "loss": 0.5319,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2918005585670471,
      "rewards/margins": 0.7504169940948486,
      "rewards/rejected": -1.042217493057251,
      "step": 2020
    },
    {
      "epoch": 0.9743220542356611,
      "grad_norm": 52.5,
      "learning_rate": 9.861355204825173e-09,
      "logits/chosen": -2.6138572692871094,
      "logits/rejected": -2.4997787475585938,
      "logps/chosen": -259.1074523925781,
      "logps/rejected": -211.08761596679688,
      "loss": 0.4733,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.19954562187194824,
      "rewards/margins": 1.0070618391036987,
      "rewards/rejected": -1.206607460975647,
      "step": 2030
    },
    {
      "epoch": 0.9791216702663786,
      "grad_norm": 36.5,
      "learning_rate": 6.492611647420932e-09,
      "logits/chosen": -2.594069004058838,
      "logits/rejected": -2.459033250808716,
      "logps/chosen": -230.0123291015625,
      "logps/rejected": -206.733642578125,
      "loss": 0.4503,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.22017189860343933,
      "rewards/margins": 1.0728034973144531,
      "rewards/rejected": -1.2929753065109253,
      "step": 2040
    },
    {
      "epoch": 0.9839212862970962,
      "grad_norm": 51.25,
      "learning_rate": 3.8246147890763645e-09,
      "logits/chosen": -2.643317699432373,
      "logits/rejected": -2.504284143447876,
      "logps/chosen": -246.441162109375,
      "logps/rejected": -210.845458984375,
      "loss": 0.494,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2079886943101883,
      "rewards/margins": 0.7775675058364868,
      "rewards/rejected": -0.9855562448501587,
      "step": 2050
    },
    {
      "epoch": 0.9887209023278137,
      "grad_norm": 38.0,
      "learning_rate": 1.8581144130089269e-09,
      "logits/chosen": -2.567206859588623,
      "logits/rejected": -2.506730794906616,
      "logps/chosen": -222.69644165039062,
      "logps/rejected": -218.9879913330078,
      "loss": 0.4439,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1257813274860382,
      "rewards/margins": 1.1031692028045654,
      "rewards/rejected": -1.2289507389068604,
      "step": 2060
    },
    {
      "epoch": 0.9935205183585313,
      "grad_norm": 53.25,
      "learning_rate": 5.936631619152256e-10,
      "logits/chosen": -2.615421772003174,
      "logits/rejected": -2.54068660736084,
      "logps/chosen": -233.13818359375,
      "logps/rejected": -214.350341796875,
      "loss": 0.4651,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.03799530863761902,
      "rewards/margins": 1.0076466798782349,
      "rewards/rejected": -1.0456420183181763,
      "step": 2070
    },
    {
      "epoch": 0.9983201343892488,
      "grad_norm": 55.75,
      "learning_rate": 3.161638266302447e-11,
      "logits/chosen": -2.651322841644287,
      "logits/rejected": -2.516923666000366,
      "logps/chosen": -235.73876953125,
      "logps/rejected": -220.57421875,
      "loss": 0.47,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.25053197145462036,
      "rewards/margins": 0.9338071942329407,
      "rewards/rejected": -1.184339165687561,
      "step": 2080
    },
    {
      "epoch": 0.9997600191984641,
      "step": 2083,
      "total_flos": 0.0,
      "train_loss": 0.49830314429746014,
      "train_runtime": 2918.5416,
      "train_samples_per_second": 11.421,
      "train_steps_per_second": 0.714
    }
  ],
  "logging_steps": 10,
  "max_steps": 2083,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}