{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.47609219126247,
  "eval_steps": 100,
  "global_step": 1800,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0,
      "eval_entropy": 1.6422131361525043,
      "eval_logits/chosen": -3.145482037066458,
      "eval_logits/rejected": -3.077003725568922,
      "eval_logps/chosen": -109.81586472505059,
      "eval_logps/rejected": -150.2786612167857,
      "eval_loss": 0.6931471228599548,
      "eval_mean_token_accuracy": 0.49587641841445873,
      "eval_num_tokens": 0.0,
      "eval_rewards/accuracies": 0.0,
      "eval_rewards/chosen": 0.0,
      "eval_rewards/margins": 0.0,
      "eval_rewards/rejected": 0.0,
      "eval_runtime": 47.3541,
      "eval_samples_per_second": 12.924,
      "eval_steps_per_second": 6.462,
      "step": 0
    },
    {
      "entropy": 1.6841309666633606,
      "epoch": 0.0013759889920880633,
      "grad_norm": 33.75,
      "learning_rate": 0.0,
      "logits/chosen": -3.163112659670131,
      "logits/rejected": -3.0765846200610514,
      "logps/chosen": -106.31735277175903,
      "logps/rejected": -147.35992622375488,
      "loss": 0.6931471824645996,
      "mean_token_accuracy": 0.46893012523651123,
      "num_tokens": 6745.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "entropy": 1.631808305780093,
      "epoch": 0.013759889920880633,
      "grad_norm": 11.9375,
      "learning_rate": 8.256880733944956e-07,
      "logits/chosen": -3.156389862999097,
      "logits/rejected": -3.0980865636330024,
      "logps/chosen": -108.15034267637465,
      "logps/rejected": -151.11039204067654,
      "loss": 0.5401247872246636,
      "mean_token_accuracy": 0.5070789319773515,
      "num_tokens": 70146.0,
      "rewards/accuracies": 0.8333333333333334,
      "rewards/chosen": 0.24961456178344735,
      "rewards/margins": 0.39007408768197316,
      "rewards/rejected": -0.1404595249849889,
      "step": 10
    },
    {
      "entropy": 1.4963100239634515,
      "epoch": 0.027519779841761266,
      "grad_norm": 0.03759765625,
      "learning_rate": 1.743119266055046e-06,
      "logits/chosen": -3.23790955675432,
      "logits/rejected": -3.0797231596356247,
      "logps/chosen": -85.77736120223999,
      "logps/rejected": -178.22046175003052,
      "loss": 0.047147470712661746,
      "mean_token_accuracy": 0.5772599957883358,
      "num_tokens": 145514.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4378453686833383,
      "rewards/margins": 4.863506855070591,
      "rewards/rejected": -2.425661455094814,
      "step": 20
    },
    {
      "entropy": 1.3544968962669373,
      "epoch": 0.0412796697626419,
      "grad_norm": 0.01318359375,
      "learning_rate": 2.6605504587155968e-06,
      "logits/chosen": -3.332316438054062,
      "logits/rejected": -3.0564629823512917,
      "logps/chosen": -69.85810537338257,
      "logps/rejected": -219.8774684906006,
      "loss": 0.00012127072550356388,
      "mean_token_accuracy": 0.6386784806847572,
      "num_tokens": 216437.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.659462496638298,
      "rewards/margins": 10.62372499704361,
      "rewards/rejected": -6.964262568950653,
      "step": 30
    },
    {
      "entropy": 1.3117856681346893,
      "epoch": 0.05503955968352253,
      "grad_norm": 0.000865936279296875,
      "learning_rate": 3.5779816513761473e-06,
      "logits/chosen": -3.3248741358928164,
      "logits/rejected": -3.0449267511018827,
      "logps/chosen": -68.93827650547027,
      "logps/rejected": -239.04862117767334,
      "loss": 2.7265999233350157e-05,
      "mean_token_accuracy": 0.658780874311924,
      "num_tokens": 291812.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.162062160670757,
      "rewards/margins": 13.01834270954132,
      "rewards/rejected": -8.856280529499054,
      "step": 40
    },
    {
      "entropy": 1.3213376998901367,
      "epoch": 0.06879944960440317,
      "grad_norm": 0.0019683837890625,
      "learning_rate": 4.4954128440366975e-06,
      "logits/chosen": -3.3137762086238127,
      "logits/rejected": -3.042837344703322,
      "logps/chosen": -65.1577981710434,
      "logps/rejected": -232.65355396270752,
      "loss": 2.894022036343813e-05,
      "mean_token_accuracy": 0.6745641171932221,
      "num_tokens": 357296.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.219022332131862,
      "rewards/margins": 12.86244809627533,
      "rewards/rejected": -8.643425732851028,
      "step": 50
    },
    {
      "entropy": 1.310352511703968,
      "epoch": 0.0825593395252838,
      "grad_norm": 0.0012359619140625,
      "learning_rate": 5.412844036697248e-06,
      "logits/chosen": -3.316903429840562,
      "logits/rejected": -3.0264215624626054,
      "logps/chosen": -66.44462168216705,
      "logps/rejected": -245.0753698348999,
      "loss": 1.3988211867399513e-05,
      "mean_token_accuracy": 0.6762391120195389,
      "num_tokens": 426934.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.29778570085764,
      "rewards/margins": 13.754501032829285,
      "rewards/rejected": -9.456715273857117,
      "step": 60
    },
    {
      "entropy": 1.3216734871268272,
      "epoch": 0.09631922944616443,
      "grad_norm": 0.0003986358642578125,
      "learning_rate": 6.330275229357799e-06,
      "logits/chosen": -3.3179312487999497,
      "logits/rejected": -3.0427430948262106,
      "logps/chosen": -66.5002682685852,
      "logps/rejected": -240.90171585083007,
      "loss": 3.928586666006595e-05,
      "mean_token_accuracy": 0.6831246554851532,
      "num_tokens": 495203.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.366098646819592,
      "rewards/margins": 13.849577975273132,
      "rewards/rejected": -9.483479243516921,
      "step": 70
    },
    {
      "entropy": 1.3014291919767857,
      "epoch": 0.11007911936704506,
      "grad_norm": 0.002532958984375,
      "learning_rate": 7.247706422018349e-06,
      "logits/chosen": -3.3127891155448475,
      "logits/rejected": -3.031499314118935,
      "logps/chosen": -67.93886163234711,
      "logps/rejected": -251.54950313568116,
      "loss": 1.0652156197465956e-05,
      "mean_token_accuracy": 0.674790358543396,
      "num_tokens": 570516.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.4089073032140735,
      "rewards/margins": 13.979358959197999,
      "rewards/rejected": -9.57045162320137,
      "step": 80
    },
    {
      "entropy": 1.3178453385829925,
      "epoch": 0.1238390092879257,
      "grad_norm": 0.000690460205078125,
      "learning_rate": 8.1651376146789e-06,
      "logits/chosen": -3.3227773672405094,
      "logits/rejected": -3.040103606137686,
      "logps/chosen": -65.88669891357422,
      "logps/rejected": -249.75017547607422,
      "loss": 1.5238465857692063e-05,
      "mean_token_accuracy": 0.6740365795791149,
      "num_tokens": 645105.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.395410162210465,
      "rewards/margins": 14.197139203548431,
      "rewards/rejected": -9.80172905921936,
      "step": 90
    },
    {
      "entropy": 1.333089178800583,
      "epoch": 0.13759889920880633,
      "grad_norm": 0.007080078125,
      "learning_rate": 9.08256880733945e-06,
      "logits/chosen": -3.3304067715679757,
      "logits/rejected": -3.0432493623910477,
      "logps/chosen": -64.5229917526245,
      "logps/rejected": -253.41995105743408,
      "loss": 8.196531416615471e-06,
      "mean_token_accuracy": 0.680670789629221,
      "num_tokens": 716266.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.433503310382366,
      "rewards/margins": 14.470386946201325,
      "rewards/rejected": -10.036883610486985,
      "step": 100
    },
    {
      "epoch": 0.13759889920880633,
      "eval_entropy": 1.3168993187106512,
      "eval_logits/chosen": -3.322066357791712,
      "eval_logits/rejected": -3.021388443109954,
      "eval_logps/chosen": -65.62298910602246,
      "eval_logps/rejected": -252.51850432352302,
      "eval_loss": 9.113909982261248e-06,
      "eval_mean_token_accuracy": 0.6737550743459876,
      "eval_num_tokens": 716266.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 4.419287687033609,
      "eval_rewards/margins": 14.643272069544574,
      "eval_rewards/rejected": -10.223984398873023,
      "eval_runtime": 46.4829,
      "eval_samples_per_second": 13.166,
      "eval_steps_per_second": 6.583,
      "step": 100
    },
    {
      "entropy": 1.3123450234532357,
      "epoch": 0.15135878912968695,
      "grad_norm": 0.0001239776611328125,
      "learning_rate": 1e-05,
      "logits/chosen": -3.331090101614931,
      "logits/rejected": -3.038216613475043,
      "logps/chosen": -59.395786821842194,
      "logps/rejected": -254.9130277633667,
      "loss": 1.392655831295997e-05,
      "mean_token_accuracy": 0.6849603869020939,
      "num_tokens": 793653.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.183176264166832,
      "rewards/margins": 14.316125285625457,
      "rewards/rejected": -10.132948994636536,
      "step": 110
    },
    {
      "entropy": 1.2991145364940166,
      "epoch": 0.1651186790505676,
      "grad_norm": 0.000370025634765625,
      "learning_rate": 1.091743119266055e-05,
      "logits/chosen": -3.321651732468068,
      "logits/rejected": -3.013930166082175,
      "logps/chosen": -64.8379723072052,
      "logps/rejected": -258.61792945861816,
      "loss": 1.0242476855637506e-05,
      "mean_token_accuracy": 0.6761004894971847,
      "num_tokens": 870785.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.295385938882828,
      "rewards/margins": 14.680606830120087,
      "rewards/rejected": -10.385220789909363,
      "step": 120
    },
    {
      "entropy": 1.3162065342068672,
      "epoch": 0.17887856897144822,
      "grad_norm": 0.0022125244140625,
      "learning_rate": 1.1834862385321102e-05,
      "logits/chosen": -3.315425730055486,
      "logits/rejected": -3.022912476453253,
      "logps/chosen": -65.0863746881485,
      "logps/rejected": -252.13726730346679,
      "loss": 1.1276499571977184e-05,
      "mean_token_accuracy": 0.6769515044987202,
      "num_tokens": 944782.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.321224331855774,
      "rewards/margins": 14.760894215106964,
      "rewards/rejected": -10.439669924974442,
      "step": 130
    },
    {
      "entropy": 1.323652510344982,
      "epoch": 0.19263845889232886,
      "grad_norm": 6.437301635742188e-05,
      "learning_rate": 1.2752293577981652e-05,
      "logits/chosen": -3.3024622470454594,
      "logits/rejected": -3.034751877652379,
      "logps/chosen": -65.11928598880768,
      "logps/rejected": -261.44528636932375,
      "loss": 2.951903297798708e-06,
      "mean_token_accuracy": 0.6833141274750233,
      "num_tokens": 1014352.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.368463853001595,
      "rewards/margins": 15.111195981502533,
      "rewards/rejected": -10.742732137441635,
      "step": 140
    },
    {
      "entropy": 1.2840273901820183,
      "epoch": 0.20639834881320948,
      "grad_norm": 0.00396728515625,
      "learning_rate": 1.3669724770642203e-05,
      "logits/chosen": -3.3172900030591683,
      "logits/rejected": -3.0113253204883432,
      "logps/chosen": -62.9616664648056,
      "logps/rejected": -264.60894412994384,
      "loss": 3.129429023829289e-06,
      "mean_token_accuracy": 0.6832372471690178,
      "num_tokens": 1088461.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.151188506186008,
      "rewards/margins": 15.375397801399231,
      "rewards/rejected": -11.224209105968475,
      "step": 150
    },
    {
      "entropy": 1.2966242536902428,
      "epoch": 0.22015823873409013,
      "grad_norm": 0.00154876708984375,
      "learning_rate": 1.4587155963302753e-05,
      "logits/chosen": -3.3151515082339706,
      "logits/rejected": -3.018693861306375,
      "logps/chosen": -62.411375761032104,
      "logps/rejected": -260.221813583374,
      "loss": 5.149983917362988e-06,
      "mean_token_accuracy": 0.6751810491085053,
      "num_tokens": 1164520.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.324609813094139,
      "rewards/margins": 15.264867746829987,
      "rewards/rejected": -10.940258049964905,
      "step": 160
    },
    {
      "entropy": 1.3339879259467124,
      "epoch": 0.23391812865497075,
      "grad_norm": 0.004364013671875,
      "learning_rate": 1.5504587155963304e-05,
      "logits/chosen": -3.2697318298829514,
      "logits/rejected": -3.011607351493888,
      "logps/chosen": -76.78109130859374,
      "logps/rejected": -254.79522857666015,
      "loss": 0.00010178745724260807,
      "mean_token_accuracy": 0.6379761226475239,
      "num_tokens": 1236473.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0435921967029573,
      "rewards/margins": 13.237756145000457,
      "rewards/rejected": -10.194163882732392,
      "step": 170
    },
    {
      "entropy": 1.34509097635746,
      "epoch": 0.2476780185758514,
      "grad_norm": 0.0003337860107421875,
      "learning_rate": 1.6422018348623852e-05,
      "logits/chosen": -3.2700272303608466,
      "logits/rejected": -3.0345531468099116,
      "logps/chosen": -83.20087404251099,
      "logps/rejected": -280.89504661560056,
      "loss": 7.803936023265124e-06,
      "mean_token_accuracy": 0.5892071291804314,
      "num_tokens": 1307721.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.824196879565716,
      "rewards/margins": 15.685904848575593,
      "rewards/rejected": -12.86170780658722,
      "step": 180
    },
    {
      "entropy": 1.2848817110061646,
      "epoch": 0.26143790849673204,
      "grad_norm": 0.00023746490478515625,
      "learning_rate": 1.7339449541284407e-05,
      "logits/chosen": -3.2647128370329717,
      "logits/rejected": -3.050985384983876,
      "logps/chosen": -82.21896734237671,
      "logps/rejected": -295.75822124481203,
      "loss": 2.34917170018889e-06,
      "mean_token_accuracy": 0.5797355584800243,
      "num_tokens": 1376355.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.341776143759489,
      "rewards/margins": 17.304658508300783,
      "rewards/rejected": -14.962882256507873,
      "step": 190
    },
    {
      "entropy": 1.2651859059929849,
      "epoch": 0.27519779841761266,
      "grad_norm": 3.9577484130859375e-05,
      "learning_rate": 1.8256880733944955e-05,
      "logits/chosen": -3.2684643828569073,
      "logits/rejected": -3.038824479770841,
      "logps/chosen": -81.89948139190673,
      "logps/rejected": -302.0743787765503,
      "loss": 9.769327334652188e-07,
      "mean_token_accuracy": 0.5773227371275425,
      "num_tokens": 1448355.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4392453789711,
      "rewards/margins": 18.04160568714142,
      "rewards/rejected": -15.602360212802887,
      "step": 200
    },
    {
      "epoch": 0.27519779841761266,
      "eval_entropy": 1.2808947543692744,
      "eval_logits/chosen": -3.2662835644748456,
      "eval_logits/rejected": -3.030983888937842,
      "eval_logps/chosen": -85.24166726131065,
      "eval_logps/rejected": -310.94012012357024,
      "eval_loss": 5.58038095732627e-07,
      "eval_mean_token_accuracy": 0.5686757371706121,
      "eval_num_tokens": 1448355.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.457419808199203,
      "eval_rewards/margins": 18.523565981123184,
      "eval_rewards/rejected": -16.066146196103563,
      "eval_runtime": 46.6414,
      "eval_samples_per_second": 13.121,
      "eval_steps_per_second": 6.561,
      "step": 200
    },
    {
      "entropy": 1.2722094446420669,
      "epoch": 0.2889576883384933,
      "grad_norm": 3.7670135498046875e-05,
      "learning_rate": 1.9174311926605506e-05,
      "logits/chosen": -3.274673464599805,
      "logits/rejected": -3.0529926146478457,
      "logps/chosen": -80.9079598903656,
      "logps/rejected": -311.301748085022,
      "loss": 2.0904919892927864e-07,
      "mean_token_accuracy": 0.5677489962428808,
      "num_tokens": 1518187.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.386711198091507,
      "rewards/margins": 18.499667203426363,
      "rewards/rejected": -16.112956094741822,
      "step": 210
    },
    {
      "entropy": 1.2693706572055816,
      "epoch": 0.3027175782593739,
      "grad_norm": 0.00010538101196289062,
      "learning_rate": 1.9999987193541166e-05,
      "logits/chosen": -3.2678955318973606,
      "logits/rejected": -3.0331391966114434,
      "logps/chosen": -82.6086644411087,
      "logps/rejected": -318.87992038726804,
      "loss": 4.3506806832738223e-07,
      "mean_token_accuracy": 0.5785332221537829,
      "num_tokens": 1592692.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4188714638352393,
      "rewards/margins": 18.80655164718628,
      "rewards/rejected": -16.387680435180663,
      "step": 220
    },
    {
      "entropy": 1.2727026402950288,
      "epoch": 0.3164774681802546,
      "grad_norm": 3.814697265625e-05,
      "learning_rate": 1.9998450458169935e-05,
      "logits/chosen": -3.263175326214511,
      "logits/rejected": -3.0319927272644476,
      "logps/chosen": -89.12346024513245,
      "logps/rejected": -318.03904724121094,
      "loss": 2.9918028303654865e-07,
      "mean_token_accuracy": 0.5649005927145481,
      "num_tokens": 1665764.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3706042617559433,
      "rewards/margins": 18.84476692676544,
      "rewards/rejected": -16.474162554740907,
      "step": 230
    },
    {
      "entropy": 1.2772931650280952,
      "epoch": 0.3302373581011352,
      "grad_norm": 0.000888824462890625,
      "learning_rate": 1.9994352882027634e-05,
      "logits/chosen": -3.275916983604721,
      "logits/rejected": -3.045261285523395,
      "logps/chosen": -85.88168826103211,
      "logps/rejected": -314.67186374664306,
      "loss": 7.497599654016085e-07,
      "mean_token_accuracy": 0.5647643093019724,
      "num_tokens": 1738039.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3492226861417294,
      "rewards/margins": 18.621605122089385,
      "rewards/rejected": -16.272382295131685,
      "step": 240
    },
    {
      "entropy": 1.2810615047812461,
      "epoch": 0.3439972480220158,
      "grad_norm": 0.0001277923583984375,
      "learning_rate": 1.9987695514600882e-05,
      "logits/chosen": -3.2587914698091525,
      "logits/rejected": -3.039852993690482,
      "logps/chosen": -84.9753231048584,
      "logps/rejected": -316.3932315826416,
      "loss": 2.315052597623435e-07,
      "mean_token_accuracy": 0.5717308051884175,
      "num_tokens": 1809878.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4452888280153275,
      "rewards/margins": 18.718373239040375,
      "rewards/rejected": -16.273084354400634,
      "step": 250
    },
    {
      "entropy": 1.2803627982735635,
      "epoch": 0.35775713794289643,
      "grad_norm": 1.4841556549072266e-05,
      "learning_rate": 1.997848006099969e-05,
      "logits/chosen": -3.243708828438259,
      "logits/rejected": -3.023015530025309,
      "logps/chosen": -85.64196882247924,
      "logps/rejected": -320.8713493347168,
      "loss": 4.767807695316151e-06,
      "mean_token_accuracy": 0.5577906873077154,
      "num_tokens": 1880471.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.241638096421957,
      "rewards/margins": 18.844607853889464,
      "rewards/rejected": -16.602969908714293,
      "step": 260
    },
    {
      "entropy": 1.2728056520223618,
      "epoch": 0.3715170278637771,
      "grad_norm": 2.9087066650390625e-05,
      "learning_rate": 1.9966708881520725e-05,
      "logits/chosen": -3.1717140539593514,
      "logits/rejected": -2.9289080783786265,
      "logps/chosen": -109.57758569717407,
      "logps/rejected": -343.55820388793944,
      "loss": 3.192686335751205e-07,
      "mean_token_accuracy": 0.5119530703872442,
      "num_tokens": 1954954.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2822640292346478,
      "rewards/margins": 19.274557650089264,
      "rewards/rejected": -18.99229363203049,
      "step": 270
    },
    {
      "entropy": 1.258153322339058,
      "epoch": 0.3852769177846577,
      "grad_norm": 4.1961669921875e-05,
      "learning_rate": 1.9952384991042794e-05,
      "logits/chosen": -3.166911320420769,
      "logits/rejected": -2.914455899618946,
      "logps/chosen": -109.23124113082886,
      "logps/rejected": -342.96191749572756,
      "loss": 6.130666406534147e-07,
      "mean_token_accuracy": 0.5128110606223345,
      "num_tokens": 2027701.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11331945937126875,
      "rewards/margins": 19.085047161579134,
      "rewards/rejected": -19.198366713523864,
      "step": 280
    },
    {
      "entropy": 1.2625004753470421,
      "epoch": 0.39903680770553834,
      "grad_norm": 0.00022029876708984375,
      "learning_rate": 1.9935512058254642e-05,
      "logits/chosen": -3.158640726546935,
      "logits/rejected": -2.919291940387602,
      "logps/chosen": -109.37000555992127,
      "logps/rejected": -348.1477699279785,
      "loss": 3.579470103431959e-07,
      "mean_token_accuracy": 0.5032326009124517,
      "num_tokens": 2097529.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2131506633013487,
      "rewards/margins": 19.291283941268922,
      "rewards/rejected": -19.504434680938722,
      "step": 290
    },
    {
      "entropy": 1.2408581115305424,
      "epoch": 0.41279669762641896,
      "grad_norm": 0.0006561279296875,
      "learning_rate": 1.9916094404715338e-05,
      "logits/chosen": -3.146471768568003,
      "logits/rejected": -2.9109541862173147,
      "logps/chosen": -108.3150071144104,
      "logps/rejected": -346.6201908111572,
      "loss": 4.624724624591181e-07,
      "mean_token_accuracy": 0.5017246019095183,
      "num_tokens": 2164725.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2834041160531342,
      "rewards/margins": 19.647798562049864,
      "rewards/rejected": -19.931202745437623,
      "step": 300
    },
    {
      "epoch": 0.41279669762641896,
      "eval_entropy": 1.2567425892243977,
      "eval_logits/chosen": -3.1541197386317377,
      "eval_logits/rejected": -2.9072247599638548,
      "eval_logps/chosen": -110.04411519119164,
      "eval_logps/rejected": -345.3444980297213,
      "eval_loss": 1.5399403991978033e-06,
      "eval_mean_token_accuracy": 0.503020686164401,
      "eval_num_tokens": 2164725.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.022825035449190467,
      "eval_rewards/margins": 19.483759047938328,
      "eval_rewards/rejected": -19.506584064633238,
      "eval_runtime": 46.7375,
      "eval_samples_per_second": 13.094,
      "eval_steps_per_second": 6.547,
      "step": 300
    },
    {
      "entropy": 1.2747977674007416,
      "epoch": 0.42655658754729964,
      "grad_norm": 4.363059997558594e-05,
      "learning_rate": 1.9894137003747404e-05,
      "logits/chosen": -3.191046102409698,
      "logits/rejected": -2.934731907158622,
      "logps/chosen": -105.27848653793335,
      "logps/rejected": -349.7217990875244,
      "loss": 3.7774039810756222e-06,
      "mean_token_accuracy": 0.5230023309588432,
      "num_tokens": 2240508.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6149379150010645,
      "rewards/margins": 20.34483733177185,
      "rewards/rejected": -19.72989946603775,
      "step": 310
    },
    {
      "entropy": 1.2644944354891776,
      "epoch": 0.44031647746818026,
      "grad_norm": 2.4437904357910156e-05,
      "learning_rate": 1.9869645479163035e-05,
      "logits/chosen": -3.200015615413726,
      "logits/rejected": -2.935862106651629,
      "logps/chosen": -98.56399579048157,
      "logps/rejected": -345.60072326660156,
      "loss": 1.755031689754105e-07,
      "mean_token_accuracy": 0.5542162295430899,
      "num_tokens": 2311444.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1905555572360753,
      "rewards/margins": 20.78347454071045,
      "rewards/rejected": -19.592919039726258,
      "step": 320
    },
    {
      "entropy": 1.2220299974083901,
      "epoch": 0.4540763673890609,
      "grad_norm": 8.158385753631592e-07,
      "learning_rate": 1.984262610382371e-05,
      "logits/chosen": -3.2061829556179062,
      "logits/rejected": -2.9369843729159575,
      "logps/chosen": -88.93789119720459,
      "logps/rejected": -350.99997215271,
      "loss": 1.22557608506213e-08,
      "mean_token_accuracy": 0.5557950928807258,
      "num_tokens": 2380803.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3155047172680496,
      "rewards/margins": 21.637652850151063,
      "rewards/rejected": -20.322148036956786,
      "step": 330
    },
    {
      "entropy": 1.2497497342526913,
      "epoch": 0.4678362573099415,
      "grad_norm": 1.4230608940124512e-06,
      "learning_rate": 1.981308579803356e-05,
      "logits/chosen": -3.2039305073988444,
      "logits/rejected": -2.935270721662338,
      "logps/chosen": -91.58367230892182,
      "logps/rejected": -352.92971839904783,
      "loss": 2.8684135600087758e-08,
      "mean_token_accuracy": 0.5493994299322367,
      "num_tokens": 2451152.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3400580430403353,
      "rewards/margins": 21.626568651199342,
      "rewards/rejected": -20.28651065826416,
      "step": 340
    },
    {
      "entropy": 1.2471516221761703,
      "epoch": 0.48159614723082217,
      "grad_norm": 8.083879947662354e-07,
      "learning_rate": 1.9781032127766897e-05,
      "logits/chosen": -3.192242463166784,
      "logits/rejected": -2.939047130509837,
      "logps/chosen": -91.81347761154174,
      "logps/rejected": -351.9379182815552,
      "loss": 4.979311825081823e-08,
      "mean_token_accuracy": 0.5597729679197073,
      "num_tokens": 2520962.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1749618555419148,
      "rewards/margins": 21.238007771968842,
      "rewards/rejected": -20.063046038150787,
      "step": 350
    },
    {
      "entropy": 1.250276905298233,
      "epoch": 0.4953560371517028,
      "grad_norm": 4.7206878662109375e-05,
      "learning_rate": 1.974647330273043e-05,
      "logits/chosen": -3.204588056757823,
      "logits/rejected": -2.9460527693957155,
      "logps/chosen": -96.47925109863282,
      "logps/rejected": -357.84326496124265,
      "loss": 9.702721399662551e-08,
      "mean_token_accuracy": 0.5596798613667489,
      "num_tokens": 2596708.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4331018382683396,
      "rewards/margins": 21.43447951078415,
      "rewards/rejected": -20.00137746334076,
      "step": 360
    },
    {
      "entropy": 1.2356461606919766,
      "epoch": 0.5091159270725835,
      "grad_norm": 1.0192394256591797e-05,
      "learning_rate": 1.9709418174260523e-05,
      "logits/chosen": -3.202175928278776,
      "logits/rejected": -2.9334896374443966,
      "logps/chosen": -93.25146460533142,
      "logps/rejected": -348.1914405822754,
      "loss": 1.4943527730792995e-07,
      "mean_token_accuracy": 0.5434729922562838,
      "num_tokens": 2663359.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1101246356964112,
      "rewards/margins": 21.274962854385375,
      "rewards/rejected": -20.16483825445175,
      "step": 370
    },
    {
      "entropy": 1.2542329356074333,
      "epoch": 0.5228758169934641,
      "grad_norm": 2.9087066650390625e-05,
      "learning_rate": 1.9669876233056173e-05,
      "logits/chosen": -3.2033071908214827,
      "logits/rejected": -2.9458752015881497,
      "logps/chosen": -95.33303580284118,
      "logps/rejected": -360.0554260253906,
      "loss": 6.821481406404928e-08,
      "mean_token_accuracy": 0.5537502180784941,
      "num_tokens": 2735950.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1969825938344,
      "rewards/margins": 21.316751837730408,
      "rewards/rejected": -20.119769501686097,
      "step": 380
    },
    {
      "entropy": 1.2611618615686893,
      "epoch": 0.5366357069143447,
      "grad_norm": 9.238719940185547e-06,
      "learning_rate": 1.962785760674822e-05,
      "logits/chosen": -3.205445972704493,
      "logits/rejected": -2.9503712190165086,
      "logps/chosen": -94.55541400909424,
      "logps/rejected": -358.6264102935791,
      "loss": 5.3916863862468745e-08,
      "mean_token_accuracy": 0.53789919950068,
      "num_tokens": 2809336.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2039084580726922,
      "rewards/margins": 21.302520561218262,
      "rewards/rejected": -20.098612093925475,
      "step": 390
    },
    {
      "entropy": 1.2611239463090897,
      "epoch": 0.5503955968352253,
      "grad_norm": 3.528594970703125e-05,
      "learning_rate": 1.9583373057305417e-05,
      "logits/chosen": -3.213530981959783,
      "logits/rejected": -2.9513509889366025,
      "logps/chosen": -97.29888286590577,
      "logps/rejected": -342.16270599365237,
      "loss": 7.968355930643156e-08,
      "mean_token_accuracy": 0.5469969604164362,
      "num_tokens": 2881281.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.30086164213717,
      "rewards/margins": 20.653638982772826,
      "rewards/rejected": -19.352777242660522,
      "step": 400
    },
    {
      "epoch": 0.5503955968352253,
      "eval_entropy": 1.2446555466044182,
      "eval_logits/chosen": -3.206116940076223,
      "eval_logits/rejected": -2.932523076753512,
      "eval_logps/chosen": -96.21067571328356,
      "eval_logps/rejected": -350.58938982595805,
      "eval_loss": 1.5858041990668426e-07,
      "eval_mean_token_accuracy": 0.5460730443398157,
      "eval_num_tokens": 2881281.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.360518928933767,
      "eval_rewards/margins": 21.391592069388995,
      "eval_rewards/rejected": -20.031073205611285,
      "eval_runtime": 47.366,
      "eval_samples_per_second": 12.921,
      "eval_steps_per_second": 6.46,
      "step": 400
    },
    {
      "entropy": 1.2470418080687522,
      "epoch": 0.5641554867561059,
      "grad_norm": 4.857778549194336e-06,
      "learning_rate": 1.9536433978278034e-05,
      "logits/chosen": -3.2050009974244076,
      "logits/rejected": -2.940295918899248,
      "logps/chosen": -98.1803062915802,
      "logps/rejected": -353.2428401947021,
      "loss": 6.607142495340668e-08,
      "mean_token_accuracy": 0.5483775656670332,
      "num_tokens": 2951215.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3665587238967418,
      "rewards/margins": 21.411585366725923,
      "rewards/rejected": -20.045026814937593,
      "step": 410
    },
    {
      "entropy": 1.2767939895391465,
      "epoch": 0.5779153766769866,
      "grad_norm": 6.735324859619141e-06,
      "learning_rate": 1.948705239187969e-05,
      "logits/chosen": -3.2111276235458264,
      "logits/rejected": -2.950971977866732,
      "logps/chosen": -98.60952830314636,
      "logps/rejected": -356.9093845367432,
      "loss": 3.040120191144524e-07,
      "mean_token_accuracy": 0.5495793126523495,
      "num_tokens": 3025409.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.322571473289281,
      "rewards/margins": 21.419764494895936,
      "rewards/rejected": -20.097193121910095,
      "step": 420
    },
    {
      "entropy": 1.2340076759457588,
      "epoch": 0.5916752665978672,
      "grad_norm": 3.3855438232421875e-05,
      "learning_rate": 1.9435240945908197e-05,
      "logits/chosen": -3.201115388808664,
      "logits/rejected": -2.9359388030639337,
      "logps/chosen": -93.70817131996155,
      "logps/rejected": -349.5262786865234,
      "loss": 3.1302016623158126e-08,
      "mean_token_accuracy": 0.5430944249033928,
      "num_tokens": 3095506.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0892294511198997,
      "rewards/margins": 21.276236748695375,
      "rewards/rejected": -20.187007427215576,
      "step": 430
    },
    {
      "entropy": 1.2297961503267287,
      "epoch": 0.6054351565187478,
      "grad_norm": 8.52346420288086e-06,
      "learning_rate": 1.9381012910506146e-05,
      "logits/chosen": -3.209182764942939,
      "logits/rejected": -2.9383067598099455,
      "logps/chosen": -94.46383814811706,
      "logps/rejected": -347.6718967437744,
      "loss": 1.1145983762617106e-07,
      "mean_token_accuracy": 0.5509139101952314,
      "num_tokens": 3167474.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.320781459286809,
      "rewards/margins": 21.1637371301651,
      "rewards/rejected": -19.842955470085144,
      "step": 440
    },
    {
      "entropy": 1.238859149813652,
      "epoch": 0.6191950464396285,
      "grad_norm": 3.933906555175781e-06,
      "learning_rate": 1.9324382174762103e-05,
      "logits/chosen": -3.203365340339883,
      "logits/rejected": -2.948441983583794,
      "logps/chosen": -95.94863700866699,
      "logps/rejected": -354.1850410461426,
      "loss": 1.6536616385565139e-06,
      "mean_token_accuracy": 0.5478381294757128,
      "num_tokens": 3237066.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1904785027727485,
      "rewards/margins": 21.46895523071289,
      "rewards/rejected": -20.278476548194885,
      "step": 450
    },
    {
      "entropy": 1.2301946364343166,
      "epoch": 0.6329549363605091,
      "grad_norm": 0.00018310546875,
      "learning_rate": 1.9265363243153302e-05,
      "logits/chosen": -3.2076860666693108,
      "logits/rejected": -2.9415288658974417,
      "logps/chosen": -91.41148219108581,
      "logps/rejected": -354.1453170776367,
      "loss": 9.839211543294368e-08,
      "mean_token_accuracy": 0.5448024846613407,
      "num_tokens": 3307883.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3097239159047604,
      "rewards/margins": 21.563816690444945,
      "rewards/rejected": -20.25409264564514,
      "step": 460
    },
    {
      "entropy": 1.2488635182380676,
      "epoch": 0.6467148262813898,
      "grad_norm": 4.38690185546875e-05,
      "learning_rate": 1.9203971231830683e-05,
      "logits/chosen": -3.2141295170389634,
      "logits/rejected": -2.938989290807725,
      "logps/chosen": -95.46482665538788,
      "logps/rejected": -353.38847160339355,
      "loss": 8.967840017248819e-08,
      "mean_token_accuracy": 0.544929426535964,
      "num_tokens": 3379860.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4560122258961201,
      "rewards/margins": 21.57176778316498,
      "rewards/rejected": -20.1157555103302,
      "step": 470
    },
    {
      "entropy": 1.2580321684479714,
      "epoch": 0.6604747162022704,
      "grad_norm": 1.8700957298278809e-06,
      "learning_rate": 1.9140221864747304e-05,
      "logits/chosen": -3.2029551675787014,
      "logits/rejected": -2.9481494396152192,
      "logps/chosen": -97.8405794620514,
      "logps/rejected": -353.5988826751709,
      "loss": 5.062532181909774e-07,
      "mean_token_accuracy": 0.5462357386946678,
      "num_tokens": 3449532.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3817669712938367,
      "rewards/margins": 21.57014834880829,
      "rewards/rejected": -20.188381361961365,
      "step": 480
    },
    {
      "entropy": 1.2382771328091622,
      "epoch": 0.674234606123151,
      "grad_norm": 1.5497207641601562e-06,
      "learning_rate": 1.9074131469631047e-05,
      "logits/chosen": -3.200084511483456,
      "logits/rejected": -2.9279012361838785,
      "logps/chosen": -96.45669221878052,
      "logps/rejected": -360.72495155334474,
      "loss": 2.6530278773861936e-08,
      "mean_token_accuracy": 0.5516861360520124,
      "num_tokens": 3520115.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3708888601511717,
      "rewards/margins": 22.229914927482604,
      "rewards/rejected": -20.85902613401413,
      "step": 490
    },
    {
      "entropy": 1.231078492105007,
      "epoch": 0.6879944960440316,
      "grad_norm": 2.8461217880249023e-06,
      "learning_rate": 1.9005716973802707e-05,
      "logits/chosen": -3.1932053921101393,
      "logits/rejected": -2.9393451806687594,
      "logps/chosen": -92.85203433036804,
      "logps/rejected": -356.5406131744385,
      "loss": 2.374201386601271e-08,
      "mean_token_accuracy": 0.5493354115635156,
      "num_tokens": 3586919.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2406577363610267,
      "rewards/margins": 21.57927179336548,
      "rewards/rejected": -20.33861403465271,
      "step": 500
    },
    {
      "epoch": 0.6879944960440316,
      "eval_entropy": 1.2378660793008367,
      "eval_logits/chosen": -3.208702183064752,
      "eval_logits/rejected": -2.9324433814833553,
      "eval_logps/chosen": -94.60752372492372,
      "eval_logps/rejected": -354.49684058295355,
      "eval_loss": 1.1862820059604928e-07,
      "eval_mean_token_accuracy": 0.5471634168445675,
      "eval_num_tokens": 3586919.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5208341437574548,
      "eval_rewards/margins": 21.942652421839096,
      "eval_rewards/rejected": -20.421818287544003,
      "eval_runtime": 48.0212,
      "eval_samples_per_second": 12.744,
      "eval_steps_per_second": 6.372,
      "step": 500
    },
    {
      "entropy": 1.2257853604853153,
      "epoch": 0.7017543859649122,
      "grad_norm": 4.6193599700927734e-06,
      "learning_rate": 1.8934995899840512e-05,
      "logits/chosen": -3.209992468977989,
      "logits/rejected": -2.944684840016539,
      "logps/chosen": -92.81335635185242,
      "logps/rejected": -354.2427951812744,
      "loss": 1.865128069766797e-08,
      "mean_token_accuracy": 0.5567719202488661,
      "num_tokens": 3658756.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5606805901974439,
      "rewards/margins": 22.114846849441527,
      "rewards/rejected": -20.554166197776794,
      "step": 510
    },
    {
      "entropy": 1.2566522896289825,
      "epoch": 0.7155142758857929,
      "grad_norm": 1.0073184967041016e-05,
      "learning_rate": 1.886198636109216e-05,
      "logits/chosen": -3.2030215467134,
      "logits/rejected": -2.9602792303248324,
      "logps/chosen": -93.38595032691956,
      "logps/rejected": -351.78721618652344,
      "loss": 7.779188422318839e-08,
      "mean_token_accuracy": 0.5507799163460732,
      "num_tokens": 3729889.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2870149585418402,
      "rewards/margins": 21.421838641166687,
      "rewards/rejected": -20.134823727607728,
      "step": 520
    },
    {
      "entropy": 1.223513600230217,
      "epoch": 0.7292741658066736,
      "grad_norm": 1.3113021850585938e-05,
      "learning_rate": 1.8786707057035556e-05,
      "logits/chosen": -3.197827496796243,
      "logits/rejected": -2.939210284225032,
      "logps/chosen": -90.15343255996704,
      "logps/rejected": -351.67856483459474,
      "loss": 3.7565149568763444e-08,
      "mean_token_accuracy": 0.5483428921550513,
      "num_tokens": 3797040.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3190150810405612,
      "rewards/margins": 21.713546490669252,
      "rewards/rejected": -20.3945313334465,
      "step": 530
    },
    {
      "entropy": 1.2207642510533332,
      "epoch": 0.7430340557275542,
      "grad_norm": 2.250075340270996e-06,
      "learning_rate": 1.8709177268489455e-05,
      "logits/chosen": -3.2128090846377537,
      "logits/rejected": -2.9295009129558585,
      "logps/chosen": -91.98796620368958,
      "logps/rejected": -357.48371505737305,
      "loss": 4.0783984331937975e-07,
      "mean_token_accuracy": 0.5455800645053387,
      "num_tokens": 3873245.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.308496705815196,
      "rewards/margins": 21.80317234992981,
      "rewards/rejected": -20.494675588607787,
      "step": 540
    },
    {
      "entropy": 1.2357498720288276,
      "epoch": 0.7567939456484348,
      "grad_norm": 5.997717380523682e-07,
      "learning_rate": 1.862941685267516e-05,
      "logits/chosen": -3.2110368310073283,
      "logits/rejected": -2.941539484264484,
      "logps/chosen": -88.533620262146,
      "logps/rejected": -346.5173583984375,
      "loss": 2.4679090415702377e-08,
      "mean_token_accuracy": 0.5551324781030417,
      "num_tokens": 3947480.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5138933001086117,
      "rewards/margins": 21.399971747398375,
      "rewards/rejected": -19.886078691482545,
      "step": 550
    },
    {
      "entropy": 1.2267142876982688,
      "epoch": 0.7705538355693154,
      "grad_norm": 8.52346420288086e-06,
      "learning_rate": 1.8547446238130632e-05,
      "logits/chosen": -3.215176303042424,
      "logits/rejected": -2.9349409611311232,
      "logps/chosen": -97.80274991989135,
      "logps/rejected": -359.84800300598147,
      "loss": 1.0752984280770761e-08,
      "mean_token_accuracy": 0.5577903117984533,
      "num_tokens": 4022803.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5468438116833567,
      "rewards/margins": 22.347780632972718,
      "rewards/rejected": -20.800936925411225,
      "step": 560
    },
    {
      "entropy": 1.2494088008999824,
      "epoch": 0.7843137254901961,
      "grad_norm": 1.9371509552001953e-06,
      "learning_rate": 1.8463286419478256e-05,
      "logits/chosen": -3.211196248953847,
      "logits/rejected": -2.948767881274639,
      "logps/chosen": -95.37720813751221,
      "logps/rejected": -348.9063579559326,
      "loss": 5.1451485205689095e-08,
      "mean_token_accuracy": 0.5470235139131546,
      "num_tokens": 4094199.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3498335065320135,
      "rewards/margins": 21.68865497112274,
      "rewards/rejected": -20.338821601867675,
      "step": 570
    },
    {
      "entropy": 1.2267412588000297,
      "epoch": 0.7980736154110767,
      "grad_norm": 0.000453948974609375,
      "learning_rate": 1.8376958952047606e-05,
      "logits/chosen": -3.205934045300051,
      "logits/rejected": -2.926654426249549,
      "logps/chosen": -93.34790635108948,
      "logps/rejected": -348.32058773040774,
      "loss": 1.6058717164924018e-07,
      "mean_token_accuracy": 0.5513480927795171,
      "num_tokens": 4165161.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4272053100168705,
      "rewards/margins": 21.51763561964035,
      "rewards/rejected": -20.090430426597596,
      "step": 580
    },
    {
      "entropy": 1.2365659534931184,
      "epoch": 0.8118335053319573,
      "grad_norm": 1.773238182067871e-06,
      "learning_rate": 1.8288485946354623e-05,
      "logits/chosen": -3.2136005483338552,
      "logits/rejected": -2.9395607851534296,
      "logps/chosen": -92.54814214706421,
      "logps/rejected": -356.7799785614014,
      "loss": 9.530706392979483e-08,
      "mean_token_accuracy": 0.5449393007904291,
      "num_tokens": 4234086.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4757858499884606,
      "rewards/margins": 22.288536047935487,
      "rewards/rejected": -20.81275030374527,
      "step": 590
    },
    {
      "entropy": 1.2298890359699726,
      "epoch": 0.8255933952528379,
      "grad_norm": 9.179115295410156e-06,
      "learning_rate": 1.8197890062438593e-05,
      "logits/chosen": -3.207707571541648,
      "logits/rejected": -2.946438175072392,
      "logps/chosen": -90.75098037719727,
      "logps/rejected": -361.0145431518555,
      "loss": 1.4683616882393836e-07,
      "mean_token_accuracy": 0.5618239611387252,
      "num_tokens": 4310423.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4461902618408202,
      "rewards/margins": 21.96159906387329,
      "rewards/rejected": -20.515408909320833,
      "step": 600
    },
    {
      "epoch": 0.8255933952528379,
      "eval_entropy": 1.236954848750744,
      "eval_logits/chosen": -3.2089251643222068,
      "eval_logits/rejected": -2.9321425863537924,
      "eval_logps/chosen": -94.51317416297064,
      "eval_logps/rejected": -354.90605422873904,
      "eval_loss": 1.1335689720226583e-07,
      "eval_mean_token_accuracy": 0.5470167230936437,
      "eval_num_tokens": 4310423.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5302690837069663,
      "eval_rewards/margins": 21.993008691500993,
      "eval_rewards/rejected": -20.462739564234916,
      "eval_runtime": 48.6435,
      "eval_samples_per_second": 12.581,
      "eval_steps_per_second": 6.291,
      "step": 600
    },
    {
      "entropy": 1.2389264971017837,
      "epoch": 0.8393532851737187,
      "grad_norm": 4.8160552978515625e-05,
      "learning_rate": 1.810519450405835e-05,
      "logits/chosen": -3.210960691597433,
      "logits/rejected": -2.935060390790913,
      "logps/chosen": -93.46412029266358,
      "logps/rejected": -361.8876728057861,
      "loss": 3.212771616745158e-07,
      "mean_token_accuracy": 0.5596938874572516,
      "num_tokens": 4383786.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4644759917166084,
      "rewards/margins": 22.466219902038574,
      "rewards/rejected": -21.001744067668916,
      "step": 610
    },
    {
      "entropy": 1.2347271621227265,
      "epoch": 0.8531131750945993,
      "grad_norm": 0.000308990478515625,
      "learning_rate": 1.801042301274927e-05,
      "logits/chosen": -3.205084209862984,
      "logits/rejected": -2.928795499158591,
      "logps/chosen": -93.73975834846496,
      "logps/rejected": -358.22800674438474,
      "loss": 8.303017580146843e-08,
      "mean_token_accuracy": 0.5511996760964394,
      "num_tokens": 4455322.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4866585440933704,
      "rewards/margins": 22.33163278102875,
      "rewards/rejected": -20.844974386692048,
      "step": 620
    },
    {
      "entropy": 1.2407884091138839,
      "epoch": 0.8668730650154799,
      "grad_norm": 4.76837158203125e-06,
      "learning_rate": 1.7913599861742475e-05,
      "logits/chosen": -3.196434341605011,
      "logits/rejected": -2.9346165308967778,
      "logps/chosen": -94.46012229919434,
      "logps/rejected": -360.2606533050537,
      "loss": 2.6523449037085812e-08,
      "mean_token_accuracy": 0.553185648098588,
      "num_tokens": 4524646.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5154777869582177,
      "rewards/margins": 22.38102035522461,
      "rewards/rejected": -20.865542578697205,
      "step": 630
    },
    {
      "entropy": 1.2456171631813049,
      "epoch": 0.8806329549363605,
      "grad_norm": 1.9371509552001953e-06,
      "learning_rate": 1.7814749849747906e-05,
      "logits/chosen": -3.2061220261144827,
      "logits/rejected": -2.933136296525797,
      "logps/chosen": -93.17085399627686,
      "logps/rejected": -345.74973487854004,
      "loss": 4.866203198616858e-07,
      "mean_token_accuracy": 0.5430413067340851,
      "num_tokens": 4591613.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4402503899065777,
      "rewards/margins": 21.325499212741853,
      "rewards/rejected": -19.885248792171478,
      "step": 640
    },
    {
      "entropy": 1.224543235450983,
      "epoch": 0.8943928448572411,
      "grad_norm": 3.203749656677246e-06,
      "learning_rate": 1.771389829460275e-05,
      "logits/chosen": -3.21605593581562,
      "logits/rejected": -2.937328653648714,
      "logps/chosen": -89.61981456279754,
      "logps/rejected": -357.4316173553467,
      "loss": 1.3461053640639876e-07,
      "mean_token_accuracy": 0.5541570287197828,
      "num_tokens": 4659715.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5124846341088414,
      "rewards/margins": 22.234719467163085,
      "rewards/rejected": -20.72223507165909,
      "step": 650
    },
    {
      "entropy": 1.2212258189916612,
      "epoch": 0.9081527347781218,
      "grad_norm": 0.00021266937255859375,
      "learning_rate": 1.7611071026786975e-05,
      "logits/chosen": -3.193745319470957,
      "logits/rejected": -2.9283068821586755,
      "logps/chosen": -94.82682394981384,
      "logps/rejected": -358.56394958496094,
      "loss": 1.0348767318646423e-07,
      "mean_token_accuracy": 0.5502128433436155,
      "num_tokens": 4726358.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2972642586566507,
      "rewards/margins": 22.062623143196106,
      "rewards/rejected": -20.765359020233156,
      "step": 660
    },
    {
      "entropy": 1.2308818057179451,
      "epoch": 0.9219126246990024,
      "grad_norm": 3.56137752532959e-06,
      "learning_rate": 1.7506294382807524e-05,
      "logits/chosen": -3.2158995908819286,
      "logits/rejected": -2.9366110423420593,
      "logps/chosen": -86.88749198913574,
      "logps/rejected": -353.19364166259766,
      "loss": 2.415685571577342e-08,
      "mean_token_accuracy": 0.5482585165649653,
      "num_tokens": 4795784.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7150141508318484,
      "rewards/margins": 22.53857171535492,
      "rewards/rejected": -20.823557567596435,
      "step": 670
    },
    {
      "entropy": 1.2435053884983063,
      "epoch": 0.935672514619883,
      "grad_norm": 2.7120113372802734e-06,
      "learning_rate": 1.7399595198452875e-05,
      "logits/chosen": -3.195924183348283,
      "logits/rejected": -2.937464334599706,
      "logps/chosen": -90.25515475273133,
      "logps/rejected": -365.6073051452637,
      "loss": 8.592822098307806e-09,
      "mean_token_accuracy": 0.557866370305419,
      "num_tokens": 4865396.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5782869830727577,
      "rewards/margins": 22.804828882217407,
      "rewards/rejected": -21.22654196023941,
      "step": 680
    },
    {
      "entropy": 1.2501905634999275,
      "epoch": 0.9494324045407637,
      "grad_norm": 9.655952453613281e-06,
      "learning_rate": 1.7291000801919807e-05,
      "logits/chosen": -3.2138292712700163,
      "logits/rejected": -2.93742158108878,
      "logps/chosen": -95.1417423248291,
      "logps/rejected": -363.6313949584961,
      "loss": 2.6834425170818577e-08,
      "mean_token_accuracy": 0.5553566936403513,
      "num_tokens": 4940187.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3729901384562253,
      "rewards/margins": 22.39214894771576,
      "rewards/rejected": -21.01915898323059,
      "step": 690
    },
    {
      "entropy": 1.2206627286970615,
      "epoch": 0.9631922944616443,
      "grad_norm": 1.633167266845703e-05,
      "learning_rate": 1.7180539006813973e-05,
      "logits/chosen": -3.2035463098229506,
      "logits/rejected": -2.933229209285195,
      "logps/chosen": -89.3526448249817,
      "logps/rejected": -358.9783546447754,
      "loss": 9.578674564636459e-09,
      "mean_token_accuracy": 0.5566618263721466,
      "num_tokens": 5007712.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5610000245273112,
      "rewards/margins": 22.329492390155792,
      "rewards/rejected": -20.768492436408998,
      "step": 700
    },
    {
      "epoch": 0.9631922944616443,
      "eval_entropy": 1.2349938719101201,
      "eval_logits/chosen": -3.2095329852412915,
      "eval_logits/rejected": -2.930840597158518,
      "eval_logps/chosen": -93.99583950541378,
      "eval_logps/rejected": -357.03034364787584,
      "eval_loss": 1.0149173590434657e-07,
      "eval_mean_token_accuracy": 0.5478390133848378,
      "eval_num_tokens": 5007712.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5820025781449538,
      "eval_rewards/margins": 22.257171013775995,
      "eval_rewards/rejected": -20.67516856100045,
      "eval_runtime": 49.0639,
      "eval_samples_per_second": 12.474,
      "eval_steps_per_second": 6.237,
      "step": 700
    },
    {
      "entropy": 1.2194250263273716,
      "epoch": 0.976952184382525,
      "grad_norm": 9.08970832824707e-07,
      "learning_rate": 1.7068238105026167e-05,
      "logits/chosen": -3.20297050334334,
      "logits/rejected": -2.9201588284997833,
      "logps/chosen": -92.99847078323364,
      "logps/rejected": -361.15418434143066,
      "loss": 1.1712397451901779e-08,
      "mean_token_accuracy": 0.5613618675619364,
      "num_tokens": 5080481.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.535900639742613,
      "rewards/margins": 22.68755226135254,
      "rewards/rejected": -21.15165193080902,
      "step": 710
    },
    {
      "entropy": 1.256634522974491,
      "epoch": 0.9907120743034056,
      "grad_norm": 1.1086463928222656e-05,
      "learning_rate": 1.6954126859486123e-05,
      "logits/chosen": -3.2114024947419955,
      "logits/rejected": -2.9574638812658067,
      "logps/chosen": -93.72816143035888,
      "logps/rejected": -361.8413818359375,
      "loss": 5.007818799640518e-08,
      "mean_token_accuracy": 0.5503214780241251,
      "num_tokens": 5148455.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.52496765749529,
      "rewards/margins": 22.30840722322464,
      "rewards/rejected": -20.783439564704896,
      "step": 720
    },
    {
      "entropy": 1.2511392663686702,
      "epoch": 1.0041279669762642,
      "grad_norm": 1.817941665649414e-06,
      "learning_rate": 1.6838234496795632e-05,
      "logits/chosen": -3.208696360325024,
      "logits/rejected": -2.938165258133346,
      "logps/chosen": -94.96354098197742,
      "logps/rejected": -359.0692009559044,
      "loss": 8.297415092783922e-08,
      "mean_token_accuracy": 0.5419222819499481,
      "num_tokens": 5217649.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5244833397177548,
      "rewards/margins": 22.175074552878357,
      "rewards/rejected": -20.650591312310635,
      "step": 730
    },
    {
      "entropy": 1.2317955523729325,
      "epoch": 1.0178878568971448,
      "grad_norm": 0.0002613067626953125,
      "learning_rate": 1.67205906997429e-05,
      "logits/chosen": -3.193782121863356,
      "logits/rejected": -2.939726684571892,
      "logps/chosen": -98.34705619812011,
      "logps/rejected": -357.925959777832,
      "loss": 8.716393722352222e-08,
      "mean_token_accuracy": 0.5531163465231657,
      "num_tokens": 5287632.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4051031542010606,
      "rewards/margins": 22.358167839050292,
      "rewards/rejected": -20.953064560890198,
      "step": 740
    },
    {
      "entropy": 1.2180696174502372,
      "epoch": 1.0316477468180254,
      "grad_norm": 9.47713851928711e-06,
      "learning_rate": 1.660122559970011e-05,
      "logits/chosen": -3.1999433298614353,
      "logits/rejected": -2.9382047283630732,
      "logps/chosen": -91.44652361869812,
      "logps/rejected": -355.18139419555666,
      "loss": 3.3117328257503684e-08,
      "mean_token_accuracy": 0.5552334699779748,
      "num_tokens": 5359755.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4298429565504194,
      "rewards/margins": 22.017473244667052,
      "rewards/rejected": -20.587630295753478,
      "step": 750
    },
    {
      "entropy": 1.2397649094462395,
      "epoch": 1.045407636738906,
      "grad_norm": 7.450580596923828e-06,
      "learning_rate": 1.6480169768906044e-05,
      "logits/chosen": -3.201849871038413,
      "logits/rejected": -2.942824107119408,
      "logps/chosen": -94.76110773086548,
      "logps/rejected": -354.3981412887573,
      "loss": 6.155430583021371e-08,
      "mean_token_accuracy": 0.5530436370521784,
      "num_tokens": 5430991.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3717019651085138,
      "rewards/margins": 21.781417500972747,
      "rewards/rejected": -20.40971568822861,
      "step": 760
    },
    {
      "entropy": 1.2326208755373955,
      "epoch": 1.0591675266597866,
      "grad_norm": 2.1457672119140625e-06,
      "learning_rate": 1.6357454212635798e-05,
      "logits/chosen": -3.1993549874995666,
      "logits/rejected": -2.9411404620494945,
      "logps/chosen": -92.11231818199158,
      "logps/rejected": -353.9612159729004,
      "loss": 3.313938350402168e-08,
      "mean_token_accuracy": 0.5549323987215757,
      "num_tokens": 5495381.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6085279668681323,
      "rewards/margins": 22.037530279159547,
      "rewards/rejected": -20.429002428054808,
      "step": 770
    },
    {
      "entropy": 1.2556634098291397,
      "epoch": 1.0729274165806673,
      "grad_norm": 1.996755599975586e-06,
      "learning_rate": 1.6233110361259616e-05,
      "logits/chosen": -3.2060114017402634,
      "logits/rejected": -2.9339854814430995,
      "logps/chosen": -95.85781788825989,
      "logps/rejected": -363.0917518615723,
      "loss": 1.4122252878223662e-08,
      "mean_token_accuracy": 0.5444140575826169,
      "num_tokens": 5565468.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5266890531405806,
      "rewards/margins": 22.333020043373107,
      "rewards/rejected": -20.806330823898314,
      "step": 780
    },
    {
      "entropy": 1.234340064227581,
      "epoch": 1.086687306501548,
      "grad_norm": 0.0002460479736328125,
      "learning_rate": 1.610717006219282e-05,
      "logits/chosen": -3.20874590791633,
      "logits/rejected": -2.9412810311356044,
      "logps/chosen": -92.43198065757751,
      "logps/rejected": -358.45581817626953,
      "loss": 1.5261637145158602e-07,
      "mean_token_accuracy": 0.5527918070554734,
      "num_tokens": 5639764.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5685145473107696,
      "rewards/margins": 22.137369656562804,
      "rewards/rejected": -20.568855047225952,
      "step": 790
    },
    {
      "entropy": 1.2221609637141229,
      "epoch": 1.1004471964224287,
      "grad_norm": 2.0116567611694336e-06,
      "learning_rate": 1.5979665571738908e-05,
      "logits/chosen": -3.2068134907404735,
      "logits/rejected": -2.93636597301207,
      "logps/chosen": -98.61505641937256,
      "logps/rejected": -354.62015495300295,
      "loss": 1.0114164297192474e-07,
      "mean_token_accuracy": 0.552211070433259,
      "num_tokens": 5713950.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5757936414331197,
      "rewards/margins": 22.034456276893614,
      "rewards/rejected": -20.4586629986763,
      "step": 800
    },
    {
      "epoch": 1.1004471964224287,
      "eval_entropy": 1.2343543432896433,
      "eval_logits/chosen": -3.209811952371513,
      "eval_logits/rejected": -2.930982134660661,
      "eval_logps/chosen": -93.95115639805014,
      "eval_logps/rejected": -357.3008830749911,
      "eval_loss": 1.0059742550083683e-07,
      "eval_mean_token_accuracy": 0.5480056442076864,
      "eval_num_tokens": 5713950.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5864708358604533,
      "eval_rewards/margins": 22.288693184946098,
      "eval_rewards/rejected": -20.702222353492687,
      "eval_runtime": 49.896,
      "eval_samples_per_second": 12.266,
      "eval_steps_per_second": 6.133,
      "step": 800
    },
    {
      "entropy": 1.2298845812678336,
      "epoch": 1.1142070863433093,
      "grad_norm": 2.60770320892334e-06,
      "learning_rate": 1.5850629546827977e-05,
      "logits/chosen": -3.2075300177973256,
      "logits/rejected": -2.9281061202537826,
      "logps/chosen": -90.7022216796875,
      "logps/rejected": -358.142036819458,
      "loss": 2.4776889517852398e-08,
      "mean_token_accuracy": 0.5414935078471899,
      "num_tokens": 5783814.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.41559614893049,
      "rewards/margins": 22.017850613594057,
      "rewards/rejected": -20.602254581451415,
      "step": 810
    },
    {
      "entropy": 1.2270297072827816,
      "epoch": 1.12796697626419,
      "grad_norm": 1.4543533325195312e-05,
      "learning_rate": 1.5720095036652486e-05,
      "logits/chosen": -3.20386469479259,
      "logits/rejected": -2.933272782764143,
      "logps/chosen": -89.29530630111694,
      "logps/rejected": -357.4134010314941,
      "loss": 1.849425927957782e-08,
      "mean_token_accuracy": 0.5607247911393642,
      "num_tokens": 5852617.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.58894761223346,
      "rewards/margins": 22.511793541908265,
      "rewards/rejected": -20.922845947742463,
      "step": 820
    },
    {
      "entropy": 1.2241738960146904,
      "epoch": 1.1417268661850706,
      "grad_norm": 7.599592208862305e-07,
      "learning_rate": 1.5588095474202597e-05,
      "logits/chosen": -3.2111792369422907,
      "logits/rejected": -2.939014422286019,
      "logps/chosen": -96.20790944099426,
      "logps/rejected": -361.97329750061033,
      "loss": 8.978912546808715e-08,
      "mean_token_accuracy": 0.5441163565963507,
      "num_tokens": 5922241.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.36717337500304,
      "rewards/margins": 22.38765777349472,
      "rewards/rejected": -21.020484483242036,
      "step": 830
    },
    {
      "entropy": 1.2215713784098625,
      "epoch": 1.1554867561059512,
      "grad_norm": 2.207234501838684e-07,
      "learning_rate": 1.5454664667703148e-05,
      "logits/chosen": -3.207690730104448,
      "logits/rejected": -2.9357031268257865,
      "logps/chosen": -91.12561550140381,
      "logps/rejected": -355.57129364013673,
      "loss": 2.337985023359579e-08,
      "mean_token_accuracy": 0.5488512497395277,
      "num_tokens": 5985778.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4972319712862372,
      "rewards/margins": 22.422241616249085,
      "rewards/rejected": -20.92500967979431,
      "step": 840
    },
    {
      "entropy": 1.258409896492958,
      "epoch": 1.1692466460268318,
      "grad_norm": 5.550682544708252e-07,
      "learning_rate": 1.531983679195462e-05,
      "logits/chosen": -3.226955490438931,
      "logits/rejected": -2.945719325954898,
      "logps/chosen": -95.44598877429962,
      "logps/rejected": -366.71283493041994,
      "loss": 4.6230468342400854e-09,
      "mean_token_accuracy": 0.550101426243782,
      "num_tokens": 6064389.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.780148933455348,
      "rewards/margins": 22.78461835384369,
      "rewards/rejected": -21.004469299316405,
      "step": 850
    },
    {
      "entropy": 1.2130897045135498,
      "epoch": 1.1830065359477124,
      "grad_norm": 1.233816146850586e-05,
      "learning_rate": 1.5183646379580108e-05,
      "logits/chosen": -3.2055889715400085,
      "logits/rejected": -2.930225354490301,
      "logps/chosen": -86.66465277671814,
      "logps/rejected": -355.93296508789064,
      "loss": 3.668304771053954e-08,
      "mean_token_accuracy": 0.5639011237770319,
      "num_tokens": 6136897.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5607147116214037,
      "rewards/margins": 22.342380928993226,
      "rewards/rejected": -20.781666159629822,
      "step": 860
    },
    {
      "entropy": 1.2411843314766884,
      "epoch": 1.196766425868593,
      "grad_norm": 0.00017261505126953125,
      "learning_rate": 1.5046128312180735e-05,
      "logits/chosen": -3.2021021642991614,
      "logits/rejected": -2.9457325465225077,
      "logps/chosen": -95.11519818305969,
      "logps/rejected": -357.90135345458987,
      "loss": 1.1841185596495051e-07,
      "mean_token_accuracy": 0.5472324322909117,
      "num_tokens": 6206206.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3694644190371037,
      "rewards/margins": 21.965344977378845,
      "rewards/rejected": -20.595880722999574,
      "step": 870
    },
    {
      "entropy": 1.234082168340683,
      "epoch": 1.2105263157894737,
      "grad_norm": 1.0207295417785645e-06,
      "learning_rate": 1.4907317811401609e-05,
      "logits/chosen": -3.2189416776847666,
      "logits/rejected": -2.9338163368790253,
      "logps/chosen": -91.24591059684754,
      "logps/rejected": -364.68142623901366,
      "loss": 1.026212856913844e-08,
      "mean_token_accuracy": 0.5534303054213524,
      "num_tokens": 6279994.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7311106204986573,
      "rewards/margins": 22.925514221191406,
      "rewards/rejected": -21.194403612613677,
      "step": 880
    },
    {
      "entropy": 1.2325060382485389,
      "epoch": 1.2242862057103543,
      "grad_norm": 8.67992639541626e-07,
      "learning_rate": 1.4767250429910754e-05,
      "logits/chosen": -3.210597847276174,
      "logits/rejected": -2.9429847990942415,
      "logps/chosen": -88.37438547611237,
      "logps/rejected": -357.3102340698242,
      "loss": 1.2495095802478317e-08,
      "mean_token_accuracy": 0.5496666703373194,
      "num_tokens": 6353089.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4847000421956182,
      "rewards/margins": 22.177469778060914,
      "rewards/rejected": -20.692769980430604,
      "step": 890
    },
    {
      "entropy": 1.2352219231426715,
      "epoch": 1.238046095631235,
      "grad_norm": 4.696846008300781e-05,
      "learning_rate": 1.4625962042293212e-05,
      "logits/chosen": -3.200865099898492,
      "logits/rejected": -2.9249128824488015,
      "logps/chosen": -97.9710922241211,
      "logps/rejected": -356.9550182342529,
      "loss": 4.1527565031174164e-08,
      "mean_token_accuracy": 0.54386255890131,
      "num_tokens": 6423626.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4338832331821323,
      "rewards/margins": 22.15778750181198,
      "rewards/rejected": -20.72390434741974,
      "step": 900
    },
    {
      "epoch": 1.238046095631235,
      "eval_entropy": 1.2343237298765992,
      "eval_logits/chosen": -3.2097772658901613,
      "eval_logits/rejected": -2.930761755546573,
      "eval_logps/chosen": -93.91436023338169,
      "eval_logps/rejected": -357.41727881338085,
      "eval_loss": 9.47132434703235e-08,
      "eval_mean_token_accuracy": 0.5485258311812394,
      "eval_num_tokens": 6423626.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.590150504939112,
      "eval_rewards/margins": 22.30401264764125,
      "eval_rewards/rejected": -20.713862163568635,
      "eval_runtime": 51.361,
      "eval_samples_per_second": 11.916,
      "eval_steps_per_second": 5.958,
      "step": 900
    },
    {
      "entropy": 1.2400694966316224,
      "epoch": 1.2518059855521155,
      "grad_norm": 2.7120113372802734e-06,
      "learning_rate": 1.44834888358627e-05,
      "logits/chosen": -3.218890531500263,
      "logits/rejected": -2.9362049726909754,
      "logps/chosen": -89.21487245559692,
      "logps/rejected": -356.256791305542,
      "loss": 2.0597786942744277e-08,
      "mean_token_accuracy": 0.5380889017134904,
      "num_tokens": 6490312.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4560846026055514,
      "rewards/margins": 22.216789221763612,
      "rewards/rejected": -20.76070455312729,
      "step": 910
    },
    {
      "entropy": 1.2324326142668725,
      "epoch": 1.2655658754729961,
      "grad_norm": 5.900859832763672e-06,
      "learning_rate": 1.4339867301393204e-05,
      "logits/chosen": -3.2058008679050474,
      "logits/rejected": -2.9334829204994004,
      "logps/chosen": -90.68490617275238,
      "logps/rejected": -352.11963920593263,
      "loss": 3.130011236862629e-08,
      "mean_token_accuracy": 0.5447090461850166,
      "num_tokens": 6557228.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4577140986919404,
      "rewards/margins": 22.14238922595978,
      "rewards/rejected": -20.68467507362366,
      "step": 920
    },
    {
      "entropy": 1.236050571501255,
      "epoch": 1.2793257653938768,
      "grad_norm": 2.428889274597168e-06,
      "learning_rate": 1.41951342237728e-05,
      "logits/chosen": -3.2088376395139306,
      "logits/rejected": -2.9461211800539613,
      "logps/chosen": -94.4995069026947,
      "logps/rejected": -360.5341571807861,
      "loss": 1.521080861266455e-08,
      "mean_token_accuracy": 0.5474192447960377,
      "num_tokens": 6628952.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4530406618490814,
      "rewards/margins": 22.35727779865265,
      "rewards/rejected": -20.9042368888855,
      "step": 930
    },
    {
      "entropy": 1.2395083367824555,
      "epoch": 1.2930856553147576,
      "grad_norm": 0.0003032684326171875,
      "learning_rate": 1.4049326672582186e-05,
      "logits/chosen": -3.2151515899768377,
      "logits/rejected": -2.9367740687639,
      "logps/chosen": -90.78844072818757,
      "logps/rejected": -352.5623176574707,
      "loss": 9.801140095078154e-08,
      "mean_token_accuracy": 0.5518607906997204,
      "num_tokens": 6694115.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5609057364985346,
      "rewards/margins": 22.212734603881835,
      "rewards/rejected": -20.65182896852493,
      "step": 940
    },
    {
      "entropy": 1.2343350157141686,
      "epoch": 1.306845545235638,
      "grad_norm": 4.4345855712890625e-05,
      "learning_rate": 1.390248199260027e-05,
      "logits/chosen": -3.2039629217549477,
      "logits/rejected": -2.9385683242624454,
      "logps/chosen": -93.59937400817871,
      "logps/rejected": -358.15326766967775,
      "loss": 3.500546768009371e-08,
      "mean_token_accuracy": 0.5573053862899542,
      "num_tokens": 6766186.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5620607184246182,
      "rewards/margins": 22.00651559829712,
      "rewards/rejected": -20.44445481300354,
      "step": 950
    },
    {
      "entropy": 1.2210247091948987,
      "epoch": 1.3206054351565188,
      "grad_norm": 3.501772880554199e-07,
      "learning_rate": 1.3754637794239303e-05,
      "logits/chosen": -3.1871140209822464,
      "logits/rejected": -2.9242806765491824,
      "logps/chosen": -92.62435631752014,
      "logps/rejected": -361.9002456665039,
      "loss": 2.329596213712648e-08,
      "mean_token_accuracy": 0.5678409364074468,
      "num_tokens": 6833190.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4230625696480275,
      "rewards/margins": 22.756867027282716,
      "rewards/rejected": -21.333804500102996,
      "step": 960
    },
    {
      "entropy": 1.2278488397598266,
      "epoch": 1.3343653250773992,
      "grad_norm": 1.1444091796875e-05,
      "learning_rate": 1.3605831943911923e-05,
      "logits/chosen": -3.20925780292652,
      "logits/rejected": -2.948473146985262,
      "logps/chosen": -95.62888393402099,
      "logps/rejected": -354.1160026550293,
      "loss": 1.2392845860631496e-08,
      "mean_token_accuracy": 0.5485764220356941,
      "num_tokens": 6900474.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.473580021597445,
      "rewards/margins": 22.098388063907624,
      "rewards/rejected": -20.624807834625244,
      "step": 970
    },
    {
      "entropy": 1.2570519968867302,
      "epoch": 1.34812521499828,
      "grad_norm": 1.7136335372924805e-06,
      "learning_rate": 1.3456102554332717e-05,
      "logits/chosen": -3.2114092807728825,
      "logits/rejected": -2.9454987107403863,
      "logps/chosen": -99.15215730667114,
      "logps/rejected": -366.17869873046874,
      "loss": 7.894200848568289e-08,
      "mean_token_accuracy": 0.5519696664065122,
      "num_tokens": 6978618.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4953111509792507,
      "rewards/margins": 22.495365333557128,
      "rewards/rejected": -21.000054240226746,
      "step": 980
    },
    {
      "entropy": 1.239316475391388,
      "epoch": 1.3618851049191607,
      "grad_norm": 2.491474151611328e-05,
      "learning_rate": 1.3305487974756601e-05,
      "logits/chosen": -3.2142083873833514,
      "logits/rejected": -2.9419815492384034,
      "logps/chosen": -96.75212671756745,
      "logps/rejected": -360.60957794189454,
      "loss": 6.680644446532824e-08,
      "mean_token_accuracy": 0.551985003426671,
      "num_tokens": 7054612.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4342084027826787,
      "rewards/margins": 22.23996684551239,
      "rewards/rejected": -20.805758452415468,
      "step": 990
    },
    {
      "entropy": 1.239124795794487,
      "epoch": 1.3756449948400413,
      "grad_norm": 1.9222497940063477e-06,
      "learning_rate": 1.3154026781156697e-05,
      "logits/chosen": -3.20129278083637,
      "logits/rejected": -2.9428580494057384,
      "logps/chosen": -93.50243964195252,
      "logps/rejected": -360.9033599853516,
      "loss": 7.131031338758476e-08,
      "mean_token_accuracy": 0.5573917739093304,
      "num_tokens": 7126656.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5027323237620294,
      "rewards/margins": 22.239959049224854,
      "rewards/rejected": -20.73722686767578,
      "step": 1000
    },
    {
      "epoch": 1.3756449948400413,
      "eval_entropy": 1.2343096980472017,
      "eval_logits/chosen": -3.2097842934074605,
      "eval_logits/rejected": -2.930549885680492,
      "eval_logps/chosen": -93.8916666492138,
      "eval_logps/rejected": -357.49578269011056,
      "eval_loss": 9.931152789022235e-08,
      "eval_mean_token_accuracy": 0.5481461427764955,
      "eval_num_tokens": 7126656.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5924198131497,
      "eval_rewards/margins": 22.314132154377457,
      "eval_rewards/rejected": -20.721712392919205,
      "eval_runtime": 52.097,
      "eval_samples_per_second": 11.747,
      "eval_steps_per_second": 5.874,
      "step": 1000
    },
    {
      "entropy": 1.2653382867574692,
      "epoch": 1.389404884760922,
      "grad_norm": 2.1010637283325195e-06,
      "learning_rate": 1.300175776634409e-05,
      "logits/chosen": -3.2129485356574863,
      "logits/rejected": -2.954222626255789,
      "logps/chosen": -92.84323039054871,
      "logps/rejected": -361.7887050628662,
      "loss": 2.7653899792312586e-08,
      "mean_token_accuracy": 0.5468194488435983,
      "num_tokens": 7201540.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3965744260232895,
      "rewards/margins": 21.90877208709717,
      "rewards/rejected": -20.512197828292848,
      "step": 1010
    },
    {
      "entropy": 1.2396557286381722,
      "epoch": 1.4031647746818026,
      "grad_norm": 3.9637088775634766e-06,
      "learning_rate": 1.2848719930032083e-05,
      "logits/chosen": -3.194560362905095,
      "logits/rejected": -2.9334940937288825,
      "logps/chosen": -89.86825790405274,
      "logps/rejected": -355.7735252380371,
      "loss": 2.492495525530103e-08,
      "mean_token_accuracy": 0.5589596729725599,
      "num_tokens": 7274153.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.48170037958771,
      "rewards/margins": 22.06132137775421,
      "rewards/rejected": -20.57962087392807,
      "step": 1020
    },
    {
      "entropy": 1.2463233694434166,
      "epoch": 1.4169246646026832,
      "grad_norm": 9.238719940185547e-06,
      "learning_rate": 1.269495246884743e-05,
      "logits/chosen": -3.2081193037821327,
      "logits/rejected": -2.9385929283645895,
      "logps/chosen": -90.74200019836425,
      "logps/rejected": -359.9956394195557,
      "loss": 3.979102700668591e-08,
      "mean_token_accuracy": 0.5487319093197585,
      "num_tokens": 7344517.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6196894589811563,
      "rewards/margins": 22.411945462226868,
      "rewards/rejected": -20.792256021499632,
      "step": 1030
    },
    {
      "entropy": 1.2206041820347309,
      "epoch": 1.4306845545235638,
      "grad_norm": 1.8835067749023438e-05,
      "learning_rate": 1.2540494766291151e-05,
      "logits/chosen": -3.2122664430868433,
      "logits/rejected": -2.940406184250665,
      "logps/chosen": -95.85816128253937,
      "logps/rejected": -356.52550678253175,
      "loss": 2.728211256908253e-08,
      "mean_token_accuracy": 0.5568196009844542,
      "num_tokens": 7416599.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.451601443067193,
      "rewards/margins": 22.1426992893219,
      "rewards/rejected": -20.69109789133072,
      "step": 1040
    },
    {
      "entropy": 1.2360372744500636,
      "epoch": 1.4444444444444444,
      "grad_norm": 1.436471939086914e-05,
      "learning_rate": 1.2385386382651503e-05,
      "logits/chosen": -3.2093854919794973,
      "logits/rejected": -2.9219334413945957,
      "logps/chosen": -96.11689052581787,
      "logps/rejected": -367.30119705200195,
      "loss": 7.263046342131929e-09,
      "mean_token_accuracy": 0.5502486415207386,
      "num_tokens": 7487006.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4450448545627297,
      "rewards/margins": 22.97078468799591,
      "rewards/rejected": -21.52573983669281,
      "step": 1050
    },
    {
      "entropy": 1.2560921475291251,
      "epoch": 1.458204334365325,
      "grad_norm": 5.21540641784668e-06,
      "learning_rate": 1.2229667044871603e-05,
      "logits/chosen": -3.2238342267568925,
      "logits/rejected": -2.954690942822835,
      "logps/chosen": -95.17089319229126,
      "logps/rejected": -358.49853858947756,
      "loss": 1.2689355344264186e-07,
      "mean_token_accuracy": 0.5482176937162876,
      "num_tokens": 7567622.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6461190782487392,
      "rewards/margins": 22.06815986633301,
      "rewards/rejected": -20.422040891647338,
      "step": 1060
    },
    {
      "entropy": 1.2410757690668106,
      "epoch": 1.4719642242862057,
      "grad_norm": 2.3603439331054688e-05,
      "learning_rate": 1.2073376636374467e-05,
      "logits/chosen": -3.2181867929946835,
      "logits/rejected": -2.9455415466787125,
      "logps/chosen": -95.13691272735596,
      "logps/rejected": -360.3422981262207,
      "loss": 6.41182055005629e-08,
      "mean_token_accuracy": 0.5514874663203955,
      "num_tokens": 7640270.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5654873698949814,
      "rewards/margins": 22.5183806180954,
      "rewards/rejected": -20.95289328098297,
      "step": 1070
    },
    {
      "entropy": 1.223815706372261,
      "epoch": 1.4857241142070863,
      "grad_norm": 7.562339305877686e-07,
      "learning_rate": 1.191655518684788e-05,
      "logits/chosen": -3.213393593248744,
      "logits/rejected": -2.9516763163029864,
      "logps/chosen": -85.92939200401307,
      "logps/rejected": -354.56640129089357,
      "loss": 5.5562270517839354e-08,
      "mean_token_accuracy": 0.5542758256196976,
      "num_tokens": 7714968.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.601221213489771,
      "rewards/margins": 22.047427451610567,
      "rewards/rejected": -20.446206474304198,
      "step": 1080
    },
    {
      "entropy": 1.2295074120163918,
      "epoch": 1.499484004127967,
      "grad_norm": 2.652406692504883e-06,
      "learning_rate": 1.1759242861991855e-05,
      "logits/chosen": -3.208135379165089,
      "logits/rejected": -2.9249312915782903,
      "logps/chosen": -91.00948677062988,
      "logps/rejected": -348.90424194335935,
      "loss": 7.343320476138616e-08,
      "mean_token_accuracy": 0.5440276902168989,
      "num_tokens": 7783981.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6290188884362578,
      "rewards/margins": 21.78258399963379,
      "rewards/rejected": -20.1535649061203,
      "step": 1090
    },
    {
      "entropy": 1.2454005569219588,
      "epoch": 1.5132438940488475,
      "grad_norm": 4.351139068603516e-06,
      "learning_rate": 1.1601479953231251e-05,
      "logits/chosen": -3.2080805895734223,
      "logits/rejected": -2.940618116061205,
      "logps/chosen": -95.01053881645203,
      "logps/rejected": -350.12212295532225,
      "loss": 2.5034836426129914e-08,
      "mean_token_accuracy": 0.5546459078788757,
      "num_tokens": 7847253.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6118897259235383,
      "rewards/margins": 22.07096242904663,
      "rewards/rejected": -20.459072756767274,
      "step": 1100
    },
    {
      "epoch": 1.5132438940488475,
      "eval_entropy": 1.2339863264872357,
      "eval_logits/chosen": -3.2096496999814828,
      "eval_logits/rejected": -2.930534343593307,
      "eval_logps/chosen": -93.90245786679337,
      "eval_logps/rejected": -357.6898516486673,
      "eval_loss": 9.994760574727479e-08,
      "eval_mean_token_accuracy": 0.5476265988903108,
      "eval_num_tokens": 7847253.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5913407134707847,
      "eval_rewards/margins": 22.332460110483606,
      "eval_rewards/rejected": -20.741119384765625,
      "eval_runtime": 53.2502,
      "eval_samples_per_second": 11.493,
      "eval_steps_per_second": 5.746,
      "step": 1100
    },
    {
      "entropy": 1.2328468561172485,
      "epoch": 1.5270037839697284,
      "grad_norm": 3.874301910400391e-06,
      "learning_rate": 1.1443306867396164e-05,
      "logits/chosen": -3.2139558810032733,
      "logits/rejected": -2.936539280420114,
      "logps/chosen": -87.71727166175842,
      "logps/rejected": -350.89677238464355,
      "loss": 3.940974124816421e-08,
      "mean_token_accuracy": 0.5458600226789713,
      "num_tokens": 7914853.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.639497240446508,
      "rewards/margins": 21.940470695495605,
      "rewards/rejected": -20.300973522663117,
      "step": 1110
    },
    {
      "entropy": 1.2204807847738266,
      "epoch": 1.5407636738906088,
      "grad_norm": 1.049041748046875e-05,
      "learning_rate": 1.1284764116372788e-05,
      "logits/chosen": -3.1927449778795194,
      "logits/rejected": -2.930753806827826,
      "logps/chosen": -89.22110188007355,
      "logps/rejected": -357.37607154846194,
      "loss": 4.953578809363535e-08,
      "mean_token_accuracy": 0.5495975099503994,
      "num_tokens": 7984117.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3059963096864522,
      "rewards/margins": 22.082072138786316,
      "rewards/rejected": -20.776075863838194,
      "step": 1120
    },
    {
      "entropy": 1.2255059480667114,
      "epoch": 1.5545235638114896,
      "grad_norm": 1.5273690223693848e-07,
      "learning_rate": 1.1125892306727362e-05,
      "logits/chosen": -3.213309220424828,
      "logits/rejected": -2.9230614100860066,
      "logps/chosen": -99.92382683753968,
      "logps/rejected": -364.37585296630857,
      "loss": 2.4999820880111656e-08,
      "mean_token_accuracy": 0.5512117072939873,
      "num_tokens": 8055365.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5203925285488367,
      "rewards/margins": 22.711718726158143,
      "rewards/rejected": -21.191326379776,
      "step": 1130
    },
    {
      "entropy": 1.2493881568312646,
      "epoch": 1.56828345373237,
      "grad_norm": 2.6345252990722656e-05,
      "learning_rate": 1.096673212930585e-05,
      "logits/chosen": -3.216281187013641,
      "logits/rejected": -2.943805298211413,
      "logps/chosen": -89.30674161911011,
      "logps/rejected": -357.40013275146487,
      "loss": 5.5321879699477e-08,
      "mean_token_accuracy": 0.5533328153192997,
      "num_tokens": 8128635.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4588562190067023,
      "rewards/margins": 21.81429543495178,
      "rewards/rejected": -20.35543934106827,
      "step": 1140
    },
    {
      "entropy": 1.2265586256980896,
      "epoch": 1.5820433436532508,
      "grad_norm": 6.668269634246826e-07,
      "learning_rate": 1.0807324348812065e-05,
      "logits/chosen": -3.1953608613402964,
      "logits/rejected": -2.931393446827495,
      "logps/chosen": -91.45233364105225,
      "logps/rejected": -355.4373069763184,
      "loss": 1.3633338369345438e-08,
      "mean_token_accuracy": 0.5539218068122864,
      "num_tokens": 8193593.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4279992308467626,
      "rewards/margins": 22.249031281471254,
      "rewards/rejected": -20.821032071113585,
      "step": 1150
    },
    {
      "entropy": 1.2320809975266456,
      "epoch": 1.5958032335741315,
      "grad_norm": 2.944469451904297e-05,
      "learning_rate": 1.0647709793366843e-05,
      "logits/chosen": -3.1978028499428817,
      "logits/rejected": -2.924464054780402,
      "logps/chosen": -91.58377666473389,
      "logps/rejected": -357.0462505340576,
      "loss": 1.6318038831286684e-08,
      "mean_token_accuracy": 0.5578478880226612,
      "num_tokens": 8262660.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5524121232330799,
      "rewards/margins": 22.37659764289856,
      "rewards/rejected": -20.824185419082642,
      "step": 1160
    },
    {
      "entropy": 1.2370183646678925,
      "epoch": 1.609563123495012,
      "grad_norm": 2.467632293701172e-05,
      "learning_rate": 1.0487929344051005e-05,
      "logits/chosen": -3.2050213860609924,
      "logits/rejected": -2.9398967008651766,
      "logps/chosen": -90.65321626663209,
      "logps/rejected": -364.4670093536377,
      "loss": 1.9538390461093513e-08,
      "mean_token_accuracy": 0.5525426868349314,
      "num_tokens": 8337583.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4652544744312763,
      "rewards/margins": 22.466693067550658,
      "rewards/rejected": -21.001438570022582,
      "step": 1170
    },
    {
      "entropy": 1.2341183677315712,
      "epoch": 1.6233230134158927,
      "grad_norm": 6.67572021484375e-06,
      "learning_rate": 1.0328023924434731e-05,
      "logits/chosen": -3.222561275162723,
      "logits/rejected": -2.9361925600402694,
      "logps/chosen": -94.25461287498474,
      "logps/rejected": -365.5099418640137,
      "loss": 8.563840765418718e-08,
      "mean_token_accuracy": 0.5535896979272366,
      "num_tokens": 8408914.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.589152716472745,
      "rewards/margins": 22.772674751281738,
      "rewards/rejected": -21.18352210521698,
      "step": 1180
    },
    {
      "entropy": 1.2446356549859048,
      "epoch": 1.6370829033367733,
      "grad_norm": 4.2282044887542725e-07,
      "learning_rate": 1.0168034490096068e-05,
      "logits/chosen": -3.2167415967969895,
      "logits/rejected": -2.9377228327041256,
      "logps/chosen": -91.5360399723053,
      "logps/rejected": -365.2527450561523,
      "loss": 3.7552769072135564e-08,
      "mean_token_accuracy": 0.5535377282649279,
      "num_tokens": 8477253.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6271376270800828,
      "rewards/margins": 22.49522852897644,
      "rewards/rejected": -20.868090736865998,
      "step": 1190
    },
    {
      "entropy": 1.238689461350441,
      "epoch": 1.650842793257654,
      "grad_norm": 4.1425228118896484e-06,
      "learning_rate": 1.0008002018131222e-05,
      "logits/chosen": -3.2078820613389305,
      "logits/rejected": -2.953614994023544,
      "logps/chosen": -92.05451779365539,
      "logps/rejected": -362.38125648498533,
      "loss": 6.920008672750555e-08,
      "mean_token_accuracy": 0.55484758913517,
      "num_tokens": 8553796.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4701514573767782,
      "rewards/margins": 22.34501588344574,
      "rewards/rejected": -20.874864387512208,
      "step": 1200
    },
    {
      "epoch": 1.650842793257654,
      "eval_entropy": 1.2338554866952833,
      "eval_logits/chosen": -3.2098196420074143,
      "eval_logits/rejected": -2.930539210739882,
      "eval_logps/chosen": -93.88720486834158,
      "eval_logps/rejected": -357.73581810246884,
      "eval_loss": 9.633028241751163e-08,
      "eval_mean_token_accuracy": 0.5478904262866849,
      "eval_num_tokens": 8553796.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5928660004819726,
      "eval_rewards/margins": 22.33858197654774,
      "eval_rewards/rejected": -20.745715982773724,
      "eval_runtime": 53.4054,
      "eval_samples_per_second": 11.46,
      "eval_steps_per_second": 5.73,
      "step": 1200
    },
    {
      "entropy": 1.2338899701833725,
      "epoch": 1.6646026831785345,
      "grad_norm": 0.00341796875,
      "learning_rate": 9.84796749665937e-06,
      "logits/chosen": -3.2209590147273546,
      "logits/rejected": -2.926932008123792,
      "logps/chosen": -92.47018933296204,
      "logps/rejected": -364.83294715881345,
      "loss": 4.6182253754523117e-07,
      "mean_token_accuracy": 0.550224631279707,
      "num_tokens": 8627941.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.383915454056114,
      "rewards/margins": 22.66803867816925,
      "rewards/rejected": -21.28412308692932,
      "step": 1210
    },
    {
      "entropy": 1.219579754024744,
      "epoch": 1.6783625730994152,
      "grad_norm": 3.5762786865234375e-05,
      "learning_rate": 9.687971914324607e-06,
      "logits/chosen": -3.208296211567128,
      "logits/rejected": -2.918006915466585,
      "logps/chosen": -92.74604415893555,
      "logps/rejected": -358.02383651733396,
      "loss": 1.0100798135681543e-07,
      "mean_token_accuracy": 0.5535013288259506,
      "num_tokens": 8699432.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5753983343020082,
      "rewards/margins": 22.859029364585876,
      "rewards/rejected": -21.283631086349487,
      "step": 1220
    },
    {
      "entropy": 1.2542573563754558,
      "epoch": 1.6921224630202958,
      "grad_norm": 6.288290023803711e-06,
      "learning_rate": 9.528056249797801e-06,
      "logits/chosen": -3.204274880340815,
      "logits/rejected": -2.9406342860725028,
      "logps/chosen": -97.6590735912323,
      "logps/rejected": -361.1593727111816,
      "loss": 1.0381215531651833e-08,
      "mean_token_accuracy": 0.5510105274617672,
      "num_tokens": 8770988.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5584924587979914,
      "rewards/margins": 22.114416193962096,
      "rewards/rejected": -20.555923748016358,
      "step": 1230
    },
    {
      "entropy": 1.2104025840759278,
      "epoch": 1.7058823529411766,
      "grad_norm": 6.079673767089844e-06,
      "learning_rate": 9.36826146128095e-06,
      "logits/chosen": -3.194136407385158,
      "logits/rejected": -2.92752970132409,
      "logps/chosen": -89.4238208770752,
      "logps/rejected": -367.14545822143555,
      "loss": 8.342782820136563e-09,
      "mean_token_accuracy": 0.5553437374532223,
      "num_tokens": 8839680.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4051247568801046,
      "rewards/margins": 23.100000309944154,
      "rewards/rejected": -21.694875502586363,
      "step": 1240
    },
    {
      "entropy": 1.2199999384582043,
      "epoch": 1.719642242862057,
      "grad_norm": 1.4156103134155273e-06,
      "learning_rate": 9.208628476016851e-06,
      "logits/chosen": -3.2222275132636247,
      "logits/rejected": -2.9386455534159923,
      "logps/chosen": -88.96111073493958,
      "logps/rejected": -359.295943069458,
      "loss": 4.11268274547183e-08,
      "mean_token_accuracy": 0.559587674215436,
      "num_tokens": 8914608.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5049045903608202,
      "rewards/margins": 22.171202778816223,
      "rewards/rejected": -20.666298282146453,
      "step": 1250
    },
    {
      "entropy": 1.2306214161217213,
      "epoch": 1.7334021327829379,
      "grad_norm": 2.905726432800293e-06,
      "learning_rate": 9.049198179806639e-06,
      "logits/chosen": -3.202567711907225,
      "logits/rejected": -2.9338417581047036,
      "logps/chosen": -92.89101815223694,
      "logps/rejected": -363.7858856201172,
      "loss": 1.9706928355844867e-08,
      "mean_token_accuracy": 0.5425420433282853,
      "num_tokens": 8988203.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5503723615780474,
      "rewards/margins": 22.790516090393066,
      "rewards/rejected": -21.240143823623658,
      "step": 1260
    },
    {
      "entropy": 1.2146451964974403,
      "epoch": 1.7471620227038183,
      "grad_norm": 3.0040740966796875e-05,
      "learning_rate": 8.890011406537962e-06,
      "logits/chosen": -3.1990021274553615,
      "logits/rejected": -2.9203509087743083,
      "logps/chosen": -98.18809704780578,
      "logps/rejected": -361.4477626800537,
      "loss": 4.6641707740491255e-08,
      "mean_token_accuracy": 0.5551731705665588,
      "num_tokens": 9061240.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2511278545483946,
      "rewards/margins": 22.257884681224823,
      "rewards/rejected": -21.00675710439682,
      "step": 1270
    },
    {
      "entropy": 1.24142100289464,
      "epoch": 1.760921912624699,
      "grad_norm": 5.692243576049805e-06,
      "learning_rate": 8.731108927726456e-06,
      "logits/chosen": -3.207833236413998,
      "logits/rejected": -2.946940461163376,
      "logps/chosen": -94.46060605049134,
      "logps/rejected": -364.594189453125,
      "loss": 2.9929975653431026e-08,
      "mean_token_accuracy": 0.547786720469594,
      "num_tokens": 9137771.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4769228994846344,
      "rewards/margins": 22.50814254283905,
      "rewards/rejected": -21.031219601631165,
      "step": 1280
    },
    {
      "entropy": 1.2370137855410577,
      "epoch": 1.7746818025455795,
      "grad_norm": 9.015202522277832e-07,
      "learning_rate": 8.57253144207319e-06,
      "logits/chosen": -3.2113461060224298,
      "logits/rejected": -2.947444498776847,
      "logps/chosen": -92.91609952449798,
      "logps/rejected": -361.47128639221194,
      "loss": 2.0866957584075864e-08,
      "mean_token_accuracy": 0.5487309109419585,
      "num_tokens": 9205311.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4436738032847642,
      "rewards/margins": 22.531071925163268,
      "rewards/rejected": -21.08739824295044,
      "step": 1290
    },
    {
      "entropy": 1.2511209398508072,
      "epoch": 1.7884416924664603,
      "grad_norm": 1.5944242477416992e-06,
      "learning_rate": 8.414319565040767e-06,
      "logits/chosen": -3.2049623372893365,
      "logits/rejected": -2.939238119128755,
      "logps/chosen": -90.87593531608582,
      "logps/rejected": -362.4408033370972,
      "loss": 1.4193443576004939e-08,
      "mean_token_accuracy": 0.550374123826623,
      "num_tokens": 9276318.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4517353810369968,
      "rewards/margins": 22.48921357393265,
      "rewards/rejected": -21.037478232383727,
      "step": 1300
    },
    {
      "epoch": 1.7884416924664603,
      "eval_entropy": 1.231158983863257,
      "eval_logits/chosen": -3.2081173073045157,
      "eval_logits/rejected": -2.9273825382166865,
      "eval_logps/chosen": -94.19876768074784,
      "eval_logps/rejected": -359.8468223023259,
      "eval_loss": 9.118837596133744e-08,
      "eval_mean_token_accuracy": 0.5473919115424936,
      "eval_num_tokens": 9276318.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5617097491045404,
      "eval_rewards/margins": 22.518526158301658,
      "eval_rewards/rejected": -20.956816346037623,
      "eval_runtime": 54.4555,
      "eval_samples_per_second": 11.239,
      "eval_steps_per_second": 5.619,
      "step": 1300
    },
    {
      "entropy": 1.215628532320261,
      "epoch": 1.8022015823873407,
      "grad_norm": 3.170967102050781e-05,
      "learning_rate": 8.256513818450732e-06,
      "logits/chosen": -3.2075363655001468,
      "logits/rejected": -2.942603792959435,
      "logps/chosen": -88.86163854598999,
      "logps/rejected": -366.01112480163573,
      "loss": 2.054591732303379e-08,
      "mean_token_accuracy": 0.5597702454775572,
      "num_tokens": 9352151.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.412201857380569,
      "rewards/margins": 22.351779055595397,
      "rewards/rejected": -20.939577162265778,
      "step": 1310
    },
    {
      "entropy": 1.2229851864278316,
      "epoch": 1.8159614723082216,
      "grad_norm": 2.2530555725097656e-05,
      "learning_rate": 8.099154620104965e-06,
      "logits/chosen": -3.2086390495519246,
      "logits/rejected": -2.9236085096964888,
      "logps/chosen": -92.50195686817169,
      "logps/rejected": -357.3075210571289,
      "loss": 8.650430345369387e-08,
      "mean_token_accuracy": 0.5526527386158705,
      "num_tokens": 9423542.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6557919790968298,
      "rewards/margins": 22.534833192825317,
      "rewards/rejected": -20.87904134988785,
      "step": 1320
    },
    {
      "entropy": 1.230076377093792,
      "epoch": 1.8297213622291022,
      "grad_norm": 7.497146725654602e-08,
      "learning_rate": 7.942282273433732e-06,
      "logits/chosen": -3.194487966932489,
      "logits/rejected": -2.9350706353122984,
      "logps/chosen": -88.21589787006378,
      "logps/rejected": -364.98791046142577,
      "loss": 5.49991739262623e-08,
      "mean_token_accuracy": 0.559753555431962,
      "num_tokens": 9490789.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3836110661271959,
      "rewards/margins": 22.71821274757385,
      "rewards/rejected": -21.33460156917572,
      "step": 1330
    },
    {
      "entropy": 1.2405915766954423,
      "epoch": 1.8434812521499828,
      "grad_norm": 2.076849341392517e-07,
      "learning_rate": 7.785936957173026e-06,
      "logits/chosen": -3.2164539935100174,
      "logits/rejected": -2.9248479991999337,
      "logps/chosen": -89.55370953083039,
      "logps/rejected": -362.57230072021486,
      "loss": 1.4500157874408615e-08,
      "mean_token_accuracy": 0.5470205157995224,
      "num_tokens": 9556753.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5294401086866856,
      "rewards/margins": 22.728866374492647,
      "rewards/rejected": -21.19942626953125,
      "step": 1340
    },
    {
      "entropy": 1.2255703970789908,
      "epoch": 1.8572411420708634,
      "grad_norm": 2.0265579223632812e-06,
      "learning_rate": 7.630158715073813e-06,
      "logits/chosen": -3.208081273055396,
      "logits/rejected": -2.9360034267996244,
      "logps/chosen": -94.43741364479065,
      "logps/rejected": -355.30568141937255,
      "loss": 3.467322358119418e-08,
      "mean_token_accuracy": 0.5444860707968473,
      "num_tokens": 9627619.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4317927658557892,
      "rewards/margins": 22.225133633613588,
      "rewards/rejected": -20.79334100484848,
      "step": 1350
    },
    {
      "entropy": 1.2434842370450496,
      "epoch": 1.871001031991744,
      "grad_norm": 8.121132850646973e-07,
      "learning_rate": 7.4749874456458945e-06,
      "logits/chosen": -3.212613006457194,
      "logits/rejected": -2.9251595540262754,
      "logps/chosen": -91.65022368431092,
      "logps/rejected": -357.33582496643066,
      "loss": 1.5621509419361246e-08,
      "mean_token_accuracy": 0.5496355626732111,
      "num_tokens": 9701352.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5653375163674355,
      "rewards/margins": 22.173942589759825,
      "rewards/rejected": -20.608605122566225,
      "step": 1360
    },
    {
      "entropy": 1.2268347784876823,
      "epoch": 1.8847609219126247,
      "grad_norm": 2.1338462829589844e-05,
      "learning_rate": 7.3204628919389485e-06,
      "logits/chosen": -3.2073015388607997,
      "logits/rejected": -2.933093921397797,
      "logps/chosen": -96.34979357719422,
      "logps/rejected": -358.03315315246584,
      "loss": 2.1239095815417386e-08,
      "mean_token_accuracy": 0.5452436875551939,
      "num_tokens": 9770885.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5329102117568254,
      "rewards/margins": 22.605379486083983,
      "rewards/rejected": -21.07246913909912,
      "step": 1370
    },
    {
      "entropy": 1.2450409799814224,
      "epoch": 1.8985208118335053,
      "grad_norm": 2.518296241760254e-06,
      "learning_rate": 7.166624631363386e-06,
      "logits/chosen": -3.2212513769590827,
      "logits/rejected": -2.9404137559127967,
      "logps/chosen": -92.94262833595276,
      "logps/rejected": -358.9796733856201,
      "loss": 6.693504133181704e-08,
      "mean_token_accuracy": 0.5537132691591978,
      "num_tokens": 9844261.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5716754218563438,
      "rewards/margins": 22.33404486179352,
      "rewards/rejected": -20.76236960887909,
      "step": 1380
    },
    {
      "entropy": 1.243330615758896,
      "epoch": 1.912280701754386,
      "grad_norm": 3.4868717193603516e-06,
      "learning_rate": 7.01351206555364e-06,
      "logits/chosen": -3.202905518009641,
      "logits/rejected": -2.95122810678823,
      "logps/chosen": -92.49315547943115,
      "logps/rejected": -368.03791732788085,
      "loss": 5.302155727804347e-08,
      "mean_token_accuracy": 0.5490838758647442,
      "num_tokens": 9916718.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.408531033806503,
      "rewards/margins": 22.847691893577576,
      "rewards/rejected": -21.439160919189455,
      "step": 1390
    },
    {
      "entropy": 1.2381026059389115,
      "epoch": 1.9260405916752665,
      "grad_norm": 0.00011587142944335938,
      "learning_rate": 6.861164410276469e-06,
      "logits/chosen": -3.209368117524053,
      "logits/rejected": -2.9519032131182743,
      "logps/chosen": -90.15022115707397,
      "logps/rejected": -360.4969120025635,
      "loss": 3.583730858736089e-08,
      "mean_token_accuracy": 0.554393558204174,
      "num_tokens": 9992414.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.602032061293721,
      "rewards/margins": 22.397946190834045,
      "rewards/rejected": -20.795914244651794,
      "step": 1400
    },
    {
      "epoch": 1.9260405916752665,
      "eval_entropy": 1.2311562377642962,
      "eval_logits/chosen": -3.208069794657062,
      "eval_logits/rejected": -2.9272682126130256,
      "eval_logps/chosen": -94.2417087305605,
      "eval_logps/rejected": -359.93737563588263,
      "eval_loss": 9.088540053880934e-08,
      "eval_mean_token_accuracy": 0.5476740352273767,
      "eval_num_tokens": 9992414.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5574156118715121,
      "eval_rewards/margins": 22.523287224613764,
      "eval_rewards/rejected": -20.9658716644337,
      "eval_runtime": 55.0297,
      "eval_samples_per_second": 11.121,
      "eval_steps_per_second": 5.561,
      "step": 1400
    },
    {
      "entropy": 1.2308048702776433,
      "epoch": 1.9398004815961474,
      "grad_norm": 2.0563602447509766e-06,
      "learning_rate": 6.709620685386857e-06,
      "logits/chosen": -3.21866885172357,
      "logits/rejected": -2.924553737218475,
      "logps/chosen": -95.29532494544983,
      "logps/rejected": -351.67724494934083,
      "loss": 6.201777864589531e-08,
      "mean_token_accuracy": 0.5582938764244318,
      "num_tokens": 10073883.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5281909180805087,
      "rewards/margins": 21.83339207172394,
      "rewards/rejected": -20.30520099401474,
      "step": 1410
    },
    {
      "entropy": 1.2180306166410446,
      "epoch": 1.9535603715170278,
      "grad_norm": 3.241002559661865e-07,
      "learning_rate": 6.558919704834121e-06,
      "logits/chosen": -3.2067047975104517,
      "logits/rejected": -2.931083820952747,
      "logps/chosen": -89.81032228469849,
      "logps/rejected": -363.26050567626953,
      "loss": 1.7645916727815347e-08,
      "mean_token_accuracy": 0.5559442441910505,
      "num_tokens": 10145775.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3647572711110114,
      "rewards/margins": 22.6029949426651,
      "rewards/rejected": -21.238237845897675,
      "step": 1420
    },
    {
      "entropy": 1.2391163140535355,
      "epoch": 1.9673202614379086,
      "grad_norm": 4.880130290985107e-07,
      "learning_rate": 6.409100066720724e-06,
      "logits/chosen": -3.208245858330298,
      "logits/rejected": -2.927146088570569,
      "logps/chosen": -96.24866271018982,
      "logps/rejected": -374.9543224334717,
      "loss": 3.9638800330976666e-08,
      "mean_token_accuracy": 0.5497354336082936,
      "num_tokens": 10217226.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.399499694549013,
      "rewards/margins": 23.0203608751297,
      "rewards/rejected": -21.620861136913298,
      "step": 1430
    },
    {
      "entropy": 1.227453202009201,
      "epoch": 1.981080151358789,
      "grad_norm": 5.4836273193359375e-06,
      "learning_rate": 6.260200143416385e-06,
      "logits/chosen": -3.213358957807857,
      "logits/rejected": -2.9454548998081966,
      "logps/chosen": -90.53418622016906,
      "logps/rejected": -359.1464733123779,
      "loss": 2.0817584811538837e-08,
      "mean_token_accuracy": 0.5452344041317702,
      "num_tokens": 10289925.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5044855073094368,
      "rewards/margins": 22.441794085502625,
      "rewards/rejected": -20.937308418750764,
      "step": 1440
    },
    {
      "entropy": 1.2387429229915141,
      "epoch": 1.9948400412796699,
      "grad_norm": 1.2218952178955078e-06,
      "learning_rate": 6.112258071729999e-06,
      "logits/chosen": -3.2053667412365074,
      "logits/rejected": -2.9334120491094025,
      "logps/chosen": -96.13532671928405,
      "logps/rejected": -364.0920166015625,
      "loss": 1.0310853326700454e-08,
      "mean_token_accuracy": 0.5550614651292562,
      "num_tokens": 10365037.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5084194824099542,
      "rewards/margins": 22.576842379570007,
      "rewards/rejected": -21.068422937393187,
      "step": 1450
    },
    {
      "entropy": 1.2330439381110363,
      "epoch": 2.0082559339525283,
      "grad_norm": 8.754432201385498e-07,
      "learning_rate": 5.965311743141904e-06,
      "logits/chosen": -3.208575671654014,
      "logits/rejected": -2.9326438852599463,
      "logps/chosen": -91.21437239035582,
      "logps/rejected": -363.5683351174379,
      "loss": 2.8951188824066777e-08,
      "mean_token_accuracy": 0.5454204422541153,
      "num_tokens": 10438552.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3954702159151053,
      "rewards/margins": 22.230517093951885,
      "rewards/rejected": -20.83504707385332,
      "step": 1460
    },
    {
      "entropy": 1.2330299749970437,
      "epoch": 2.022015823873409,
      "grad_norm": 3.166496753692627e-07,
      "learning_rate": 5.819398794098965e-06,
      "logits/chosen": -3.1973997733416484,
      "logits/rejected": -2.9382980669061474,
      "logps/chosen": -93.53767242431641,
      "logps/rejected": -366.40000381469724,
      "loss": 5.971180172537061e-09,
      "mean_token_accuracy": 0.5506532412022352,
      "num_tokens": 10508891.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3291676957160234,
      "rewards/margins": 22.63092885017395,
      "rewards/rejected": -21.30176111459732,
      "step": 1470
    },
    {
      "entropy": 1.2254330806434155,
      "epoch": 2.0357757137942896,
      "grad_norm": 1.8477439880371094e-06,
      "learning_rate": 5.674556596374993e-06,
      "logits/chosen": -3.2044453379794327,
      "logits/rejected": -2.9349608519596306,
      "logps/chosen": -96.1356906414032,
      "logps/rejected": -366.6023094177246,
      "loss": 1.3861522063507437e-08,
      "mean_token_accuracy": 0.5465929575264454,
      "num_tokens": 10581981.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3926833651959896,
      "rewards/margins": 22.933032035827637,
      "rewards/rejected": -21.540348649024963,
      "step": 1480
    },
    {
      "entropy": 1.2419807285070419,
      "epoch": 2.0495356037151704,
      "grad_norm": 3.635883331298828e-06,
      "learning_rate": 5.530822247498926e-06,
      "logits/chosen": -3.203796424428083,
      "logits/rejected": -2.9470883391626623,
      "logps/chosen": -92.74581410884858,
      "logps/rejected": -362.62787170410155,
      "loss": 6.57019540994952e-08,
      "mean_token_accuracy": 0.5511626075953245,
      "num_tokens": 10654203.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5442042749375104,
      "rewards/margins": 22.415554237365722,
      "rewards/rejected": -20.87134999036789,
      "step": 1490
    },
    {
      "entropy": 1.2162199713289739,
      "epoch": 2.063295493636051,
      "grad_norm": 2.682209014892578e-06,
      "learning_rate": 5.388232561253301e-06,
      "logits/chosen": -3.1954805100926533,
      "logits/rejected": -2.9325630461122247,
      "logps/chosen": -98.36348547935486,
      "logps/rejected": -367.20038681030275,
      "loss": 4.9491802656120853e-08,
      "mean_token_accuracy": 0.5584958054125309,
      "num_tokens": 10724322.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2897070567123592,
      "rewards/margins": 22.65980746746063,
      "rewards/rejected": -21.370100212097167,
      "step": 1500
    },
    {
      "epoch": 2.063295493636051,
      "eval_entropy": 1.2309571816251168,
      "eval_logits/chosen": -3.208181417563463,
      "eval_logits/rejected": -2.927348986347919,
      "eval_logps/chosen": -94.22639578775643,
      "eval_logps/rejected": -359.94768000584025,
      "eval_loss": 9.004938306134136e-08,
      "eval_mean_token_accuracy": 0.5473804468033361,
      "eval_num_tokens": 10724322.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5589469410235586,
      "eval_rewards/margins": 22.525849155351228,
      "eval_rewards/rejected": -20.966902284061206,
      "eval_runtime": 56.0305,
      "eval_samples_per_second": 10.923,
      "eval_steps_per_second": 5.461,
      "step": 1500
    },
    {
      "entropy": 1.2275754123926164,
      "epoch": 2.0770553835569316,
      "grad_norm": 1.4826655387878418e-06,
      "learning_rate": 5.24682405824535e-06,
      "logits/chosen": -3.2175959418317355,
      "logits/rejected": -2.9383502084395783,
      "logps/chosen": -92.50881714820862,
      "logps/rejected": -371.0841016769409,
      "loss": 2.6978818823408802e-08,
      "mean_token_accuracy": 0.55126539953053,
      "num_tokens": 10800237.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4649929033592344,
      "rewards/margins": 22.734590768814087,
      "rewards/rejected": -21.269598031044005,
      "step": 1510
    },
    {
      "entropy": 1.2253120809793472,
      "epoch": 2.090815273477812,
      "grad_norm": 1.5124678611755371e-06,
      "learning_rate": 5.106632956553195e-06,
      "logits/chosen": -3.1973274281094786,
      "logits/rejected": -2.9305039646676687,
      "logps/chosen": -90.93314514160156,
      "logps/rejected": -365.3414768218994,
      "loss": 1.938408900059585e-08,
      "mean_token_accuracy": 0.5518848109990359,
      "num_tokens": 10871898.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.426638696854934,
      "rewards/margins": 22.47057647705078,
      "rewards/rejected": -21.043937778472902,
      "step": 1520
    },
    {
      "entropy": 1.233763538300991,
      "epoch": 2.104575163398693,
      "grad_norm": 2.2165477275848389e-07,
      "learning_rate": 4.967695162449564e-06,
      "logits/chosen": -3.205103492080074,
      "logits/rejected": -2.9324556564221185,
      "logps/chosen": -91.14850578308105,
      "logps/rejected": -364.838455581665,
      "loss": 3.638180317011574e-08,
      "mean_token_accuracy": 0.5595021311193704,
      "num_tokens": 10941931.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4626295033842325,
      "rewards/margins": 22.897762179374695,
      "rewards/rejected": -21.43513250350952,
      "step": 1530
    },
    {
      "entropy": 1.2216242380440234,
      "epoch": 2.1183350533195733,
      "grad_norm": 5.3942203521728516e-06,
      "learning_rate": 4.830046261205322e-06,
      "logits/chosen": -3.2007257440208505,
      "logits/rejected": -2.921705737174231,
      "logps/chosen": -86.60471982955933,
      "logps/rejected": -351.9167873382568,
      "loss": 1.817294901229616e-08,
      "mean_token_accuracy": 0.5593262724578381,
      "num_tokens": 11007312.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4826515832915903,
      "rewards/margins": 21.87851518392563,
      "rewards/rejected": -20.395863604545593,
      "step": 1540
    },
    {
      "entropy": 1.2366574123501777,
      "epoch": 2.132094943240454,
      "grad_norm": 3.427267074584961e-06,
      "learning_rate": 4.693721507975237e-06,
      "logits/chosen": -3.2084120035887294,
      "logits/rejected": -2.9482167846435265,
      "logps/chosen": -88.57906460762024,
      "logps/rejected": -358.5270462036133,
      "loss": 3.240177193220006e-08,
      "mean_token_accuracy": 0.5516232702881098,
      "num_tokens": 11084161.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.534480356797576,
      "rewards/margins": 21.942111873626708,
      "rewards/rejected": -20.40763156414032,
      "step": 1550
    },
    {
      "entropy": 1.2354340434074402,
      "epoch": 2.1458548331613345,
      "grad_norm": 4.559755325317383e-06,
      "learning_rate": 4.558755818768313e-06,
      "logits/chosen": -3.2086426875744265,
      "logits/rejected": -2.930807029613111,
      "logps/chosen": -97.39483575820923,
      "logps/rejected": -363.5681007385254,
      "loss": 7.447766847690218e-08,
      "mean_token_accuracy": 0.541871952265501,
      "num_tokens": 11162793.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2772513107396661,
      "rewards/margins": 22.164503431320192,
      "rewards/rejected": -20.887252259254456,
      "step": 1560
    },
    {
      "entropy": 1.249245548248291,
      "epoch": 2.1596147230822154,
      "grad_norm": 1.2069940567016602e-06,
      "learning_rate": 4.425183761504959e-06,
      "logits/chosen": -3.215003107093657,
      "logits/rejected": -2.93700778830525,
      "logps/chosen": -96.33298497200012,
      "logps/rejected": -364.01307106018066,
      "loss": 2.6582091550153562e-08,
      "mean_token_accuracy": 0.5509772460907698,
      "num_tokens": 11233429.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.645207718014717,
      "rewards/margins": 22.903429484367372,
      "rewards/rejected": -21.258221745491028,
      "step": 1570
    },
    {
      "entropy": 1.2453189969062806,
      "epoch": 2.173374613003096,
      "grad_norm": 4.00543212890625e-05,
      "learning_rate": 4.293039547163317e-06,
      "logits/chosen": -3.203883989884722,
      "logits/rejected": -2.9435132493578235,
      "logps/chosen": -96.69977416992188,
      "logps/rejected": -356.1233158111572,
      "loss": 1.787334326763812e-07,
      "mean_token_accuracy": 0.5541251346468925,
      "num_tokens": 11306856.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.564822706952691,
      "rewards/margins": 22.051422691345216,
      "rewards/rejected": -20.486599802970886,
      "step": 1580
    },
    {
      "entropy": 1.2330203399062156,
      "epoch": 2.1871345029239766,
      "grad_norm": 9.059906005859375e-06,
      "learning_rate": 4.162357021017024e-06,
      "logits/chosen": -3.2046981821484644,
      "logits/rejected": -2.9331812450409367,
      "logps/chosen": -96.95174942016601,
      "logps/rejected": -360.5036224365234,
      "loss": 1.6826841431338834e-08,
      "mean_token_accuracy": 0.5593572478741408,
      "num_tokens": 11382019.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5032939210534095,
      "rewards/margins": 22.360925555229187,
      "rewards/rejected": -20.857631814479827,
      "step": 1590
    },
    {
      "entropy": 1.2257887080311776,
      "epoch": 2.2008943928448574,
      "grad_norm": 1.259148120880127e-06,
      "learning_rate": 4.033169653966643e-06,
      "logits/chosen": -3.194304146133926,
      "logits/rejected": -2.929467087673367,
      "logps/chosen": -90.85989444255829,
      "logps/rejected": -357.8446460723877,
      "loss": 1.9598689959821058e-08,
      "mean_token_accuracy": 0.5546761188656092,
      "num_tokens": 11450315.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3403418456204235,
      "rewards/margins": 22.388022828102113,
      "rewards/rejected": -21.0476811170578,
      "step": 1600
    },
    {
      "epoch": 2.2008943928448574,
      "eval_entropy": 1.2307752834426031,
      "eval_logits/chosen": -3.208144838054297,
      "eval_logits/rejected": -2.927330668770716,
      "eval_logps/chosen": -94.22898377935871,
      "eval_logps/rejected": -360.023319817836,
      "eval_loss": 8.89587852270779e-08,
      "eval_mean_token_accuracy": 0.5471583587281844,
      "eval_num_tokens": 11450315.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5586880991856258,
      "eval_rewards/margins": 22.533154122969684,
      "eval_rewards/rejected": -20.974466049593257,
      "eval_runtime": 57.2229,
      "eval_samples_per_second": 10.695,
      "eval_steps_per_second": 5.348,
      "step": 1600
    },
    {
      "entropy": 1.2212409198284149,
      "epoch": 2.214654282765738,
      "grad_norm": 1.0073184967041016e-05,
      "learning_rate": 3.905510533966959e-06,
      "logits/chosen": -3.214314473031172,
      "logits/rejected": -2.9391068323773606,
      "logps/chosen": -91.97887825965881,
      "logps/rejected": -363.52599411010743,
      "loss": 1.9494005698561522e-08,
      "mean_token_accuracy": 0.5564212694764137,
      "num_tokens": 11526978.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.525117878522724,
      "rewards/margins": 22.85406105518341,
      "rewards/rejected": -21.328943157196044,
      "step": 1610
    },
    {
      "entropy": 1.2366786934435368,
      "epoch": 2.2284141726866187,
      "grad_norm": 1.1324882507324219e-05,
      "learning_rate": 3.7794123575523646e-06,
      "logits/chosen": -3.2020117108366244,
      "logits/rejected": -2.941936707092504,
      "logps/chosen": -95.76478714942932,
      "logps/rejected": -358.69820404052734,
      "loss": 6.403648740160861e-08,
      "mean_token_accuracy": 0.5473370250314474,
      "num_tokens": 11597783.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4697597675025462,
      "rewards/margins": 22.156175112724306,
      "rewards/rejected": -20.686415362358094,
      "step": 1620
    },
    {
      "entropy": 1.2357837766408921,
      "epoch": 2.242174062607499,
      "grad_norm": 3.266334533691406e-05,
      "learning_rate": 3.6549074214625004e-06,
      "logits/chosen": -3.207695843054857,
      "logits/rejected": -2.931019461868485,
      "logps/chosen": -95.36823711395263,
      "logps/rejected": -364.51545028686525,
      "loss": 1.6160251448127384e-08,
      "mean_token_accuracy": 0.5486490651965141,
      "num_tokens": 11667690.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.452584043238312,
      "rewards/margins": 22.741941976547242,
      "rewards/rejected": -21.28935799598694,
      "step": 1630
    },
    {
      "entropy": 1.2502774253487587,
      "epoch": 2.25593395252838,
      "grad_norm": 1.2040138244628906e-05,
      "learning_rate": 3.532027614370309e-06,
      "logits/chosen": -3.2043903527517044,
      "logits/rejected": -2.9334508568932933,
      "logps/chosen": -92.87388520240783,
      "logps/rejected": -353.56325378417966,
      "loss": 1.9827470509881095e-08,
      "mean_token_accuracy": 0.5545094706118107,
      "num_tokens": 11736002.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.63247759337537,
      "rewards/margins": 22.165972471237183,
      "rewards/rejected": -20.53349506855011,
      "step": 1640
    },
    {
      "entropy": 1.223061728477478,
      "epoch": 2.2696938424492603,
      "grad_norm": 3.9674341678619385e-07,
      "learning_rate": 3.4108044087145764e-06,
      "logits/chosen": -3.212080244158158,
      "logits/rejected": -2.9380040624704558,
      "logps/chosen": -91.98516411781311,
      "logps/rejected": -363.08466148376465,
      "loss": 2.2882196049067714e-08,
      "mean_token_accuracy": 0.5499565359205008,
      "num_tokens": 11806524.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5206007588654757,
      "rewards/margins": 22.941864371299744,
      "rewards/rejected": -21.42126350402832,
      "step": 1650
    },
    {
      "entropy": 1.2240382507443428,
      "epoch": 2.283453732370141,
      "grad_norm": 4.9173831939697266e-06,
      "learning_rate": 3.2912688526390747e-06,
      "logits/chosen": -3.201106158061068,
      "logits/rejected": -2.94364090708882,
      "logps/chosen": -90.90681099891663,
      "logps/rejected": -357.92090034484863,
      "loss": 3.113183311143075e-08,
      "mean_token_accuracy": 0.5508798561990261,
      "num_tokens": 11877466.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3562086425721644,
      "rewards/margins": 22.361026167869568,
      "rewards/rejected": -21.004817533493043,
      "step": 1660
    },
    {
      "entropy": 1.229287575185299,
      "epoch": 2.2972136222910216,
      "grad_norm": 5.662441253662109e-07,
      "learning_rate": 3.173451562040428e-06,
      "logits/chosen": -3.2092518895630406,
      "logits/rejected": -2.9406627581018197,
      "logps/chosen": -93.75379481315613,
      "logps/rejected": -355.3395336151123,
      "loss": 1.1599519211813458e-07,
      "mean_token_accuracy": 0.5449749272316694,
      "num_tokens": 11946957.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5091715334914624,
      "rewards/margins": 22.31476628780365,
      "rewards/rejected": -20.80559469461441,
      "step": 1670
    },
    {
      "entropy": 1.2276184171438218,
      "epoch": 2.3109735122119024,
      "grad_norm": 1.6093254089355469e-06,
      "learning_rate": 3.0573827127266388e-06,
      "logits/chosen": -3.211323672958944,
      "logits/rejected": -2.9291713412088,
      "logps/chosen": -92.78370037078858,
      "logps/rejected": -362.05942516326905,
      "loss": 4.019505581709382e-08,
      "mean_token_accuracy": 0.5594781070947648,
      "num_tokens": 12019258.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4811159919947385,
      "rewards/margins": 22.234603309631346,
      "rewards/rejected": -20.753487145900728,
      "step": 1680
    },
    {
      "entropy": 1.243286745250225,
      "epoch": 2.324733402132783,
      "grad_norm": 4.116445779800415e-07,
      "learning_rate": 2.9430920326883406e-06,
      "logits/chosen": -3.198338749012816,
      "logits/rejected": -2.936215425086737,
      "logps/chosen": -97.7242082118988,
      "logps/rejected": -358.0124053955078,
      "loss": 3.1689586421634884e-08,
      "mean_token_accuracy": 0.536598714068532,
      "num_tokens": 12086564.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2556954478845,
      "rewards/margins": 22.314459955692293,
      "rewards/rejected": -21.058764576911926,
      "step": 1690
    },
    {
      "entropy": 1.2481802843511105,
      "epoch": 2.3384932920536636,
      "grad_norm": 1.080334186553955e-06,
      "learning_rate": 2.830608794484756e-06,
      "logits/chosen": -3.1943609337265597,
      "logits/rejected": -2.9488356300852865,
      "logps/chosen": -89.5389600276947,
      "logps/rejected": -359.9536491394043,
      "loss": 2.3795587367203554e-08,
      "mean_token_accuracy": 0.5535460270941257,
      "num_tokens": 12158539.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.362912526819855,
      "rewards/margins": 22.151762986183165,
      "rewards/rejected": -20.78885051012039,
      "step": 1700
    },
    {
      "epoch": 2.3384932920536636,
      "eval_entropy": 1.2307304270516812,
      "eval_logits/chosen": -3.2081578712434293,
      "eval_logits/rejected": -2.9272117784230485,
      "eval_logps/chosen": -94.22525123521393,
      "eval_logps/rejected": -360.04312303331164,
      "eval_loss": 8.878492252506476e-08,
      "eval_mean_token_accuracy": 0.5476229925171222,
      "eval_num_tokens": 12158539.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5590613652142435,
      "eval_rewards/margins": 22.535507769366497,
      "eval_rewards/rejected": -20.976446469624836,
      "eval_runtime": 56.6188,
      "eval_samples_per_second": 10.809,
      "eval_steps_per_second": 5.405,
      "step": 1700
    },
    {
      "entropy": 1.2194796189665795,
      "epoch": 2.352253181974544,
      "grad_norm": 2.905726432800293e-06,
      "learning_rate": 2.7199618077463007e-06,
      "logits/chosen": -3.2052349518897323,
      "logits/rejected": -2.938724784978354,
      "logps/chosen": -97.47988610267639,
      "logps/rejected": -361.4190784454346,
      "loss": 1.1008413025592744e-08,
      "mean_token_accuracy": 0.5539489138871432,
      "num_tokens": 12228074.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5363455599173903,
      "rewards/margins": 22.715231454372407,
      "rewards/rejected": -21.178886151313783,
      "step": 1710
    },
    {
      "entropy": 1.2418527334928513,
      "epoch": 2.366013071895425,
      "grad_norm": 9.012222290039062e-05,
      "learning_rate": 2.6111794117957612e-06,
      "logits/chosen": -3.211556303584559,
      "logits/rejected": -2.93789795511538,
      "logps/chosen": -93.4774594783783,
      "logps/rejected": -356.88815116882324,
      "loss": 4.893282152806933e-08,
      "mean_token_accuracy": 0.547554525360465,
      "num_tokens": 12292110.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5211389977484941,
      "rewards/margins": 22.537738013267518,
      "rewards/rejected": -21.016598844528197,
      "step": 1720
    },
    {
      "entropy": 1.240048848092556,
      "epoch": 2.3797729618163057,
      "grad_norm": 4.506111145019531e-05,
      "learning_rate": 2.504289468389899e-06,
      "logits/chosen": -3.206205666054303,
      "logits/rejected": -2.943638671152749,
      "logps/chosen": -90.50992622375489,
      "logps/rejected": -362.01258277893066,
      "loss": 8.840201530802005e-08,
      "mean_token_accuracy": 0.5487963829189539,
      "num_tokens": 12358835.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.542650154232979,
      "rewards/margins": 22.581651854515076,
      "rewards/rejected": -21.039001631736756,
      "step": 1730
    },
    {
      "entropy": 1.2210087485611438,
      "epoch": 2.393532851737186,
      "grad_norm": 5.650520324707031e-05,
      "learning_rate": 2.3993193545834182e-06,
      "logits/chosen": -3.2105181749109306,
      "logits/rejected": -2.931494112684553,
      "logps/chosen": -92.28900375366212,
      "logps/rejected": -364.4213104248047,
      "loss": 7.528050787186657e-08,
      "mean_token_accuracy": 0.5661897122859955,
      "num_tokens": 12433046.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4046879611909389,
      "rewards/margins": 22.549579215049743,
      "rewards/rejected": -21.14489116668701,
      "step": 1740
    },
    {
      "entropy": 1.230967465043068,
      "epoch": 2.4072927416580665,
      "grad_norm": 4.4405460357666016e-06,
      "learning_rate": 2.2962959557170284e-06,
      "logits/chosen": -3.214696198045111,
      "logits/rejected": -2.92904874748245,
      "logps/chosen": -96.72074069976807,
      "logps/rejected": -371.5300548553467,
      "loss": 7.795250667186338e-09,
      "mean_token_accuracy": 0.5476579256355762,
      "num_tokens": 12510093.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5278369845356792,
      "rewards/margins": 22.89517023563385,
      "rewards/rejected": -21.367333316802977,
      "step": 1750
    },
    {
      "entropy": 1.2341578394174575,
      "epoch": 2.4210526315789473,
      "grad_norm": 6.765127182006836e-06,
      "learning_rate": 2.1952456585314894e-06,
      "logits/chosen": -3.212110355605135,
      "logits/rejected": -2.9278311870022096,
      "logps/chosen": -94.76394276618957,
      "logps/rejected": -366.80492820739744,
      "loss": 1.8708394122768367e-08,
      "mean_token_accuracy": 0.5509970057755709,
      "num_tokens": 12582912.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5956923257559539,
      "rewards/margins": 22.885032773017883,
      "rewards/rejected": -21.289340591430665,
      "step": 1760
    },
    {
      "entropy": 1.2361514404416085,
      "epoch": 2.434812521499828,
      "grad_norm": 7.37607479095459e-07,
      "learning_rate": 2.0961943444093204e-06,
      "logits/chosen": -3.215165102721952,
      "logits/rejected": -2.928150886837859,
      "logps/chosen": -96.75569877624511,
      "logps/rejected": -358.2433895111084,
      "loss": 1.789561281384522e-08,
      "mean_token_accuracy": 0.5518970482051373,
      "num_tokens": 12653031.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4577152743004262,
      "rewards/margins": 22.373184776306154,
      "rewards/rejected": -20.915469527244568,
      "step": 1770
    },
    {
      "entropy": 1.2188403598964215,
      "epoch": 2.4485724114207086,
      "grad_norm": 7.271766662597656e-06,
      "learning_rate": 1.9991673827459835e-06,
      "logits/chosen": -3.1989314300688627,
      "logits/rejected": -2.9344233134383053,
      "logps/chosen": -92.87870864868164,
      "logps/rejected": -356.7177398681641,
      "loss": 4.4659608988695255e-08,
      "mean_token_accuracy": 0.5518682572990656,
      "num_tokens": 12725637.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4662983087822794,
      "rewards/margins": 22.44385893344879,
      "rewards/rejected": -20.977560567855836,
      "step": 1780
    },
    {
      "entropy": 1.246582841873169,
      "epoch": 2.4623323013415894,
      "grad_norm": 6.628036499023438e-05,
      "learning_rate": 1.9041896244521584e-06,
      "logits/chosen": -3.203583763033077,
      "logits/rejected": -2.9414372493750873,
      "logps/chosen": -89.14191975593567,
      "logps/rejected": -357.88146629333494,
      "loss": 2.0817780921333907e-08,
      "mean_token_accuracy": 0.5476832162588835,
      "num_tokens": 12793669.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.506299409084022,
      "rewards/margins": 22.251666259765624,
      "rewards/rejected": -20.745366990566254,
      "step": 1790
    },
    {
      "entropy": 1.2349150389432908,
      "epoch": 2.47609219126247,
      "grad_norm": 3.5762786865234375e-05,
      "learning_rate": 1.8112853955888543e-06,
      "logits/chosen": -3.2071731207232768,
      "logits/rejected": -2.936215464591402,
      "logps/chosen": -90.68877263069153,
      "logps/rejected": -355.58887882232665,
      "loss": 3.7140389963497e-08,
      "mean_token_accuracy": 0.5518446356058121,
      "num_tokens": 12863146.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4937029276043177,
      "rewards/margins": 22.40243979692459,
      "rewards/rejected": -20.90873694419861,
      "step": 1800
    },
    {
      "epoch": 2.47609219126247,
      "eval_entropy": 1.2310942107555913,
      "eval_logits/chosen": -3.208092843314108,
      "eval_logits/rejected": -2.9271828156565594,
      "eval_logps/chosen": -94.19586368635589,
      "eval_logps/rejected": -359.9574696659263,
      "eval_loss": 9.066506834187749e-08,
      "eval_mean_token_accuracy": 0.5472219296140608,
      "eval_num_tokens": 12863146.0,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.5620001425443133,
      "eval_rewards/margins": 22.529881178164015,
      "eval_rewards/rejected": -20.967881090500775,
      "eval_runtime": 58.2977,
      "eval_samples_per_second": 10.498,
      "eval_steps_per_second": 5.249,
      "step": 1800
    }
  ],
  "logging_steps": 10,
  "max_steps": 2181,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 6.089403206876529e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}