{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.592,
  "eval_steps": 50,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "entropy": 2.555654156208038,
      "epoch": 0.08,
      "grad_norm": 35.3809700012207,
      "learning_rate": 2.8571428571428573e-06,
      "logits/chosen": 0.4110352631849171,
      "logits/rejected": 0.4300117739250047,
      "logps/chosen": -157.81439809799195,
      "logps/rejected": -150.4315113067627,
      "loss": 0.6903757572174072,
      "mean_token_accuracy": 0.45491820499300956,
      "num_tokens": 63280.0,
      "rewards/accuracies": 0.54375,
      "rewards/chosen": 0.018655257727368736,
      "rewards/margins": 0.006082923361100256,
      "rewards/rejected": 0.012572333973366767,
      "step": 5
    },
    {
      "entropy": 2.5844572484493256,
      "epoch": 0.16,
      "grad_norm": 47.283817291259766,
      "learning_rate": 4.984280524733107e-06,
      "logits/chosen": 0.2089756903127898,
      "logits/rejected": 0.3923359114864852,
      "logps/chosen": -160.2436939239502,
      "logps/rejected": -154.7825647354126,
      "loss": 0.7087126731872558,
      "mean_token_accuracy": 0.4676915630698204,
      "num_tokens": 128751.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.026459120609797538,
      "rewards/margins": -0.018515753094106913,
      "rewards/rejected": 0.04497487433254719,
      "step": 10
    },
    {
      "entropy": 2.4940366342663767,
      "epoch": 0.24,
      "grad_norm": 48.20133972167969,
      "learning_rate": 4.809698831278217e-06,
      "logits/chosen": 0.12422276096059184,
      "logits/rejected": 0.18387425940175633,
      "logps/chosen": -151.31794481277467,
      "logps/rejected": -163.5261468887329,
      "loss": 0.6959157466888428,
      "mean_token_accuracy": 0.47235521450638773,
      "num_tokens": 194142.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.059232240472920236,
      "rewards/margins": 0.021671066922135652,
      "rewards/rejected": -0.0809033086989075,
      "step": 15
    },
    {
      "entropy": 2.5471011281013487,
      "epoch": 0.32,
      "grad_norm": 42.294281005859375,
      "learning_rate": 4.454578706170075e-06,
      "logits/chosen": -0.030133471365837695,
      "logits/rejected": 0.08451241878365641,
      "logps/chosen": -135.41445970535278,
      "logps/rejected": -135.83921966552734,
      "loss": 0.6954083442687988,
      "mean_token_accuracy": 0.4549576163291931,
      "num_tokens": 254871.0,
      "rewards/accuracies": 0.5375,
      "rewards/chosen": -0.18437330080196263,
      "rewards/margins": 0.07001338726840914,
      "rewards/rejected": -0.2543866881169379,
      "step": 20
    },
    {
      "entropy": 2.4626115679740908,
      "epoch": 0.4,
      "grad_norm": 53.21565628051758,
      "learning_rate": 3.946678240449515e-06,
      "logits/chosen": 0.035210923156354317,
      "logits/rejected": 0.03131556176291432,
      "logps/chosen": -147.02334337234498,
      "logps/rejected": -139.0251603126526,
      "loss": 0.7144542694091797,
      "mean_token_accuracy": 0.4505583204329014,
      "num_tokens": 317752.0,
      "rewards/accuracies": 0.55,
      "rewards/chosen": -0.3056703276000917,
      "rewards/margins": 0.030420188792049885,
      "rewards/rejected": -0.33609051471576096,
      "step": 25
    },
    {
      "entropy": 2.418185669183731,
      "epoch": 0.48,
      "grad_norm": 50.63364791870117,
      "learning_rate": 3.3256976548879183e-06,
      "logits/chosen": -0.0063577626599184795,
      "logits/rejected": 0.0660789555792278,
      "logps/chosen": -151.3002492904663,
      "logps/rejected": -150.41238498687744,
      "loss": 0.6920054435729981,
      "mean_token_accuracy": 0.445500810444355,
      "num_tokens": 387452.0,
      "rewards/accuracies": 0.55625,
      "rewards/chosen": -0.2962045251857489,
      "rewards/margins": 0.08783619347959756,
      "rewards/rejected": -0.38404072066769,
      "step": 30
    },
    {
      "entropy": 2.42596016228199,
      "epoch": 0.56,
      "grad_norm": 47.31837844848633,
      "learning_rate": 2.6401761180929798e-06,
      "logits/chosen": -0.04082900087497934,
      "logits/rejected": 0.015806176234233794,
      "logps/chosen": -182.25081090927125,
      "logps/rejected": -175.18973140716554,
      "loss": 0.6894076824188232,
      "mean_token_accuracy": 0.46841963976621626,
      "num_tokens": 454873.0,
      "rewards/accuracies": 0.55625,
      "rewards/chosen": -0.30482858400791885,
      "rewards/margins": 0.07829219745472074,
      "rewards/rejected": -0.38312078421004114,
      "step": 35
    },
    {
      "entropy": 2.421294018626213,
      "epoch": 0.64,
      "grad_norm": 47.72428894042969,
      "learning_rate": 1.9436976651092143e-06,
      "logits/chosen": 0.04491766159482387,
      "logits/rejected": 0.15141158319763592,
      "logps/chosen": -160.55729579925537,
      "logps/rejected": -161.91398558616638,
      "loss": 0.7262070178985596,
      "mean_token_accuracy": 0.4616738587617874,
      "num_tokens": 518527.0,
      "rewards/accuracies": 0.51875,
      "rewards/chosen": -0.2627937063574791,
      "rewards/margins": 0.037597383465617897,
      "rewards/rejected": -0.30039109364151956,
      "step": 40
    },
    {
      "entropy": 2.3493838876485826,
      "epoch": 0.72,
      "grad_norm": 41.70436096191406,
      "learning_rate": 1.2907027822369006e-06,
      "logits/chosen": 0.06111884454416239,
      "logits/rejected": 0.1077471313778626,
      "logps/chosen": -155.73900985717773,
      "logps/rejected": -162.23499221801757,
      "loss": 0.6567366123199463,
      "mean_token_accuracy": 0.47065152823925016,
      "num_tokens": 586496.0,
      "rewards/accuracies": 0.5875,
      "rewards/chosen": -0.15574295134283603,
      "rewards/margins": 0.16263325293548406,
      "rewards/rejected": -0.31837619938887657,
      "step": 45
    },
    {
      "entropy": 2.3660752564668655,
      "epoch": 0.8,
      "grad_norm": 46.26735305786133,
      "learning_rate": 7.322330470336314e-07,
      "logits/chosen": 0.04797605992282007,
      "logits/rejected": 0.137719508348569,
      "logps/chosen": -165.56337423324584,
      "logps/rejected": -165.80605907440184,
      "loss": 0.6880872249603271,
      "mean_token_accuracy": 0.4618512228131294,
      "num_tokens": 659345.0,
      "rewards/accuracies": 0.51875,
      "rewards/chosen": -0.13928775684908032,
      "rewards/margins": 0.09893286619335413,
      "rewards/rejected": -0.238220629747957,
      "step": 50
    },
    {
      "epoch": 0.8,
      "eval_entropy": 2.4305524230003357,
      "eval_logits/chosen": 0.060791561005096514,
      "eval_logits/rejected": 0.1157722840661163,
      "eval_logps/chosen": -163.22620516967774,
      "eval_logps/rejected": -155.69468914794922,
      "eval_loss": 0.699152946472168,
      "eval_mean_token_accuracy": 0.4591814261674881,
      "eval_num_tokens": 659345.0,
      "eval_rewards/accuracies": 0.523,
      "eval_rewards/chosen": -0.17783968531899155,
      "eval_rewards/margins": 0.07608089716732502,
      "eval_rewards/rejected": -0.2539205798432231,
      "eval_runtime": 204.6975,
      "eval_samples_per_second": 4.885,
      "eval_steps_per_second": 1.221,
      "step": 50
    },
    {
      "entropy": 2.3940737187862395,
      "epoch": 0.88,
      "grad_norm": 48.0695915222168,
      "learning_rate": 3.119414452281158e-07,
      "logits/chosen": 0.07112344903443206,
      "logits/rejected": 0.19535127089682655,
      "logps/chosen": -150.24068050384523,
      "logps/rejected": -145.48797540664674,
      "loss": 0.697087287902832,
      "mean_token_accuracy": 0.4701634831726551,
      "num_tokens": 731475.0,
      "rewards/accuracies": 0.56875,
      "rewards/chosen": -0.14963094238191843,
      "rewards/margins": 0.08578414842486382,
      "rewards/rejected": -0.2354150922037661,
      "step": 55
    },
    {
      "entropy": 2.357463392615318,
      "epoch": 0.96,
      "grad_norm": 48.10399627685547,
      "learning_rate": 6.268021954544095e-08,
      "logits/chosen": 0.01995007363434443,
      "logits/rejected": 0.16687794622622143,
      "logps/chosen": -158.93942432403566,
      "logps/rejected": -166.66946430206298,
      "loss": 0.6701779842376709,
      "mean_token_accuracy": 0.4689980365335941,
      "num_tokens": 797757.0,
      "rewards/accuracies": 0.5375,
      "rewards/chosen": -0.1631853505037725,
      "rewards/margins": 0.13551276298239828,
      "rewards/rejected": -0.2986981191672385,
      "step": 60
    },
    {
      "epoch": 1.0,
      "eval_entropy": 2.431350471973419,
      "eval_logits/chosen": 0.058027013522972216,
      "eval_logits/rejected": 0.11281541237759458,
      "eval_logps/chosen": -163.3362089691162,
      "eval_logps/rejected": -155.8263526611328,
      "eval_loss": 0.6985306143760681,
      "eval_mean_token_accuracy": 0.45935997557640074,
      "eval_num_tokens": 829784.0,
      "eval_rewards/accuracies": 0.529,
      "eval_rewards/chosen": -0.188840083822608,
      "eval_rewards/margins": 0.07824680726230145,
      "eval_rewards/rejected": -0.26708688925206664,
      "eval_runtime": 204.726,
      "eval_samples_per_second": 4.885,
      "eval_steps_per_second": 1.221,
      "step": 63
    },
    {
      "entropy": 2.402912601828575,
      "epoch": 1.032,
      "grad_norm": 32.69980239868164,
      "learning_rate": 4.184239109116393e-06,
      "logits/chosen": 0.05448504114962113,
      "logits/rejected": 0.09865282891809334,
      "logps/chosen": -146.60927033424377,
      "logps/rejected": -172.2928113937378,
      "loss": 0.3366573750972748,
      "mean_token_accuracy": 0.4740424118936062,
      "num_tokens": 27180.0,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.5033185752108693,
      "rewards/margins": 1.529684765264392,
      "rewards/rejected": -1.0263662077486515,
      "step": 65
    },
    {
      "entropy": 2.3992972910404204,
      "epoch": 1.112,
      "grad_norm": 17.76884651184082,
      "learning_rate": 4.006586590948141e-06,
      "logits/chosen": 0.0026862717521520412,
      "logits/rejected": 0.13353262495170462,
      "logps/chosen": -153.0885206222534,
      "logps/rejected": -156.96604356765747,
      "loss": 0.3243739604949951,
      "mean_token_accuracy": 0.47915840819478034,
      "num_tokens": 86611.0,
      "rewards/accuracies": 0.8625,
      "rewards/chosen": 0.4796073419041932,
      "rewards/margins": 1.6072665184736252,
      "rewards/rejected": -1.1276591904461384,
      "step": 70
    },
    {
      "entropy": 2.3780136823654177,
      "epoch": 1.192,
      "grad_norm": 22.48740005493164,
      "learning_rate": 3.81608040719339e-06,
      "logits/chosen": -0.11707000613882948,
      "logits/rejected": -0.06640926333257077,
      "logps/chosen": -153.8634229660034,
      "logps/rejected": -163.86640148162843,
      "loss": 0.35583436489105225,
      "mean_token_accuracy": 0.4692669503390789,
      "num_tokens": 153056.0,
      "rewards/accuracies": 0.83125,
      "rewards/chosen": 0.23908968223258853,
      "rewards/margins": 1.5137944724410772,
      "rewards/rejected": -1.2747048027813435,
      "step": 75
    },
    {
      "entropy": 2.310171273350716,
      "epoch": 1.272,
      "grad_norm": 19.922534942626953,
      "learning_rate": 3.6143458894413463e-06,
      "logits/chosen": -0.3582654699758253,
      "logits/rejected": -0.2914110555931979,
      "logps/chosen": -131.75142288208008,
      "logps/rejected": -160.25545768737794,
      "loss": 0.33200573921203613,
      "mean_token_accuracy": 0.4669229932129383,
      "num_tokens": 215920.0,
      "rewards/accuracies": 0.8875,
      "rewards/chosen": 0.15461167003959417,
      "rewards/margins": 1.571430729702115,
      "rewards/rejected": -1.4168190509080887,
      "step": 80
    },
    {
      "entropy": 2.211333890259266,
      "epoch": 1.3519999999999999,
      "grad_norm": 26.273008346557617,
      "learning_rate": 3.403104165467883e-06,
      "logits/chosen": -0.4359116041083054,
      "logits/rejected": -0.3527264562702821,
      "logps/chosen": -140.573939037323,
      "logps/rejected": -165.2755883216858,
      "loss": 0.3167553186416626,
      "mean_token_accuracy": 0.49101626574993135,
      "num_tokens": 287127.0,
      "rewards/accuracies": 0.86875,
      "rewards/chosen": 0.22212664941325783,
      "rewards/margins": 1.8355195850133896,
      "rewards/rejected": -1.6133929178118707,
      "step": 85
    },
    {
      "entropy": 2.284930866956711,
      "epoch": 1.432,
      "grad_norm": 23.699182510375977,
      "learning_rate": 3.184157475180208e-06,
      "logits/chosen": -0.49555035845434314,
      "logits/rejected": -0.4411274731208847,
      "logps/chosen": -159.97724952697754,
      "logps/rejected": -158.18697261810303,
      "loss": 0.31732380390167236,
      "mean_token_accuracy": 0.46273171752691267,
      "num_tokens": 354068.0,
      "rewards/accuracies": 0.9,
      "rewards/chosen": 0.2169143119826913,
      "rewards/margins": 1.6415132291615009,
      "rewards/rejected": -1.4245989315211773,
      "step": 90
    },
    {
      "entropy": 2.185201385617256,
      "epoch": 1.512,
      "grad_norm": 19.721302032470703,
      "learning_rate": 2.9593737945414264e-06,
      "logits/chosen": -0.6350667553105876,
      "logits/rejected": -0.6234260760266512,
      "logps/chosen": -140.15507984161377,
      "logps/rejected": -165.66675672531127,
      "loss": 0.3336572885513306,
      "mean_token_accuracy": 0.46677022203803065,
      "num_tokens": 421426.0,
      "rewards/accuracies": 0.8375,
      "rewards/chosen": 0.1235144454985857,
      "rewards/margins": 1.6951029762625693,
      "rewards/rejected": -1.5715885123237967,
      "step": 95
    },
    {
      "entropy": 2.116330775618553,
      "epoch": 1.592,
      "grad_norm": 33.37752151489258,
      "learning_rate": 2.730670898658255e-06,
      "logits/chosen": -0.8269758479064379,
      "logits/rejected": -0.6836210629893789,
      "logps/chosen": -157.39115772247314,
      "logps/rejected": -160.14211597442628,
      "loss": 0.33512029647827146,
      "mean_token_accuracy": 0.4874211989343166,
      "num_tokens": 489806.0,
      "rewards/accuracies": 0.8875,
      "rewards/chosen": 0.32783279549330474,
      "rewards/margins": 1.7466752111911774,
      "rewards/rejected": -1.4188424333930016,
      "step": 100
    },
    {
      "epoch": 1.592,
      "eval_entropy": 2.1636376428604125,
      "eval_logits/chosen": -0.8136097147525514,
      "eval_logits/rejected": -0.7600604176985916,
      "eval_logps/chosen": -167.09777592468262,
      "eval_logps/rejected": -159.95339458465577,
      "eval_loss": 0.7532036304473877,
      "eval_mean_token_accuracy": 0.4550632082223892,
      "eval_num_tokens": 489806.0,
      "eval_rewards/accuracies": 0.531,
      "eval_rewards/chosen": -0.5649967772588134,
      "eval_rewards/margins": 0.1147943808734417,
      "eval_rewards/rejected": -0.679791161686182,
      "eval_runtime": 204.516,
      "eval_samples_per_second": 4.89,
      "eval_steps_per_second": 1.222,
      "step": 100
    }
  ],
  "logging_steps": 5,
  "max_steps": 189,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 5237912290590720.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}