6404 lines
224 KiB
JSON
6404 lines
224 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.6155917425310937,
|
||
|
|
"eval_steps": 10,
|
||
|
|
"global_step": 700,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.004615976407231696,
|
||
|
|
"grad_norm": 60.83765068742266,
|
||
|
|
"learning_rate": 1.1494252873563218e-08,
|
||
|
|
"logits/chosen": 0.4711977541446686,
|
||
|
|
"logits/rejected": 0.4847034811973572,
|
||
|
|
"logps/chosen": -41.84939193725586,
|
||
|
|
"logps/rejected": -44.508792877197266,
|
||
|
|
"loss": 0.6931,
|
||
|
|
"rewards/accuracies": 0.0,
|
||
|
|
"rewards/chosen": 0.0,
|
||
|
|
"rewards/margins": 0.0,
|
||
|
|
"rewards/rejected": 0.0,
|
||
|
|
"step": 2
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.009231952814463392,
|
||
|
|
"grad_norm": 89.49857504360673,
|
||
|
|
"learning_rate": 2.2988505747126436e-08,
|
||
|
|
"logits/chosen": 0.4102262556552887,
|
||
|
|
"logits/rejected": 0.4489870071411133,
|
||
|
|
"logps/chosen": -33.33359909057617,
|
||
|
|
"logps/rejected": -48.11466979980469,
|
||
|
|
"loss": 0.6965,
|
||
|
|
"rewards/accuracies": 0.4722222089767456,
|
||
|
|
"rewards/chosen": 0.016991177573800087,
|
||
|
|
"rewards/margins": -0.00249446090310812,
|
||
|
|
"rewards/rejected": 0.01948563940823078,
|
||
|
|
"step": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.01384792922169509,
|
||
|
|
"grad_norm": 91.503921376188,
|
||
|
|
"learning_rate": 3.448275862068965e-08,
|
||
|
|
"logits/chosen": 0.4212642312049866,
|
||
|
|
"logits/rejected": 0.448761522769928,
|
||
|
|
"logps/chosen": -39.75364685058594,
|
||
|
|
"logps/rejected": -51.98044967651367,
|
||
|
|
"loss": 0.7058,
|
||
|
|
"rewards/accuracies": 0.4305555522441864,
|
||
|
|
"rewards/chosen": 0.01352924108505249,
|
||
|
|
"rewards/margins": -0.021095700562000275,
|
||
|
|
"rewards/rejected": 0.034624941647052765,
|
||
|
|
"step": 6
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.018463905628926785,
|
||
|
|
"grad_norm": 74.67414376851612,
|
||
|
|
"learning_rate": 4.597701149425287e-08,
|
||
|
|
"logits/chosen": 0.3533351719379425,
|
||
|
|
"logits/rejected": 0.38716739416122437,
|
||
|
|
"logps/chosen": -42.66749954223633,
|
||
|
|
"logps/rejected": -59.93525695800781,
|
||
|
|
"loss": 0.682,
|
||
|
|
"rewards/accuracies": 0.5972222089767456,
|
||
|
|
"rewards/chosen": 0.15689438581466675,
|
||
|
|
"rewards/margins": 0.02920585870742798,
|
||
|
|
"rewards/rejected": 0.12768852710723877,
|
||
|
|
"step": 8
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.023079882036158482,
|
||
|
|
"grad_norm": 68.70984683419653,
|
||
|
|
"learning_rate": 5.747126436781609e-08,
|
||
|
|
"logits/chosen": 0.49728691577911377,
|
||
|
|
"logits/rejected": 0.5158182978630066,
|
||
|
|
"logps/chosen": -40.442108154296875,
|
||
|
|
"logps/rejected": -47.894962310791016,
|
||
|
|
"loss": 0.6784,
|
||
|
|
"rewards/accuracies": 0.5833333134651184,
|
||
|
|
"rewards/chosen": 0.20257243514060974,
|
||
|
|
"rewards/margins": 0.033093564212322235,
|
||
|
|
"rewards/rejected": 0.1694788932800293,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.023079882036158482,
|
||
|
|
"eval_logits/chosen": 0.3284655511379242,
|
||
|
|
"eval_logits/rejected": 0.3523290753364563,
|
||
|
|
"eval_logps/chosen": -41.368160247802734,
|
||
|
|
"eval_logps/rejected": -47.68316650390625,
|
||
|
|
"eval_loss": 0.6900005340576172,
|
||
|
|
"eval_rewards/accuracies": 0.5040322542190552,
|
||
|
|
"eval_rewards/chosen": 0.18856020271778107,
|
||
|
|
"eval_rewards/margins": 0.010975954122841358,
|
||
|
|
"eval_rewards/rejected": 0.1775842159986496,
|
||
|
|
"eval_runtime": 223.5149,
|
||
|
|
"eval_samples_per_second": 7.758,
|
||
|
|
"eval_steps_per_second": 1.942,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.02769585844339018,
|
||
|
|
"grad_norm": 83.08821357925031,
|
||
|
|
"learning_rate": 6.89655172413793e-08,
|
||
|
|
"logits/chosen": 0.39168137311935425,
|
||
|
|
"logits/rejected": 0.428312748670578,
|
||
|
|
"logps/chosen": -40.189659118652344,
|
||
|
|
"logps/rejected": -55.229732513427734,
|
||
|
|
"loss": 0.6825,
|
||
|
|
"rewards/accuracies": 0.5138888955116272,
|
||
|
|
"rewards/chosen": 0.10889428108930588,
|
||
|
|
"rewards/margins": 0.025906018912792206,
|
||
|
|
"rewards/rejected": 0.08298826217651367,
|
||
|
|
"step": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.032311834850621876,
|
||
|
|
"grad_norm": 83.2788469670015,
|
||
|
|
"learning_rate": 8.045977011494252e-08,
|
||
|
|
"logits/chosen": 0.4244603216648102,
|
||
|
|
"logits/rejected": 0.45606857538223267,
|
||
|
|
"logps/chosen": -45.81875228881836,
|
||
|
|
"logps/rejected": -59.79555130004883,
|
||
|
|
"loss": 0.707,
|
||
|
|
"rewards/accuracies": 0.5277777910232544,
|
||
|
|
"rewards/chosen": 0.0007680323324166238,
|
||
|
|
"rewards/margins": -0.02245757356286049,
|
||
|
|
"rewards/rejected": 0.023225605487823486,
|
||
|
|
"step": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03692781125785357,
|
||
|
|
"grad_norm": 65.27464739827121,
|
||
|
|
"learning_rate": 9.195402298850574e-08,
|
||
|
|
"logits/chosen": 0.43778783082962036,
|
||
|
|
"logits/rejected": 0.47771337628364563,
|
||
|
|
"logps/chosen": -33.643489837646484,
|
||
|
|
"logps/rejected": -47.315940856933594,
|
||
|
|
"loss": 0.6907,
|
||
|
|
"rewards/accuracies": 0.4861111044883728,
|
||
|
|
"rewards/chosen": 0.16155114769935608,
|
||
|
|
"rewards/margins": 0.009526676498353481,
|
||
|
|
"rewards/rejected": 0.15202444791793823,
|
||
|
|
"step": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04154378766508527,
|
||
|
|
"grad_norm": 60.46601141846051,
|
||
|
|
"learning_rate": 1.0344827586206897e-07,
|
||
|
|
"logits/chosen": 0.4576772153377533,
|
||
|
|
"logits/rejected": 0.4669303894042969,
|
||
|
|
"logps/chosen": -49.01601791381836,
|
||
|
|
"logps/rejected": -44.165489196777344,
|
||
|
|
"loss": 0.7024,
|
||
|
|
"rewards/accuracies": 0.4166666567325592,
|
||
|
|
"rewards/chosen": 0.1639019399881363,
|
||
|
|
"rewards/margins": -0.013081331737339497,
|
||
|
|
"rewards/rejected": 0.1769832819700241,
|
||
|
|
"step": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.046159764072316964,
|
||
|
|
"grad_norm": 79.31933330847342,
|
||
|
|
"learning_rate": 1.1494252873563217e-07,
|
||
|
|
"logits/chosen": 0.40101033449172974,
|
||
|
|
"logits/rejected": 0.4429229199886322,
|
||
|
|
"logps/chosen": -42.295860290527344,
|
||
|
|
"logps/rejected": -61.62363052368164,
|
||
|
|
"loss": 0.6993,
|
||
|
|
"rewards/accuracies": 0.5694444179534912,
|
||
|
|
"rewards/chosen": 0.1744026243686676,
|
||
|
|
"rewards/margins": -0.005564332008361816,
|
||
|
|
"rewards/rejected": 0.17996692657470703,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.046159764072316964,
|
||
|
|
"eval_logits/chosen": 0.3300890624523163,
|
||
|
|
"eval_logits/rejected": 0.3539319634437561,
|
||
|
|
"eval_logps/chosen": -41.36879348754883,
|
||
|
|
"eval_logps/rejected": -47.67192840576172,
|
||
|
|
"eval_loss": 0.6927710771560669,
|
||
|
|
"eval_rewards/accuracies": 0.4694700539112091,
|
||
|
|
"eval_rewards/chosen": 0.1882432997226715,
|
||
|
|
"eval_rewards/margins": 0.005040565971285105,
|
||
|
|
"eval_rewards/rejected": 0.18320275843143463,
|
||
|
|
"eval_runtime": 220.5959,
|
||
|
|
"eval_samples_per_second": 7.861,
|
||
|
|
"eval_steps_per_second": 1.967,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05077574047954866,
|
||
|
|
"grad_norm": 74.95614553584633,
|
||
|
|
"learning_rate": 1.2643678160919542e-07,
|
||
|
|
"logits/chosen": 0.35644879937171936,
|
||
|
|
"logits/rejected": 0.39824995398521423,
|
||
|
|
"logps/chosen": -44.09666442871094,
|
||
|
|
"logps/rejected": -67.98532104492188,
|
||
|
|
"loss": 0.6849,
|
||
|
|
"rewards/accuracies": 0.4861111044883728,
|
||
|
|
"rewards/chosen": 0.061192478984594345,
|
||
|
|
"rewards/margins": 0.024405598640441895,
|
||
|
|
"rewards/rejected": 0.03678688034415245,
|
||
|
|
"step": 22
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05539171688678036,
|
||
|
|
"grad_norm": 59.95529358393387,
|
||
|
|
"learning_rate": 1.379310344827586e-07,
|
||
|
|
"logits/chosen": 0.4076593816280365,
|
||
|
|
"logits/rejected": 0.4187220335006714,
|
||
|
|
"logps/chosen": -50.34169006347656,
|
||
|
|
"logps/rejected": -52.33488464355469,
|
||
|
|
"loss": 0.6737,
|
||
|
|
"rewards/accuracies": 0.5972222089767456,
|
||
|
|
"rewards/chosen": 0.18035806715488434,
|
||
|
|
"rewards/margins": 0.046983275562524796,
|
||
|
|
"rewards/rejected": 0.13337479531764984,
|
||
|
|
"step": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06000769329401205,
|
||
|
|
"grad_norm": 58.82337491053465,
|
||
|
|
"learning_rate": 1.4942528735632184e-07,
|
||
|
|
"logits/chosen": 0.38400429487228394,
|
||
|
|
"logits/rejected": 0.3896331191062927,
|
||
|
|
"logps/chosen": -45.30482482910156,
|
||
|
|
"logps/rejected": -38.63485336303711,
|
||
|
|
"loss": 0.6927,
|
||
|
|
"rewards/accuracies": 0.5416666865348816,
|
||
|
|
"rewards/chosen": 0.12515152990818024,
|
||
|
|
"rewards/margins": 0.005594419315457344,
|
||
|
|
"rewards/rejected": 0.11955711245536804,
|
||
|
|
"step": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06462366970124375,
|
||
|
|
"grad_norm": 79.835671840217,
|
||
|
|
"learning_rate": 1.6091954022988505e-07,
|
||
|
|
"logits/chosen": 0.38133352994918823,
|
||
|
|
"logits/rejected": 0.4193841814994812,
|
||
|
|
"logps/chosen": -46.66801452636719,
|
||
|
|
"logps/rejected": -66.68572998046875,
|
||
|
|
"loss": 0.6817,
|
||
|
|
"rewards/accuracies": 0.5416666865348816,
|
||
|
|
"rewards/chosen": 0.1615171581506729,
|
||
|
|
"rewards/margins": 0.03017430752515793,
|
||
|
|
"rewards/rejected": 0.1313428282737732,
|
||
|
|
"step": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06923964610847544,
|
||
|
|
"grad_norm": 63.4551490148668,
|
||
|
|
"learning_rate": 1.7241379310344828e-07,
|
||
|
|
"logits/chosen": 0.38135284185409546,
|
||
|
|
"logits/rejected": 0.4095006585121155,
|
||
|
|
"logps/chosen": -40.06434631347656,
|
||
|
|
"logps/rejected": -49.53153610229492,
|
||
|
|
"loss": 0.6732,
|
||
|
|
"rewards/accuracies": 0.625,
|
||
|
|
"rewards/chosen": 0.18324324488639832,
|
||
|
|
"rewards/margins": 0.04405728355050087,
|
||
|
|
"rewards/rejected": 0.13918595016002655,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06923964610847544,
|
||
|
|
"eval_logits/chosen": 0.33052363991737366,
|
||
|
|
"eval_logits/rejected": 0.3544217050075531,
|
||
|
|
"eval_logps/chosen": -41.5091438293457,
|
||
|
|
"eval_logps/rejected": -47.86111068725586,
|
||
|
|
"eval_loss": 0.6812014579772949,
|
||
|
|
"eval_rewards/accuracies": 0.546658992767334,
|
||
|
|
"eval_rewards/chosen": 0.11806601285934448,
|
||
|
|
"eval_rewards/margins": 0.029455602169036865,
|
||
|
|
"eval_rewards/rejected": 0.08861041069030762,
|
||
|
|
"eval_runtime": 220.5898,
|
||
|
|
"eval_samples_per_second": 7.861,
|
||
|
|
"eval_steps_per_second": 1.967,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07385562251570714,
|
||
|
|
"grad_norm": 54.00177702879831,
|
||
|
|
"learning_rate": 1.839080459770115e-07,
|
||
|
|
"logits/chosen": 0.4309755563735962,
|
||
|
|
"logits/rejected": 0.45285335183143616,
|
||
|
|
"logps/chosen": -42.45962905883789,
|
||
|
|
"logps/rejected": -47.46916198730469,
|
||
|
|
"loss": 0.6778,
|
||
|
|
"rewards/accuracies": 0.5416666865348816,
|
||
|
|
"rewards/chosen": 0.059671804308891296,
|
||
|
|
"rewards/margins": 0.03700065612792969,
|
||
|
|
"rewards/rejected": 0.022671150043606758,
|
||
|
|
"step": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07847159892293884,
|
||
|
|
"grad_norm": 55.987754524641964,
|
||
|
|
"learning_rate": 1.9540229885057472e-07,
|
||
|
|
"logits/chosen": 0.3958838880062103,
|
||
|
|
"logits/rejected": 0.43136459589004517,
|
||
|
|
"logps/chosen": -37.61958694458008,
|
||
|
|
"logps/rejected": -52.296146392822266,
|
||
|
|
"loss": 0.6756,
|
||
|
|
"rewards/accuracies": 0.5138888955116272,
|
||
|
|
"rewards/chosen": 0.2221948355436325,
|
||
|
|
"rewards/margins": 0.04236772283911705,
|
||
|
|
"rewards/rejected": 0.17982712388038635,
|
||
|
|
"step": 34
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08308757533017054,
|
||
|
|
"grad_norm": 67.31410028619514,
|
||
|
|
"learning_rate": 2.0689655172413793e-07,
|
||
|
|
"logits/chosen": 0.44812121987342834,
|
||
|
|
"logits/rejected": 0.46431127190589905,
|
||
|
|
"logps/chosen": -42.98078155517578,
|
||
|
|
"logps/rejected": -41.65153884887695,
|
||
|
|
"loss": 0.6493,
|
||
|
|
"rewards/accuracies": 0.6388888955116272,
|
||
|
|
"rewards/chosen": 0.30534830689430237,
|
||
|
|
"rewards/margins": 0.09969804435968399,
|
||
|
|
"rewards/rejected": 0.20565026998519897,
|
||
|
|
"step": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08770355173740223,
|
||
|
|
"grad_norm": 57.904024813693326,
|
||
|
|
"learning_rate": 2.1839080459770114e-07,
|
||
|
|
"logits/chosen": 0.49128374457359314,
|
||
|
|
"logits/rejected": 0.5145975351333618,
|
||
|
|
"logps/chosen": -44.50560760498047,
|
||
|
|
"logps/rejected": -49.38070297241211,
|
||
|
|
"loss": 0.6628,
|
||
|
|
"rewards/accuracies": 0.5972222089767456,
|
||
|
|
"rewards/chosen": 0.18226391077041626,
|
||
|
|
"rewards/margins": 0.0719500482082367,
|
||
|
|
"rewards/rejected": 0.11031384021043777,
|
||
|
|
"step": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09231952814463393,
|
||
|
|
"grad_norm": 64.05496800782066,
|
||
|
|
"learning_rate": 2.2988505747126435e-07,
|
||
|
|
"logits/chosen": 0.46414005756378174,
|
||
|
|
"logits/rejected": 0.47909700870513916,
|
||
|
|
"logps/chosen": -45.80656433105469,
|
||
|
|
"logps/rejected": -48.13614273071289,
|
||
|
|
"loss": 0.6614,
|
||
|
|
"rewards/accuracies": 0.6388888955116272,
|
||
|
|
"rewards/chosen": 0.1019454374909401,
|
||
|
|
"rewards/margins": 0.07323868572711945,
|
||
|
|
"rewards/rejected": 0.028706755489110947,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09231952814463393,
|
||
|
|
"eval_logits/chosen": 0.3304091989994049,
|
||
|
|
"eval_logits/rejected": 0.35432368516921997,
|
||
|
|
"eval_logps/chosen": -41.51032638549805,
|
||
|
|
"eval_logps/rejected": -47.951194763183594,
|
||
|
|
"eval_loss": 0.6623325347900391,
|
||
|
|
"eval_rewards/accuracies": 0.5748847723007202,
|
||
|
|
"eval_rewards/chosen": 0.11747448146343231,
|
||
|
|
"eval_rewards/margins": 0.07390521466732025,
|
||
|
|
"eval_rewards/rejected": 0.04356926307082176,
|
||
|
|
"eval_runtime": 220.5888,
|
||
|
|
"eval_samples_per_second": 7.861,
|
||
|
|
"eval_steps_per_second": 1.967,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09693550455186563,
|
||
|
|
"grad_norm": 57.52177717893154,
|
||
|
|
"learning_rate": 2.413793103448276e-07,
|
||
|
|
"logits/chosen": 0.40689817070961,
|
||
|
|
"logits/rejected": 0.427402138710022,
|
||
|
|
"logps/chosen": -38.75439453125,
|
||
|
|
"logps/rejected": -44.31669235229492,
|
||
|
|
"loss": 0.6302,
|
||
|
|
"rewards/accuracies": 0.7222222089767456,
|
||
|
|
"rewards/chosen": 0.2862703502178192,
|
||
|
|
"rewards/margins": 0.14747940003871918,
|
||
|
|
"rewards/rejected": 0.13879093527793884,
|
||
|
|
"step": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10155148095909731,
|
||
|
|
"grad_norm": 64.43087177898828,
|
||
|
|
"learning_rate": 2.5287356321839084e-07,
|
||
|
|
"logits/chosen": 0.38502392172813416,
|
||
|
|
"logits/rejected": 0.42915642261505127,
|
||
|
|
"logps/chosen": -44.23611831665039,
|
||
|
|
"logps/rejected": -70.150634765625,
|
||
|
|
"loss": 0.6523,
|
||
|
|
"rewards/accuracies": 0.5277777910232544,
|
||
|
|
"rewards/chosen": 0.29370981454849243,
|
||
|
|
"rewards/margins": 0.11090421676635742,
|
||
|
|
"rewards/rejected": 0.18280558288097382,
|
||
|
|
"step": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10616745736632902,
|
||
|
|
"grad_norm": 60.64457511313282,
|
||
|
|
"learning_rate": 2.64367816091954e-07,
|
||
|
|
"logits/chosen": 0.4625084698200226,
|
||
|
|
"logits/rejected": 0.47940170764923096,
|
||
|
|
"logps/chosen": -47.40989685058594,
|
||
|
|
"logps/rejected": -50.2266731262207,
|
||
|
|
"loss": 0.6586,
|
||
|
|
"rewards/accuracies": 0.5416666865348816,
|
||
|
|
"rewards/chosen": 0.19395428895950317,
|
||
|
|
"rewards/margins": 0.09819034487009048,
|
||
|
|
"rewards/rejected": 0.09576395153999329,
|
||
|
|
"step": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11078343377356072,
|
||
|
|
"grad_norm": 48.97275141927136,
|
||
|
|
"learning_rate": 2.758620689655172e-07,
|
||
|
|
"logits/chosen": 0.40377330780029297,
|
||
|
|
"logits/rejected": 0.4251302480697632,
|
||
|
|
"logps/chosen": -40.91835021972656,
|
||
|
|
"logps/rejected": -46.69221878051758,
|
||
|
|
"loss": 0.6553,
|
||
|
|
"rewards/accuracies": 0.5277777910232544,
|
||
|
|
"rewards/chosen": 0.19001546502113342,
|
||
|
|
"rewards/margins": 0.11491294950246811,
|
||
|
|
"rewards/rejected": 0.07510250806808472,
|
||
|
|
"step": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1153994101807924,
|
||
|
|
"grad_norm": 50.405334242894924,
|
||
|
|
"learning_rate": 2.873563218390804e-07,
|
||
|
|
"logits/chosen": 0.42986366152763367,
|
||
|
|
"logits/rejected": 0.4425734579563141,
|
||
|
|
"logps/chosen": -45.240882873535156,
|
||
|
|
"logps/rejected": -45.33219528198242,
|
||
|
|
"loss": 0.6545,
|
||
|
|
"rewards/accuracies": 0.5972222089767456,
|
||
|
|
"rewards/chosen": 0.27720221877098083,
|
||
|
|
"rewards/margins": 0.10149689018726349,
|
||
|
|
"rewards/rejected": 0.17570529878139496,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1153994101807924,
|
||
|
|
"eval_logits/chosen": 0.3320508301258087,
|
||
|
|
"eval_logits/rejected": 0.35591834783554077,
|
||
|
|
"eval_logps/chosen": -41.36655807495117,
|
||
|
|
"eval_logps/rejected": -48.028892517089844,
|
||
|
|
"eval_loss": 0.6255878210067749,
|
||
|
|
"eval_rewards/accuracies": 0.6278801560401917,
|
||
|
|
"eval_rewards/chosen": 0.1893603652715683,
|
||
|
|
"eval_rewards/margins": 0.18464061617851257,
|
||
|
|
"eval_rewards/rejected": 0.004719759337604046,
|
||
|
|
"eval_runtime": 220.5277,
|
||
|
|
"eval_samples_per_second": 7.863,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1200153865880241,
|
||
|
|
"grad_norm": 59.45036930086866,
|
||
|
|
"learning_rate": 2.988505747126437e-07,
|
||
|
|
"logits/chosen": 0.4412320852279663,
|
||
|
|
"logits/rejected": 0.47745391726493835,
|
||
|
|
"logps/chosen": -38.808204650878906,
|
||
|
|
"logps/rejected": -57.61214828491211,
|
||
|
|
"loss": 0.6523,
|
||
|
|
"rewards/accuracies": 0.5277777910232544,
|
||
|
|
"rewards/chosen": 0.24403540790081024,
|
||
|
|
"rewards/margins": 0.14384596049785614,
|
||
|
|
"rewards/rejected": 0.10018942505121231,
|
||
|
|
"step": 52
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1246313629952558,
|
||
|
|
"grad_norm": 54.50350019894718,
|
||
|
|
"learning_rate": 3.103448275862069e-07,
|
||
|
|
"logits/chosen": 0.305615097284317,
|
||
|
|
"logits/rejected": 0.3378358781337738,
|
||
|
|
"logps/chosen": -41.46311950683594,
|
||
|
|
"logps/rejected": -55.873138427734375,
|
||
|
|
"loss": 0.6345,
|
||
|
|
"rewards/accuracies": 0.5694444179534912,
|
||
|
|
"rewards/chosen": 0.2176976054906845,
|
||
|
|
"rewards/margins": 0.17212893068790436,
|
||
|
|
"rewards/rejected": 0.045568663626909256,
|
||
|
|
"step": 54
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1292473394024875,
|
||
|
|
"grad_norm": 49.044811443941626,
|
||
|
|
"learning_rate": 3.218390804597701e-07,
|
||
|
|
"logits/chosen": 0.4806426763534546,
|
||
|
|
"logits/rejected": 0.5007810592651367,
|
||
|
|
"logps/chosen": -37.00300216674805,
|
||
|
|
"logps/rejected": -42.795040130615234,
|
||
|
|
"loss": 0.6005,
|
||
|
|
"rewards/accuracies": 0.6944444179534912,
|
||
|
|
"rewards/chosen": 0.41853082180023193,
|
||
|
|
"rewards/margins": 0.23157899081707,
|
||
|
|
"rewards/rejected": 0.18695180118083954,
|
||
|
|
"step": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1338633158097192,
|
||
|
|
"grad_norm": 54.19272761171978,
|
||
|
|
"learning_rate": 3.333333333333333e-07,
|
||
|
|
"logits/chosen": 0.4073159098625183,
|
||
|
|
"logits/rejected": 0.4315372109413147,
|
||
|
|
"logps/chosen": -39.63461685180664,
|
||
|
|
"logps/rejected": -41.75359344482422,
|
||
|
|
"loss": 0.5767,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.488341748714447,
|
||
|
|
"rewards/margins": 0.2964838743209839,
|
||
|
|
"rewards/rejected": 0.19185791909694672,
|
||
|
|
"step": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13847929221695088,
|
||
|
|
"grad_norm": 45.35161413256781,
|
||
|
|
"learning_rate": 3.4482758620689656e-07,
|
||
|
|
"logits/chosen": 0.3869187831878662,
|
||
|
|
"logits/rejected": 0.4154462218284607,
|
||
|
|
"logps/chosen": -40.21774673461914,
|
||
|
|
"logps/rejected": -49.05698013305664,
|
||
|
|
"loss": 0.5939,
|
||
|
|
"rewards/accuracies": 0.6944444179534912,
|
||
|
|
"rewards/chosen": 0.4058813452720642,
|
||
|
|
"rewards/margins": 0.2916874289512634,
|
||
|
|
"rewards/rejected": 0.11419390141963959,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13847929221695088,
|
||
|
|
"eval_logits/chosen": 0.3351307511329651,
|
||
|
|
"eval_logits/rejected": 0.35898876190185547,
|
||
|
|
"eval_logps/chosen": -40.8883171081543,
|
||
|
|
"eval_logps/rejected": -47.73066711425781,
|
||
|
|
"eval_loss": 0.5981891751289368,
|
||
|
|
"eval_rewards/accuracies": 0.6745391488075256,
|
||
|
|
"eval_rewards/chosen": 0.4284805655479431,
|
||
|
|
"eval_rewards/margins": 0.2746467888355255,
|
||
|
|
"eval_rewards/rejected": 0.15383380651474,
|
||
|
|
"eval_runtime": 220.5776,
|
||
|
|
"eval_samples_per_second": 7.861,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1430952686241826,
|
||
|
|
"grad_norm": 43.3905484769897,
|
||
|
|
"learning_rate": 3.5632183908045977e-07,
|
||
|
|
"logits/chosen": 0.4458725154399872,
|
||
|
|
"logits/rejected": 0.46262863278388977,
|
||
|
|
"logps/chosen": -40.7205924987793,
|
||
|
|
"logps/rejected": -47.21548080444336,
|
||
|
|
"loss": 0.6053,
|
||
|
|
"rewards/accuracies": 0.6666666865348816,
|
||
|
|
"rewards/chosen": 0.42287477850914,
|
||
|
|
"rewards/margins": 0.2391357719898224,
|
||
|
|
"rewards/rejected": 0.18373897671699524,
|
||
|
|
"step": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14771124503141428,
|
||
|
|
"grad_norm": 46.32576522285691,
|
||
|
|
"learning_rate": 3.67816091954023e-07,
|
||
|
|
"logits/chosen": 0.42775771021842957,
|
||
|
|
"logits/rejected": 0.4581214487552643,
|
||
|
|
"logps/chosen": -42.59015655517578,
|
||
|
|
"logps/rejected": -51.6392822265625,
|
||
|
|
"loss": 0.5669,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.43508684635162354,
|
||
|
|
"rewards/margins": 0.3611146807670593,
|
||
|
|
"rewards/rejected": 0.0739721804857254,
|
||
|
|
"step": 64
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.152327221438646,
|
||
|
|
"grad_norm": 42.51968356117282,
|
||
|
|
"learning_rate": 3.793103448275862e-07,
|
||
|
|
"logits/chosen": 0.4170711636543274,
|
||
|
|
"logits/rejected": 0.45475757122039795,
|
||
|
|
"logps/chosen": -38.8193359375,
|
||
|
|
"logps/rejected": -59.24808120727539,
|
||
|
|
"loss": 0.5514,
|
||
|
|
"rewards/accuracies": 0.7222222089767456,
|
||
|
|
"rewards/chosen": 0.5248206257820129,
|
||
|
|
"rewards/margins": 0.48520517349243164,
|
||
|
|
"rewards/rejected": 0.03961547836661339,
|
||
|
|
"step": 66
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15694319784587768,
|
||
|
|
"grad_norm": 59.1347578756685,
|
||
|
|
"learning_rate": 3.9080459770114945e-07,
|
||
|
|
"logits/chosen": 0.3444980978965759,
|
||
|
|
"logits/rejected": 0.38142290711402893,
|
||
|
|
"logps/chosen": -37.63268280029297,
|
||
|
|
"logps/rejected": -56.55868911743164,
|
||
|
|
"loss": 0.6421,
|
||
|
|
"rewards/accuracies": 0.6805555820465088,
|
||
|
|
"rewards/chosen": 0.28356897830963135,
|
||
|
|
"rewards/margins": 0.37217438220977783,
|
||
|
|
"rewards/rejected": -0.0886053591966629,
|
||
|
|
"step": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16155917425310937,
|
||
|
|
"grad_norm": 43.01512643240851,
|
||
|
|
"learning_rate": 4.0229885057471266e-07,
|
||
|
|
"logits/chosen": 0.47459664940834045,
|
||
|
|
"logits/rejected": 0.5048218369483948,
|
||
|
|
"logps/chosen": -37.06074905395508,
|
||
|
|
"logps/rejected": -41.83311462402344,
|
||
|
|
"loss": 0.4985,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.7238032221794128,
|
||
|
|
"rewards/margins": 0.6140663623809814,
|
||
|
|
"rewards/rejected": 0.10973668098449707,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16155917425310937,
|
||
|
|
"eval_logits/chosen": 0.3373754024505615,
|
||
|
|
"eval_logits/rejected": 0.361337274312973,
|
||
|
|
"eval_logps/chosen": -40.72093200683594,
|
||
|
|
"eval_logps/rejected": -47.9627685546875,
|
||
|
|
"eval_loss": 0.5673334002494812,
|
||
|
|
"eval_rewards/accuracies": 0.7073732614517212,
|
||
|
|
"eval_rewards/chosen": 0.512172520160675,
|
||
|
|
"eval_rewards/margins": 0.47438928484916687,
|
||
|
|
"eval_rewards/rejected": 0.03778325766324997,
|
||
|
|
"eval_runtime": 220.4667,
|
||
|
|
"eval_samples_per_second": 7.865,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16617515066034108,
|
||
|
|
"grad_norm": 58.03313651952633,
|
||
|
|
"learning_rate": 4.1379310344827586e-07,
|
||
|
|
"logits/chosen": 0.47328370809555054,
|
||
|
|
"logits/rejected": 0.516916036605835,
|
||
|
|
"logps/chosen": -39.381927490234375,
|
||
|
|
"logps/rejected": -63.04606628417969,
|
||
|
|
"loss": 0.5446,
|
||
|
|
"rewards/accuracies": 0.7222222089767456,
|
||
|
|
"rewards/chosen": 0.5364831686019897,
|
||
|
|
"rewards/margins": 0.5998459458351135,
|
||
|
|
"rewards/rejected": -0.06336280703544617,
|
||
|
|
"step": 72
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17079112706757277,
|
||
|
|
"grad_norm": 38.72118579621577,
|
||
|
|
"learning_rate": 4.25287356321839e-07,
|
||
|
|
"logits/chosen": 0.4743606746196747,
|
||
|
|
"logits/rejected": 0.48414406180381775,
|
||
|
|
"logps/chosen": -47.13395690917969,
|
||
|
|
"logps/rejected": -47.23988723754883,
|
||
|
|
"loss": 0.6296,
|
||
|
|
"rewards/accuracies": 0.6944444179534912,
|
||
|
|
"rewards/chosen": 0.46243613958358765,
|
||
|
|
"rewards/margins": 0.4000816345214844,
|
||
|
|
"rewards/rejected": 0.06235449016094208,
|
||
|
|
"step": 74
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17540710347480445,
|
||
|
|
"grad_norm": 52.02457163056824,
|
||
|
|
"learning_rate": 4.367816091954023e-07,
|
||
|
|
"logits/chosen": 0.4869605302810669,
|
||
|
|
"logits/rejected": 0.5183277726173401,
|
||
|
|
"logps/chosen": -41.5470085144043,
|
||
|
|
"logps/rejected": -52.64150619506836,
|
||
|
|
"loss": 0.5554,
|
||
|
|
"rewards/accuracies": 0.7361111044883728,
|
||
|
|
"rewards/chosen": 0.6152575612068176,
|
||
|
|
"rewards/margins": 0.5033391118049622,
|
||
|
|
"rewards/rejected": 0.11191850155591965,
|
||
|
|
"step": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18002307988203617,
|
||
|
|
"grad_norm": 39.942306949985145,
|
||
|
|
"learning_rate": 4.482758620689655e-07,
|
||
|
|
"logits/chosen": 0.4678427278995514,
|
||
|
|
"logits/rejected": 0.4919649660587311,
|
||
|
|
"logps/chosen": -36.33488845825195,
|
||
|
|
"logps/rejected": -46.28294372558594,
|
||
|
|
"loss": 0.5509,
|
||
|
|
"rewards/accuracies": 0.6944444179534912,
|
||
|
|
"rewards/chosen": 0.7630440592765808,
|
||
|
|
"rewards/margins": 0.5350204110145569,
|
||
|
|
"rewards/rejected": 0.22802363336086273,
|
||
|
|
"step": 78
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18463905628926786,
|
||
|
|
"grad_norm": 49.68265630941736,
|
||
|
|
"learning_rate": 4.597701149425287e-07,
|
||
|
|
"logits/chosen": 0.4118601381778717,
|
||
|
|
"logits/rejected": 0.4340742528438568,
|
||
|
|
"logps/chosen": -36.466026306152344,
|
||
|
|
"logps/rejected": -40.359230041503906,
|
||
|
|
"loss": 0.5161,
|
||
|
|
"rewards/accuracies": 0.7222222089767456,
|
||
|
|
"rewards/chosen": 0.7488323450088501,
|
||
|
|
"rewards/margins": 0.6430253982543945,
|
||
|
|
"rewards/rejected": 0.1058068722486496,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18463905628926786,
|
||
|
|
"eval_logits/chosen": 0.3424670994281769,
|
||
|
|
"eval_logits/rejected": 0.3665529489517212,
|
||
|
|
"eval_logps/chosen": -40.603668212890625,
|
||
|
|
"eval_logps/rejected": -48.121337890625,
|
||
|
|
"eval_loss": 0.5314013957977295,
|
||
|
|
"eval_rewards/accuracies": 0.7206221222877502,
|
||
|
|
"eval_rewards/chosen": 0.5708039999008179,
|
||
|
|
"eval_rewards/margins": 0.6123039126396179,
|
||
|
|
"eval_rewards/rejected": -0.04149990156292915,
|
||
|
|
"eval_runtime": 220.269,
|
||
|
|
"eval_samples_per_second": 7.872,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18925503269649954,
|
||
|
|
"grad_norm": 40.18854063526523,
|
||
|
|
"learning_rate": 4.712643678160919e-07,
|
||
|
|
"logits/chosen": 0.4146896302700043,
|
||
|
|
"logits/rejected": 0.44372716546058655,
|
||
|
|
"logps/chosen": -44.112205505371094,
|
||
|
|
"logps/rejected": -54.97979736328125,
|
||
|
|
"loss": 0.5066,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.5255990624427795,
|
||
|
|
"rewards/margins": 0.6622204780578613,
|
||
|
|
"rewards/rejected": -0.1366213709115982,
|
||
|
|
"step": 82
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19387100910373126,
|
||
|
|
"grad_norm": 35.8737356471814,
|
||
|
|
"learning_rate": 4.827586206896552e-07,
|
||
|
|
"logits/chosen": 0.46901315450668335,
|
||
|
|
"logits/rejected": 0.5202505588531494,
|
||
|
|
"logps/chosen": -37.11308288574219,
|
||
|
|
"logps/rejected": -64.51854705810547,
|
||
|
|
"loss": 0.4791,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.6412889957427979,
|
||
|
|
"rewards/margins": 0.8545607924461365,
|
||
|
|
"rewards/rejected": -0.213271826505661,
|
||
|
|
"step": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19848698551096294,
|
||
|
|
"grad_norm": 38.06201763208911,
|
||
|
|
"learning_rate": 4.942528735632184e-07,
|
||
|
|
"logits/chosen": 0.4869195520877838,
|
||
|
|
"logits/rejected": 0.5158190727233887,
|
||
|
|
"logps/chosen": -41.02754592895508,
|
||
|
|
"logps/rejected": -52.270511627197266,
|
||
|
|
"loss": 0.4592,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.6756889820098877,
|
||
|
|
"rewards/margins": 0.815551221370697,
|
||
|
|
"rewards/rejected": -0.13986223936080933,
|
||
|
|
"step": 86
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20310296191819463,
|
||
|
|
"grad_norm": 36.34414677933188,
|
||
|
|
"learning_rate": 4.999979670146248e-07,
|
||
|
|
"logits/chosen": 0.4226466119289398,
|
||
|
|
"logits/rejected": 0.4440222680568695,
|
||
|
|
"logps/chosen": -45.02897644042969,
|
||
|
|
"logps/rejected": -53.814674377441406,
|
||
|
|
"loss": 0.4665,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.5262306928634644,
|
||
|
|
"rewards/margins": 0.772502601146698,
|
||
|
|
"rewards/rejected": -0.24627192318439484,
|
||
|
|
"step": 88
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20771893832542634,
|
||
|
|
"grad_norm": 47.62232189356859,
|
||
|
|
"learning_rate": 4.99981703330008e-07,
|
||
|
|
"logits/chosen": 0.43458905816078186,
|
||
|
|
"logits/rejected": 0.45631253719329834,
|
||
|
|
"logps/chosen": -39.44232177734375,
|
||
|
|
"logps/rejected": -49.5074462890625,
|
||
|
|
"loss": 0.508,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.6604294180870056,
|
||
|
|
"rewards/margins": 0.6735664010047913,
|
||
|
|
"rewards/rejected": -0.013136889785528183,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20771893832542634,
|
||
|
|
"eval_logits/chosen": 0.3509339988231659,
|
||
|
|
"eval_logits/rejected": 0.37502503395080566,
|
||
|
|
"eval_logps/chosen": -40.31688690185547,
|
||
|
|
"eval_logps/rejected": -48.16210174560547,
|
||
|
|
"eval_loss": 0.4914422631263733,
|
||
|
|
"eval_rewards/accuracies": 0.7263824939727783,
|
||
|
|
"eval_rewards/chosen": 0.7141958475112915,
|
||
|
|
"eval_rewards/margins": 0.7760785222053528,
|
||
|
|
"eval_rewards/rejected": -0.06188271939754486,
|
||
|
|
"eval_runtime": 220.2045,
|
||
|
|
"eval_samples_per_second": 7.874,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21233491473265803,
|
||
|
|
"grad_norm": 33.53538998473167,
|
||
|
|
"learning_rate": 4.99949177018813e-07,
|
||
|
|
"logits/chosen": 0.4293578863143921,
|
||
|
|
"logits/rejected": 0.46211349964141846,
|
||
|
|
"logps/chosen": -34.20891571044922,
|
||
|
|
"logps/rejected": -45.82402420043945,
|
||
|
|
"loss": 0.4007,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.9528428316116333,
|
||
|
|
"rewards/margins": 1.0176244974136353,
|
||
|
|
"rewards/rejected": -0.06478171050548553,
|
||
|
|
"step": 92
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21695089113988972,
|
||
|
|
"grad_norm": 47.45137697847349,
|
||
|
|
"learning_rate": 4.999003901970474e-07,
|
||
|
|
"logits/chosen": 0.4385245442390442,
|
||
|
|
"logits/rejected": 0.45108774304389954,
|
||
|
|
"logps/chosen": -47.24710464477539,
|
||
|
|
"logps/rejected": -47.30147171020508,
|
||
|
|
"loss": 0.5534,
|
||
|
|
"rewards/accuracies": 0.6944444179534912,
|
||
|
|
"rewards/chosen": 0.644627034664154,
|
||
|
|
"rewards/margins": 0.6343204975128174,
|
||
|
|
"rewards/rejected": 0.010306484065949917,
|
||
|
|
"step": 94
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22156686754712143,
|
||
|
|
"grad_norm": 33.39629568865361,
|
||
|
|
"learning_rate": 4.998353460385512e-07,
|
||
|
|
"logits/chosen": 0.4504711329936981,
|
||
|
|
"logits/rejected": 0.48663392663002014,
|
||
|
|
"logps/chosen": -40.03446578979492,
|
||
|
|
"logps/rejected": -55.506591796875,
|
||
|
|
"loss": 0.4222,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.6967446804046631,
|
||
|
|
"rewards/margins": 1.0778069496154785,
|
||
|
|
"rewards/rejected": -0.381062388420105,
|
||
|
|
"step": 96
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22618284395435312,
|
||
|
|
"grad_norm": 34.18594601316725,
|
||
|
|
"learning_rate": 4.997540487747892e-07,
|
||
|
|
"logits/chosen": 0.38444679975509644,
|
||
|
|
"logits/rejected": 0.4130491614341736,
|
||
|
|
"logps/chosen": -37.72957992553711,
|
||
|
|
"logps/rejected": -57.71113967895508,
|
||
|
|
"loss": 0.4716,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.864948570728302,
|
||
|
|
"rewards/margins": 1.0170652866363525,
|
||
|
|
"rewards/rejected": -0.152116596698761,
|
||
|
|
"step": 98
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2307988203615848,
|
||
|
|
"grad_norm": 31.852168197293704,
|
||
|
|
"learning_rate": 4.996565036945769e-07,
|
||
|
|
"logits/chosen": 0.4658397436141968,
|
||
|
|
"logits/rejected": 0.4849558472633362,
|
||
|
|
"logps/chosen": -44.069618225097656,
|
||
|
|
"logps/rejected": -46.06491470336914,
|
||
|
|
"loss": 0.4924,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.5598255395889282,
|
||
|
|
"rewards/margins": 0.8147852420806885,
|
||
|
|
"rewards/rejected": -0.25495976209640503,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2307988203615848,
|
||
|
|
"eval_logits/chosen": 0.3590577244758606,
|
||
|
|
"eval_logits/rejected": 0.38313183188438416,
|
||
|
|
"eval_logps/chosen": -40.04033660888672,
|
||
|
|
"eval_logps/rejected": -48.23354721069336,
|
||
|
|
"eval_loss": 0.4618569314479828,
|
||
|
|
"eval_rewards/accuracies": 0.7298387289047241,
|
||
|
|
"eval_rewards/chosen": 0.852470874786377,
|
||
|
|
"eval_rewards/margins": 0.9500778913497925,
|
||
|
|
"eval_rewards/rejected": -0.09760700911283493,
|
||
|
|
"eval_runtime": 220.4716,
|
||
|
|
"eval_samples_per_second": 7.865,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23541479676881652,
|
||
|
|
"grad_norm": 32.563251146586694,
|
||
|
|
"learning_rate": 4.995427171437356e-07,
|
||
|
|
"logits/chosen": 0.41394177079200745,
|
||
|
|
"logits/rejected": 0.4560126066207886,
|
||
|
|
"logps/chosen": -36.68212890625,
|
||
|
|
"logps/rejected": -56.006553649902344,
|
||
|
|
"loss": 0.3851,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.7943739891052246,
|
||
|
|
"rewards/margins": 1.1945956945419312,
|
||
|
|
"rewards/rejected": -0.40022173523902893,
|
||
|
|
"step": 102
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2400307731760482,
|
||
|
|
"grad_norm": 35.159104202159625,
|
||
|
|
"learning_rate": 4.994126965246796e-07,
|
||
|
|
"logits/chosen": 0.43339937925338745,
|
||
|
|
"logits/rejected": 0.45789778232574463,
|
||
|
|
"logps/chosen": -40.00631332397461,
|
||
|
|
"logps/rejected": -48.161224365234375,
|
||
|
|
"loss": 0.4153,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.7441484928131104,
|
||
|
|
"rewards/margins": 1.0307202339172363,
|
||
|
|
"rewards/rejected": -0.28657177090644836,
|
||
|
|
"step": 104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24464674958327992,
|
||
|
|
"grad_norm": 35.54279884741835,
|
||
|
|
"learning_rate": 4.992664502959351e-07,
|
||
|
|
"logits/chosen": 0.42503511905670166,
|
||
|
|
"logits/rejected": 0.48626741766929626,
|
||
|
|
"logps/chosen": -36.73310852050781,
|
||
|
|
"logps/rejected": -73.78736877441406,
|
||
|
|
"loss": 0.3536,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.8739730715751648,
|
||
|
|
"rewards/margins": 1.558694839477539,
|
||
|
|
"rewards/rejected": -0.6847219467163086,
|
||
|
|
"step": 106
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2492627259905116,
|
||
|
|
"grad_norm": 45.173611856138976,
|
||
|
|
"learning_rate": 4.991039879715898e-07,
|
||
|
|
"logits/chosen": 0.4289478361606598,
|
||
|
|
"logits/rejected": 0.46912992000579834,
|
||
|
|
"logps/chosen": -40.94606399536133,
|
||
|
|
"logps/rejected": -58.62925338745117,
|
||
|
|
"loss": 0.4057,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 1.023528814315796,
|
||
|
|
"rewards/margins": 1.251956582069397,
|
||
|
|
"rewards/rejected": -0.22842761874198914,
|
||
|
|
"step": 108
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2538787023977433,
|
||
|
|
"grad_norm": 25.213187587591246,
|
||
|
|
"learning_rate": 4.989253201206736e-07,
|
||
|
|
"logits/chosen": 0.4647282361984253,
|
||
|
|
"logits/rejected": 0.4716295003890991,
|
||
|
|
"logps/chosen": -40.334922790527344,
|
||
|
|
"logps/rejected": -41.65603256225586,
|
||
|
|
"loss": 0.4339,
|
||
|
|
"rewards/accuracies": 0.7222222089767456,
|
||
|
|
"rewards/chosen": 0.9747889637947083,
|
||
|
|
"rewards/margins": 1.0528353452682495,
|
||
|
|
"rewards/rejected": -0.07804636657238007,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2538787023977433,
|
||
|
|
"eval_logits/chosen": 0.36145398020744324,
|
||
|
|
"eval_logits/rejected": 0.38587653636932373,
|
||
|
|
"eval_logps/chosen": -39.77558135986328,
|
||
|
|
"eval_logps/rejected": -48.301231384277344,
|
||
|
|
"eval_loss": 0.43463748693466187,
|
||
|
|
"eval_rewards/accuracies": 0.7379032373428345,
|
||
|
|
"eval_rewards/chosen": 0.9848493337631226,
|
||
|
|
"eval_rewards/margins": 1.1162999868392944,
|
||
|
|
"eval_rewards/rejected": -0.1314505934715271,
|
||
|
|
"eval_runtime": 220.4446,
|
||
|
|
"eval_samples_per_second": 7.866,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.258494678804975,
|
||
|
|
"grad_norm": 39.895524747857486,
|
||
|
|
"learning_rate": 4.987304583664712e-07,
|
||
|
|
"logits/chosen": 0.4972270429134369,
|
||
|
|
"logits/rejected": 0.5156663060188293,
|
||
|
|
"logps/chosen": -46.859134674072266,
|
||
|
|
"logps/rejected": -53.12602996826172,
|
||
|
|
"loss": 0.4463,
|
||
|
|
"rewards/accuracies": 0.7361111044883728,
|
||
|
|
"rewards/chosen": 0.8810398578643799,
|
||
|
|
"rewards/margins": 0.9829990863800049,
|
||
|
|
"rewards/rejected": -0.10195919126272202,
|
||
|
|
"step": 112
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.26311065521220667,
|
||
|
|
"grad_norm": 36.88032065773003,
|
||
|
|
"learning_rate": 4.985194153857662e-07,
|
||
|
|
"logits/chosen": 0.4386284351348877,
|
||
|
|
"logits/rejected": 0.4557953476905823,
|
||
|
|
"logps/chosen": -36.74658203125,
|
||
|
|
"logps/rejected": -39.56464767456055,
|
||
|
|
"loss": 0.4788,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.9255303144454956,
|
||
|
|
"rewards/margins": 0.9156983494758606,
|
||
|
|
"rewards/rejected": 0.009831971488893032,
|
||
|
|
"step": 114
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2677266316194384,
|
||
|
|
"grad_norm": 23.636821560598456,
|
||
|
|
"learning_rate": 4.982922049080163e-07,
|
||
|
|
"logits/chosen": 0.40630775690078735,
|
||
|
|
"logits/rejected": 0.4236665964126587,
|
||
|
|
"logps/chosen": -35.141971588134766,
|
||
|
|
"logps/rejected": -42.14583969116211,
|
||
|
|
"loss": 0.3872,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.8281899690628052,
|
||
|
|
"rewards/margins": 1.215153455734253,
|
||
|
|
"rewards/rejected": -0.386963427066803,
|
||
|
|
"step": 116
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2723426080266701,
|
||
|
|
"grad_norm": 38.873691089935114,
|
||
|
|
"learning_rate": 4.980488417144599e-07,
|
||
|
|
"logits/chosen": 0.37884485721588135,
|
||
|
|
"logits/rejected": 0.4280329644680023,
|
||
|
|
"logps/chosen": -41.57583999633789,
|
||
|
|
"logps/rejected": -71.53160095214844,
|
||
|
|
"loss": 0.4818,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.7444247603416443,
|
||
|
|
"rewards/margins": 1.1956461668014526,
|
||
|
|
"rewards/rejected": -0.4512213468551636,
|
||
|
|
"step": 118
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27695858443390176,
|
||
|
|
"grad_norm": 27.126567445081033,
|
||
|
|
"learning_rate": 4.977893416371544e-07,
|
||
|
|
"logits/chosen": 0.4753592908382416,
|
||
|
|
"logits/rejected": 0.4997613728046417,
|
||
|
|
"logps/chosen": -34.07433319091797,
|
||
|
|
"logps/rejected": -45.33045959472656,
|
||
|
|
"loss": 0.3838,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.8865776062011719,
|
||
|
|
"rewards/margins": 1.4225349426269531,
|
||
|
|
"rewards/rejected": -0.5359571576118469,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27695858443390176,
|
||
|
|
"eval_logits/chosen": 0.3657575249671936,
|
||
|
|
"eval_logits/rejected": 0.39033937454223633,
|
||
|
|
"eval_logps/chosen": -39.95652770996094,
|
||
|
|
"eval_logps/rejected": -48.739437103271484,
|
||
|
|
"eval_loss": 0.410579651594162,
|
||
|
|
"eval_rewards/accuracies": 0.7540322542190552,
|
||
|
|
"eval_rewards/chosen": 0.8943750858306885,
|
||
|
|
"eval_rewards/margins": 1.2449262142181396,
|
||
|
|
"eval_rewards/rejected": -0.35055097937583923,
|
||
|
|
"eval_runtime": 220.2442,
|
||
|
|
"eval_samples_per_second": 7.873,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28157456084113347,
|
||
|
|
"grad_norm": 26.815456443053705,
|
||
|
|
"learning_rate": 4.975137215579469e-07,
|
||
|
|
"logits/chosen": 0.5420396327972412,
|
||
|
|
"logits/rejected": 0.5500521659851074,
|
||
|
|
"logps/chosen": -45.788516998291016,
|
||
|
|
"logps/rejected": -42.21580505371094,
|
||
|
|
"loss": 0.4117,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.8019249439239502,
|
||
|
|
"rewards/margins": 1.2268595695495605,
|
||
|
|
"rewards/rejected": -0.42493465542793274,
|
||
|
|
"step": 122
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2861905372483652,
|
||
|
|
"grad_norm": 30.749785890404876,
|
||
|
|
"learning_rate": 4.972219994073755e-07,
|
||
|
|
"logits/chosen": 0.49169254302978516,
|
||
|
|
"logits/rejected": 0.5404393672943115,
|
||
|
|
"logps/chosen": -38.644107818603516,
|
||
|
|
"logps/rejected": -67.01266479492188,
|
||
|
|
"loss": 0.3844,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.8940033316612244,
|
||
|
|
"rewards/margins": 1.6317830085754395,
|
||
|
|
"rewards/rejected": -0.7377796173095703,
|
||
|
|
"step": 124
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2908065136555969,
|
||
|
|
"grad_norm": 29.538977791375373,
|
||
|
|
"learning_rate": 4.969141941635025e-07,
|
||
|
|
"logits/chosen": 0.47598233819007874,
|
||
|
|
"logits/rejected": 0.5060492753982544,
|
||
|
|
"logps/chosen": -40.60331344604492,
|
||
|
|
"logps/rejected": -59.37862014770508,
|
||
|
|
"loss": 0.4476,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.5469496250152588,
|
||
|
|
"rewards/margins": 1.4448275566101074,
|
||
|
|
"rewards/rejected": -0.8978776931762695,
|
||
|
|
"step": 126
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.29542249006282856,
|
||
|
|
"grad_norm": 50.663011631161446,
|
||
|
|
"learning_rate": 4.965903258506806e-07,
|
||
|
|
"logits/chosen": 0.49228647351264954,
|
||
|
|
"logits/rejected": 0.5329996943473816,
|
||
|
|
"logps/chosen": -39.90941619873047,
|
||
|
|
"logps/rejected": -61.23884963989258,
|
||
|
|
"loss": 0.347,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.8301137685775757,
|
||
|
|
"rewards/margins": 1.5421695709228516,
|
||
|
|
"rewards/rejected": -0.7120558619499207,
|
||
|
|
"step": 128
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30003846647006027,
|
||
|
|
"grad_norm": 32.98345370505989,
|
||
|
|
"learning_rate": 4.962504155382493e-07,
|
||
|
|
"logits/chosen": 0.4239842891693115,
|
||
|
|
"logits/rejected": 0.44136151671409607,
|
||
|
|
"logps/chosen": -36.07121276855469,
|
||
|
|
"logps/rejected": -41.06203079223633,
|
||
|
|
"loss": 0.3667,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.7973310351371765,
|
||
|
|
"rewards/margins": 1.2334084510803223,
|
||
|
|
"rewards/rejected": -0.4360772669315338,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30003846647006027,
|
||
|
|
"eval_logits/chosen": 0.3723231256008148,
|
||
|
|
"eval_logits/rejected": 0.3968786299228668,
|
||
|
|
"eval_logps/chosen": -39.925048828125,
|
||
|
|
"eval_logps/rejected": -48.9533576965332,
|
||
|
|
"eval_loss": 0.39173391461372375,
|
||
|
|
"eval_rewards/accuracies": 0.7753456234931946,
|
||
|
|
"eval_rewards/chosen": 0.9101160168647766,
|
||
|
|
"eval_rewards/margins": 1.3676302433013916,
|
||
|
|
"eval_rewards/rejected": -0.4575144052505493,
|
||
|
|
"eval_runtime": 220.267,
|
||
|
|
"eval_samples_per_second": 7.872,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.304654442877292,
|
||
|
|
"grad_norm": 28.392702162901728,
|
||
|
|
"learning_rate": 4.958944853391652e-07,
|
||
|
|
"logits/chosen": 0.520796537399292,
|
||
|
|
"logits/rejected": 0.5420558452606201,
|
||
|
|
"logps/chosen": -37.87763595581055,
|
||
|
|
"logps/rejected": -46.05318069458008,
|
||
|
|
"loss": 0.3819,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.932469367980957,
|
||
|
|
"rewards/margins": 1.2907413244247437,
|
||
|
|
"rewards/rejected": -0.3582719564437866,
|
||
|
|
"step": 132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30927041928452365,
|
||
|
|
"grad_norm": 27.83688192223066,
|
||
|
|
"learning_rate": 4.955225584085624e-07,
|
||
|
|
"logits/chosen": 0.42395105957984924,
|
||
|
|
"logits/rejected": 0.44882073998451233,
|
||
|
|
"logps/chosen": -36.98991775512695,
|
||
|
|
"logps/rejected": -51.79054260253906,
|
||
|
|
"loss": 0.3951,
|
||
|
|
"rewards/accuracies": 0.7638888955116272,
|
||
|
|
"rewards/chosen": 0.9578195214271545,
|
||
|
|
"rewards/margins": 1.4403272867202759,
|
||
|
|
"rewards/rejected": -0.48250770568847656,
|
||
|
|
"step": 134
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31388639569175536,
|
||
|
|
"grad_norm": 27.432482792006017,
|
||
|
|
"learning_rate": 4.951346589422467e-07,
|
||
|
|
"logits/chosen": 0.483965128660202,
|
||
|
|
"logits/rejected": 0.5153691172599792,
|
||
|
|
"logps/chosen": -37.48245620727539,
|
||
|
|
"logps/rejected": -54.50342559814453,
|
||
|
|
"loss": 0.3942,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 1.0384331941604614,
|
||
|
|
"rewards/margins": 1.5820738077163696,
|
||
|
|
"rewards/rejected": -0.5436408519744873,
|
||
|
|
"step": 136
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3185023720989871,
|
||
|
|
"grad_norm": 46.62557646275611,
|
||
|
|
"learning_rate": 4.94730812175122e-07,
|
||
|
|
"logits/chosen": 0.43841731548309326,
|
||
|
|
"logits/rejected": 0.4499746561050415,
|
||
|
|
"logps/chosen": -38.93119812011719,
|
||
|
|
"logps/rejected": -42.26424026489258,
|
||
|
|
"loss": 0.4384,
|
||
|
|
"rewards/accuracies": 0.6527777910232544,
|
||
|
|
"rewards/chosen": 0.8961164951324463,
|
||
|
|
"rewards/margins": 1.247178554534912,
|
||
|
|
"rewards/rejected": -0.3510621190071106,
|
||
|
|
"step": 138
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32311834850621873,
|
||
|
|
"grad_norm": 34.05743924648359,
|
||
|
|
"learning_rate": 4.943110443795476e-07,
|
||
|
|
"logits/chosen": 0.49757227301597595,
|
||
|
|
"logits/rejected": 0.5091323852539062,
|
||
|
|
"logps/chosen": -42.93407440185547,
|
||
|
|
"logps/rejected": -45.01084899902344,
|
||
|
|
"loss": 0.4061,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.8557752370834351,
|
||
|
|
"rewards/margins": 1.3424351215362549,
|
||
|
|
"rewards/rejected": -0.48665979504585266,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32311834850621873,
|
||
|
|
"eval_logits/chosen": 0.3763599395751953,
|
||
|
|
"eval_logits/rejected": 0.4011194705963135,
|
||
|
|
"eval_logps/chosen": -39.798316955566406,
|
||
|
|
"eval_logps/rejected": -49.11299133300781,
|
||
|
|
"eval_loss": 0.3788905441761017,
|
||
|
|
"eval_rewards/accuracies": 0.764976978302002,
|
||
|
|
"eval_rewards/chosen": 0.9734821915626526,
|
||
|
|
"eval_rewards/margins": 1.5108132362365723,
|
||
|
|
"eval_rewards/rejected": -0.5373309850692749,
|
||
|
|
"eval_runtime": 220.3233,
|
||
|
|
"eval_samples_per_second": 7.87,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32773432491345045,
|
||
|
|
"grad_norm": 36.481001944632766,
|
||
|
|
"learning_rate": 4.938753828636297e-07,
|
||
|
|
"logits/chosen": 0.4888935089111328,
|
||
|
|
"logits/rejected": 0.4963880777359009,
|
||
|
|
"logps/chosen": -46.02848815917969,
|
||
|
|
"logps/rejected": -44.94346237182617,
|
||
|
|
"loss": 0.4382,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.7767104506492615,
|
||
|
|
"rewards/margins": 1.235382080078125,
|
||
|
|
"rewards/rejected": -0.45867156982421875,
|
||
|
|
"step": 142
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33235030132068216,
|
||
|
|
"grad_norm": 27.008693506029694,
|
||
|
|
"learning_rate": 4.934238559694447e-07,
|
||
|
|
"logits/chosen": 0.460690975189209,
|
||
|
|
"logits/rejected": 0.5057052969932556,
|
||
|
|
"logps/chosen": -38.473411560058594,
|
||
|
|
"logps/rejected": -54.91615295410156,
|
||
|
|
"loss": 0.3338,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.8698298335075378,
|
||
|
|
"rewards/margins": 1.6055673360824585,
|
||
|
|
"rewards/rejected": -0.7357374429702759,
|
||
|
|
"step": 144
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3369662777279138,
|
||
|
|
"grad_norm": 32.261266015848825,
|
||
|
|
"learning_rate": 4.929564930711957e-07,
|
||
|
|
"logits/chosen": 0.4295574426651001,
|
||
|
|
"logits/rejected": 0.4522492587566376,
|
||
|
|
"logps/chosen": -39.829490661621094,
|
||
|
|
"logps/rejected": -44.733333587646484,
|
||
|
|
"loss": 0.3533,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.7346515655517578,
|
||
|
|
"rewards/margins": 1.3469676971435547,
|
||
|
|
"rewards/rejected": -0.6123161315917969,
|
||
|
|
"step": 146
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34158225413514554,
|
||
|
|
"grad_norm": 28.797840444924386,
|
||
|
|
"learning_rate": 4.924733245733008e-07,
|
||
|
|
"logits/chosen": 0.5410254001617432,
|
||
|
|
"logits/rejected": 0.5485421419143677,
|
||
|
|
"logps/chosen": -43.81610870361328,
|
||
|
|
"logps/rejected": -40.52272033691406,
|
||
|
|
"loss": 0.3651,
|
||
|
|
"rewards/accuracies": 0.7638888955116272,
|
||
|
|
"rewards/chosen": 0.9063374996185303,
|
||
|
|
"rewards/margins": 1.2729685306549072,
|
||
|
|
"rewards/rejected": -0.366630882024765,
|
||
|
|
"step": 148
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34619823054237725,
|
||
|
|
"grad_norm": 30.202896827963542,
|
||
|
|
"learning_rate": 4.91974381908416e-07,
|
||
|
|
"logits/chosen": 0.42066994309425354,
|
||
|
|
"logits/rejected": 0.4589553475379944,
|
||
|
|
"logps/chosen": -38.81809997558594,
|
||
|
|
"logps/rejected": -58.59386444091797,
|
||
|
|
"loss": 0.3446,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.6800815463066101,
|
||
|
|
"rewards/margins": 1.928250789642334,
|
||
|
|
"rewards/rejected": -1.2481693029403687,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34619823054237725,
|
||
|
|
"eval_logits/chosen": 0.3821311295032501,
|
||
|
|
"eval_logits/rejected": 0.40684476494789124,
|
||
|
|
"eval_logps/chosen": -40.001861572265625,
|
||
|
|
"eval_logps/rejected": -49.4797477722168,
|
||
|
|
"eval_loss": 0.3633531332015991,
|
||
|
|
"eval_rewards/accuracies": 0.7724654674530029,
|
||
|
|
"eval_rewards/chosen": 0.8717083930969238,
|
||
|
|
"eval_rewards/margins": 1.5924171209335327,
|
||
|
|
"eval_rewards/rejected": -0.7207087278366089,
|
||
|
|
"eval_runtime": 220.1362,
|
||
|
|
"eval_samples_per_second": 7.877,
|
||
|
|
"eval_steps_per_second": 1.972,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3508142069496089,
|
||
|
|
"grad_norm": 26.385033894551455,
|
||
|
|
"learning_rate": 4.914596975353898e-07,
|
||
|
|
"logits/chosen": 0.4991176426410675,
|
||
|
|
"logits/rejected": 0.5242553353309631,
|
||
|
|
"logps/chosen": -38.974281311035156,
|
||
|
|
"logps/rejected": -48.54939270019531,
|
||
|
|
"loss": 0.3721,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.771294355392456,
|
||
|
|
"rewards/margins": 1.5243595838546753,
|
||
|
|
"rewards/rejected": -0.7530653476715088,
|
||
|
|
"step": 152
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3554301833568406,
|
||
|
|
"grad_norm": 42.428423927932826,
|
||
|
|
"learning_rate": 4.909293049371519e-07,
|
||
|
|
"logits/chosen": 0.5288230180740356,
|
||
|
|
"logits/rejected": 0.5352779626846313,
|
||
|
|
"logps/chosen": -45.90478515625,
|
||
|
|
"logps/rejected": -44.53614044189453,
|
||
|
|
"loss": 0.3542,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.7464312314987183,
|
||
|
|
"rewards/margins": 1.5150079727172852,
|
||
|
|
"rewards/rejected": -0.7685766220092773,
|
||
|
|
"step": 154
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36004615976407234,
|
||
|
|
"grad_norm": 36.75812549927479,
|
||
|
|
"learning_rate": 4.903832386185343e-07,
|
||
|
|
"logits/chosen": 0.47585126757621765,
|
||
|
|
"logits/rejected": 0.49040529131889343,
|
||
|
|
"logps/chosen": -44.172325134277344,
|
||
|
|
"logps/rejected": -43.98606872558594,
|
||
|
|
"loss": 0.3956,
|
||
|
|
"rewards/accuracies": 0.7361111044883728,
|
||
|
|
"rewards/chosen": 0.5973650813102722,
|
||
|
|
"rewards/margins": 1.340658187866211,
|
||
|
|
"rewards/rejected": -0.743293046951294,
|
||
|
|
"step": 156
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.364662136171304,
|
||
|
|
"grad_norm": 26.152211217958236,
|
||
|
|
"learning_rate": 4.89821534104028e-07,
|
||
|
|
"logits/chosen": 0.39484938979148865,
|
||
|
|
"logits/rejected": 0.42477357387542725,
|
||
|
|
"logps/chosen": -41.93134307861328,
|
||
|
|
"logps/rejected": -56.39106750488281,
|
||
|
|
"loss": 0.3275,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.827555239200592,
|
||
|
|
"rewards/margins": 1.9599329233169556,
|
||
|
|
"rewards/rejected": -1.1323776245117188,
|
||
|
|
"step": 158
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3692781125785357,
|
||
|
|
"grad_norm": 29.041350828980583,
|
||
|
|
"learning_rate": 4.892442279354698e-07,
|
||
|
|
"logits/chosen": 0.4744550287723541,
|
||
|
|
"logits/rejected": 0.5093830227851868,
|
||
|
|
"logps/chosen": -42.794578552246094,
|
||
|
|
"logps/rejected": -59.93064498901367,
|
||
|
|
"loss": 0.3605,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.540644645690918,
|
||
|
|
"rewards/margins": 1.6665728092193604,
|
||
|
|
"rewards/rejected": -1.125928282737732,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3692781125785357,
|
||
|
|
"eval_logits/chosen": 0.38920047879219055,
|
||
|
|
"eval_logits/rejected": 0.41388043761253357,
|
||
|
|
"eval_logps/chosen": -40.3745231628418,
|
||
|
|
"eval_logps/rejected": -49.94725036621094,
|
||
|
|
"eval_loss": 0.3510279059410095,
|
||
|
|
"eval_rewards/accuracies": 0.7920507192611694,
|
||
|
|
"eval_rewards/chosen": 0.6853779554367065,
|
||
|
|
"eval_rewards/margins": 1.6398398876190186,
|
||
|
|
"eval_rewards/rejected": -0.9544618129730225,
|
||
|
|
"eval_runtime": 220.1812,
|
||
|
|
"eval_samples_per_second": 7.875,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3738940889857674,
|
||
|
|
"grad_norm": 32.36067481486556,
|
||
|
|
"learning_rate": 4.886513576696673e-07,
|
||
|
|
"logits/chosen": 0.4680570960044861,
|
||
|
|
"logits/rejected": 0.5030277371406555,
|
||
|
|
"logps/chosen": -42.39280700683594,
|
||
|
|
"logps/rejected": -58.18678283691406,
|
||
|
|
"loss": 0.392,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.7217347621917725,
|
||
|
|
"rewards/margins": 1.8615412712097168,
|
||
|
|
"rewards/rejected": -1.1398065090179443,
|
||
|
|
"step": 162
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3785100653929991,
|
||
|
|
"grad_norm": 27.802667550507227,
|
||
|
|
"learning_rate": 4.880429618759543e-07,
|
||
|
|
"logits/chosen": 0.46893131732940674,
|
||
|
|
"logits/rejected": 0.4787411093711853,
|
||
|
|
"logps/chosen": -45.52459716796875,
|
||
|
|
"logps/rejected": -46.459312438964844,
|
||
|
|
"loss": 0.3819,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.870037317276001,
|
||
|
|
"rewards/margins": 1.4820109605789185,
|
||
|
|
"rewards/rejected": -0.6119736433029175,
|
||
|
|
"step": 164
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3831260418002308,
|
||
|
|
"grad_norm": 27.278325528930967,
|
||
|
|
"learning_rate": 4.874190801336817e-07,
|
||
|
|
"logits/chosen": 0.46610963344573975,
|
||
|
|
"logits/rejected": 0.4872422218322754,
|
||
|
|
"logps/chosen": -44.28363037109375,
|
||
|
|
"logps/rejected": -51.54701232910156,
|
||
|
|
"loss": 0.323,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.6502636075019836,
|
||
|
|
"rewards/margins": 1.7216179370880127,
|
||
|
|
"rewards/rejected": -1.0713541507720947,
|
||
|
|
"step": 166
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3877420182074625,
|
||
|
|
"grad_norm": 25.09454062223173,
|
||
|
|
"learning_rate": 4.867797530296431e-07,
|
||
|
|
"logits/chosen": 0.4582709074020386,
|
||
|
|
"logits/rejected": 0.48244646191596985,
|
||
|
|
"logps/chosen": -45.76988983154297,
|
||
|
|
"logps/rejected": -55.2458610534668,
|
||
|
|
"loss": 0.2842,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.6319215297698975,
|
||
|
|
"rewards/margins": 2.007154941558838,
|
||
|
|
"rewards/rejected": -1.3752332925796509,
|
||
|
|
"step": 168
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39235799461469417,
|
||
|
|
"grad_norm": 25.014228656107395,
|
||
|
|
"learning_rate": 4.861250221554343e-07,
|
||
|
|
"logits/chosen": 0.4760267436504364,
|
||
|
|
"logits/rejected": 0.5161222219467163,
|
||
|
|
"logps/chosen": -36.09988021850586,
|
||
|
|
"logps/rejected": -58.49198913574219,
|
||
|
|
"loss": 0.317,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.6937295794487,
|
||
|
|
"rewards/margins": 2.0070507526397705,
|
||
|
|
"rewards/rejected": -1.3133213520050049,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39235799461469417,
|
||
|
|
"eval_logits/chosen": 0.39130648970603943,
|
||
|
|
"eval_logits/rejected": 0.41622862219810486,
|
||
|
|
"eval_logps/chosen": -40.4833984375,
|
||
|
|
"eval_logps/rejected": -50.18775177001953,
|
||
|
|
"eval_loss": 0.343056857585907,
|
||
|
|
"eval_rewards/accuracies": 0.796658992767334,
|
||
|
|
"eval_rewards/chosen": 0.6309407949447632,
|
||
|
|
"eval_rewards/margins": 1.7056493759155273,
|
||
|
|
"eval_rewards/rejected": -1.0747085809707642,
|
||
|
|
"eval_runtime": 220.3261,
|
||
|
|
"eval_samples_per_second": 7.87,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3969739710219259,
|
||
|
|
"grad_norm": 21.660777806253456,
|
||
|
|
"learning_rate": 4.854549301047476e-07,
|
||
|
|
"logits/chosen": 0.5408195853233337,
|
||
|
|
"logits/rejected": 0.5565234422683716,
|
||
|
|
"logps/chosen": -42.90623474121094,
|
||
|
|
"logps/rejected": -43.590702056884766,
|
||
|
|
"loss": 0.373,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.6533936262130737,
|
||
|
|
"rewards/margins": 1.5095248222351074,
|
||
|
|
"rewards/rejected": -0.8561312556266785,
|
||
|
|
"step": 172
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4015899474291576,
|
||
|
|
"grad_norm": 32.27746768838142,
|
||
|
|
"learning_rate": 4.847695204706005e-07,
|
||
|
|
"logits/chosen": 0.47649839520454407,
|
||
|
|
"logits/rejected": 0.49190616607666016,
|
||
|
|
"logps/chosen": -38.49553680419922,
|
||
|
|
"logps/rejected": -40.65150451660156,
|
||
|
|
"loss": 0.3558,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.7918031811714172,
|
||
|
|
"rewards/margins": 1.4087355136871338,
|
||
|
|
"rewards/rejected": -0.6169323325157166,
|
||
|
|
"step": 174
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40620592383638926,
|
||
|
|
"grad_norm": 31.844706703711985,
|
||
|
|
"learning_rate": 4.840688378425e-07,
|
||
|
|
"logits/chosen": 0.5188453793525696,
|
||
|
|
"logits/rejected": 0.5562708973884583,
|
||
|
|
"logps/chosen": -46.135372161865234,
|
||
|
|
"logps/rejected": -56.292930603027344,
|
||
|
|
"loss": 0.261,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.7925480604171753,
|
||
|
|
"rewards/margins": 2.1678171157836914,
|
||
|
|
"rewards/rejected": -1.3752690553665161,
|
||
|
|
"step": 176
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.410821900243621,
|
||
|
|
"grad_norm": 26.376171346573187,
|
||
|
|
"learning_rate": 4.833529278035422e-07,
|
||
|
|
"logits/chosen": 0.357127845287323,
|
||
|
|
"logits/rejected": 0.4103134572505951,
|
||
|
|
"logps/chosen": -37.78556442260742,
|
||
|
|
"logps/rejected": -67.52072143554688,
|
||
|
|
"loss": 0.2899,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.9015005826950073,
|
||
|
|
"rewards/margins": 2.719820261001587,
|
||
|
|
"rewards/rejected": -1.81831955909729,
|
||
|
|
"step": 178
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4154378766508527,
|
||
|
|
"grad_norm": 26.0393680431703,
|
||
|
|
"learning_rate": 4.826218369274459e-07,
|
||
|
|
"logits/chosen": 0.4666251540184021,
|
||
|
|
"logits/rejected": 0.5160384178161621,
|
||
|
|
"logps/chosen": -39.356258392333984,
|
||
|
|
"logps/rejected": -62.83391571044922,
|
||
|
|
"loss": 0.3066,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.8675535917282104,
|
||
|
|
"rewards/margins": 2.234145164489746,
|
||
|
|
"rewards/rejected": -1.3665915727615356,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4154378766508527,
|
||
|
|
"eval_logits/chosen": 0.3935144245624542,
|
||
|
|
"eval_logits/rejected": 0.41844189167022705,
|
||
|
|
"eval_logps/chosen": -39.861793518066406,
|
||
|
|
"eval_logps/rejected": -49.855037689208984,
|
||
|
|
"eval_loss": 0.3321295380592346,
|
||
|
|
"eval_rewards/accuracies": 0.7926267385482788,
|
||
|
|
"eval_rewards/chosen": 0.941743791103363,
|
||
|
|
"eval_rewards/margins": 1.8500969409942627,
|
||
|
|
"eval_rewards/rejected": -0.9083530902862549,
|
||
|
|
"eval_runtime": 220.3176,
|
||
|
|
"eval_samples_per_second": 7.87,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42005385305808435,
|
||
|
|
"grad_norm": 23.061889635448846,
|
||
|
|
"learning_rate": 4.818756127755237e-07,
|
||
|
|
"logits/chosen": 0.49034425616264343,
|
||
|
|
"logits/rejected": 0.5069853663444519,
|
||
|
|
"logps/chosen": -37.846553802490234,
|
||
|
|
"logps/rejected": -41.30693817138672,
|
||
|
|
"loss": 0.2693,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 1.0121811628341675,
|
||
|
|
"rewards/margins": 1.8432265520095825,
|
||
|
|
"rewards/rejected": -0.831045389175415,
|
||
|
|
"step": 182
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42466982946531606,
|
||
|
|
"grad_norm": 22.17904586209137,
|
||
|
|
"learning_rate": 4.811143038935873e-07,
|
||
|
|
"logits/chosen": 0.5580455660820007,
|
||
|
|
"logits/rejected": 0.5748550295829773,
|
||
|
|
"logps/chosen": -42.32413101196289,
|
||
|
|
"logps/rejected": -46.0750732421875,
|
||
|
|
"loss": 0.3264,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 1.0455000400543213,
|
||
|
|
"rewards/margins": 1.93173086643219,
|
||
|
|
"rewards/rejected": -0.8862307667732239,
|
||
|
|
"step": 184
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4292858058725478,
|
||
|
|
"grad_norm": 30.29917055573095,
|
||
|
|
"learning_rate": 4.803379598087899e-07,
|
||
|
|
"logits/chosen": 0.5174715518951416,
|
||
|
|
"logits/rejected": 0.5311744213104248,
|
||
|
|
"logps/chosen": -40.50711441040039,
|
||
|
|
"logps/rejected": -40.298824310302734,
|
||
|
|
"loss": 0.316,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.9772326350212097,
|
||
|
|
"rewards/margins": 1.7723863124847412,
|
||
|
|
"rewards/rejected": -0.795153796672821,
|
||
|
|
"step": 186
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.43390178227977944,
|
||
|
|
"grad_norm": 40.38001852713412,
|
||
|
|
"learning_rate": 4.795466310264034e-07,
|
||
|
|
"logits/chosen": 0.42736437916755676,
|
||
|
|
"logits/rejected": 0.463912695646286,
|
||
|
|
"logps/chosen": -39.35895919799805,
|
||
|
|
"logps/rejected": -64.93545532226562,
|
||
|
|
"loss": 0.4185,
|
||
|
|
"rewards/accuracies": 0.7638888955116272,
|
||
|
|
"rewards/chosen": 0.5966134667396545,
|
||
|
|
"rewards/margins": 1.980704665184021,
|
||
|
|
"rewards/rejected": -1.3840913772583008,
|
||
|
|
"step": 188
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.43851775868701115,
|
||
|
|
"grad_norm": 17.949323810733784,
|
||
|
|
"learning_rate": 4.787403690265335e-07,
|
||
|
|
"logits/chosen": 0.5044853091239929,
|
||
|
|
"logits/rejected": 0.5284148454666138,
|
||
|
|
"logps/chosen": -39.47854995727539,
|
||
|
|
"logps/rejected": -49.92608642578125,
|
||
|
|
"loss": 0.3266,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 1.0101630687713623,
|
||
|
|
"rewards/margins": 1.9091652631759644,
|
||
|
|
"rewards/rejected": -0.8990020751953125,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.43851775868701115,
|
||
|
|
"eval_logits/chosen": 0.3972060978412628,
|
||
|
|
"eval_logits/rejected": 0.4221220314502716,
|
||
|
|
"eval_logps/chosen": -39.831119537353516,
|
||
|
|
"eval_logps/rejected": -50.09023666381836,
|
||
|
|
"eval_loss": 0.3243154287338257,
|
||
|
|
"eval_rewards/accuracies": 0.7914746403694153,
|
||
|
|
"eval_rewards/chosen": 0.9570826292037964,
|
||
|
|
"eval_rewards/margins": 1.9830337762832642,
|
||
|
|
"eval_rewards/rejected": -1.0259510278701782,
|
||
|
|
"eval_runtime": 220.3237,
|
||
|
|
"eval_samples_per_second": 7.87,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44313373509424286,
|
||
|
|
"grad_norm": 36.065620072852695,
|
||
|
|
"learning_rate": 4.779192262607702e-07,
|
||
|
|
"logits/chosen": 0.5138534903526306,
|
||
|
|
"logits/rejected": 0.544155478477478,
|
||
|
|
"logps/chosen": -43.310760498046875,
|
||
|
|
"logps/rejected": -59.56623840332031,
|
||
|
|
"loss": 0.3542,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.9537274837493896,
|
||
|
|
"rewards/margins": 2.111888885498047,
|
||
|
|
"rewards/rejected": -1.1581614017486572,
|
||
|
|
"step": 192
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4477497115014745,
|
||
|
|
"grad_norm": 24.653058016123207,
|
||
|
|
"learning_rate": 4.770832561487758e-07,
|
||
|
|
"logits/chosen": 0.4504295885562897,
|
||
|
|
"logits/rejected": 0.46597781777381897,
|
||
|
|
"logps/chosen": -41.51498794555664,
|
||
|
|
"logps/rejected": -43.07120132446289,
|
||
|
|
"loss": 0.2587,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.9096213579177856,
|
||
|
|
"rewards/margins": 2.131375551223755,
|
||
|
|
"rewards/rejected": -1.2217543125152588,
|
||
|
|
"step": 194
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45236568790870624,
|
||
|
|
"grad_norm": 36.95305184003922,
|
||
|
|
"learning_rate": 4.762325130748097e-07,
|
||
|
|
"logits/chosen": 0.5585076808929443,
|
||
|
|
"logits/rejected": 0.5717556476593018,
|
||
|
|
"logps/chosen": -47.50046920776367,
|
||
|
|
"logps/rejected": -44.811973571777344,
|
||
|
|
"loss": 0.3412,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.9956084489822388,
|
||
|
|
"rewards/margins": 1.8879083395004272,
|
||
|
|
"rewards/rejected": -0.8922999501228333,
|
||
|
|
"step": 196
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45698166431593795,
|
||
|
|
"grad_norm": 16.999205852011567,
|
||
|
|
"learning_rate": 4.7536705238418995e-07,
|
||
|
|
"logits/chosen": 0.47373294830322266,
|
||
|
|
"logits/rejected": 0.49137142300605774,
|
||
|
|
"logps/chosen": -42.69048309326172,
|
||
|
|
"logps/rejected": -50.26279067993164,
|
||
|
|
"loss": 0.275,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.8595349788665771,
|
||
|
|
"rewards/margins": 2.2136645317077637,
|
||
|
|
"rewards/rejected": -1.3541297912597656,
|
||
|
|
"step": 198
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4615976407231696,
|
||
|
|
"grad_norm": 33.06750404898565,
|
||
|
|
"learning_rate": 4.7448693037969336e-07,
|
||
|
|
"logits/chosen": 0.5136507749557495,
|
||
|
|
"logits/rejected": 0.527184247970581,
|
||
|
|
"logps/chosen": -41.794132232666016,
|
||
|
|
"logps/rejected": -48.2490119934082,
|
||
|
|
"loss": 0.2986,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.8771340847015381,
|
||
|
|
"rewards/margins": 1.9797013998031616,
|
||
|
|
"rewards/rejected": -1.102567195892334,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4615976407231696,
|
||
|
|
"eval_logits/chosen": 0.39842745661735535,
|
||
|
|
"eval_logits/rejected": 0.42355066537857056,
|
||
|
|
"eval_logps/chosen": -40.12582778930664,
|
||
|
|
"eval_logps/rejected": -50.502620697021484,
|
||
|
|
"eval_loss": 0.3160472810268402,
|
||
|
|
"eval_rewards/accuracies": 0.7978110313415527,
|
||
|
|
"eval_rewards/chosen": 0.8097268342971802,
|
||
|
|
"eval_rewards/margins": 2.041868209838867,
|
||
|
|
"eval_rewards/rejected": -1.2321414947509766,
|
||
|
|
"eval_runtime": 220.4769,
|
||
|
|
"eval_samples_per_second": 7.865,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4662136171304013,
|
||
|
|
"grad_norm": 27.6688088244827,
|
||
|
|
"learning_rate": 4.735922043178923e-07,
|
||
|
|
"logits/chosen": 0.5529847741127014,
|
||
|
|
"logits/rejected": 0.5818406939506531,
|
||
|
|
"logps/chosen": -42.29270553588867,
|
||
|
|
"logps/rejected": -57.84202575683594,
|
||
|
|
"loss": 0.2725,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.8104487061500549,
|
||
|
|
"rewards/margins": 2.3321969509124756,
|
||
|
|
"rewards/rejected": -1.521748423576355,
|
||
|
|
"step": 202
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47082959353763304,
|
||
|
|
"grad_norm": 23.404484719369563,
|
||
|
|
"learning_rate": 4.7268293240543017e-07,
|
||
|
|
"logits/chosen": 0.48225533962249756,
|
||
|
|
"logits/rejected": 0.5109025239944458,
|
||
|
|
"logps/chosen": -40.953433990478516,
|
||
|
|
"logps/rejected": -55.026153564453125,
|
||
|
|
"loss": 0.3435,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.8147386908531189,
|
||
|
|
"rewards/margins": 2.057671546936035,
|
||
|
|
"rewards/rejected": -1.2429331541061401,
|
||
|
|
"step": 204
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4754455699448647,
|
||
|
|
"grad_norm": 29.663210206611154,
|
||
|
|
"learning_rate": 4.717591737952344e-07,
|
||
|
|
"logits/chosen": 0.48208919167518616,
|
||
|
|
"logits/rejected": 0.517291247844696,
|
||
|
|
"logps/chosen": -36.30723190307617,
|
||
|
|
"logps/rejected": -54.3764533996582,
|
||
|
|
"loss": 0.3135,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.8137081861495972,
|
||
|
|
"rewards/margins": 2.101260185241699,
|
||
|
|
"rewards/rejected": -1.287551999092102,
|
||
|
|
"step": 206
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4800615463520964,
|
||
|
|
"grad_norm": 29.39474251364716,
|
||
|
|
"learning_rate": 4.7082098858266837e-07,
|
||
|
|
"logits/chosen": 0.48040205240249634,
|
||
|
|
"logits/rejected": 0.5284512042999268,
|
||
|
|
"logps/chosen": -31.84227180480957,
|
||
|
|
"logps/rejected": -61.47830581665039,
|
||
|
|
"loss": 0.3821,
|
||
|
|
"rewards/accuracies": 0.7638888955116272,
|
||
|
|
"rewards/chosen": 0.455925315618515,
|
||
|
|
"rewards/margins": 2.105367422103882,
|
||
|
|
"rewards/rejected": -1.649442195892334,
|
||
|
|
"step": 208
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4846775227593281,
|
||
|
|
"grad_norm": 15.879511628269139,
|
||
|
|
"learning_rate": 4.698684378016222e-07,
|
||
|
|
"logits/chosen": 0.4825616478919983,
|
||
|
|
"logits/rejected": 0.5131646394729614,
|
||
|
|
"logps/chosen": -43.97586441040039,
|
||
|
|
"logps/rejected": -58.62031936645508,
|
||
|
|
"loss": 0.271,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.6879336833953857,
|
||
|
|
"rewards/margins": 2.212796211242676,
|
||
|
|
"rewards/rejected": -1.5248624086380005,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4846775227593281,
|
||
|
|
"eval_logits/chosen": 0.40579578280448914,
|
||
|
|
"eval_logits/rejected": 0.43089571595191956,
|
||
|
|
"eval_logps/chosen": -40.55123519897461,
|
||
|
|
"eval_logps/rejected": -51.04283905029297,
|
||
|
|
"eval_loss": 0.3111670911312103,
|
||
|
|
"eval_rewards/accuracies": 0.804147481918335,
|
||
|
|
"eval_rewards/chosen": 0.597020149230957,
|
||
|
|
"eval_rewards/margins": 2.099271535873413,
|
||
|
|
"eval_rewards/rejected": -1.502251386642456,
|
||
|
|
"eval_runtime": 220.3759,
|
||
|
|
"eval_samples_per_second": 7.868,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48929349916655984,
|
||
|
|
"grad_norm": 33.220388342252136,
|
||
|
|
"learning_rate": 4.6890158342054174e-07,
|
||
|
|
"logits/chosen": 0.46122825145721436,
|
||
|
|
"logits/rejected": 0.48773014545440674,
|
||
|
|
"logps/chosen": -38.094722747802734,
|
||
|
|
"logps/rejected": -50.649871826171875,
|
||
|
|
"loss": 0.3288,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.5131194591522217,
|
||
|
|
"rewards/margins": 2.1884312629699707,
|
||
|
|
"rewards/rejected": -1.6753116846084595,
|
||
|
|
"step": 212
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4939094755737915,
|
||
|
|
"grad_norm": 27.37607169791161,
|
||
|
|
"learning_rate": 4.679204883383973e-07,
|
||
|
|
"logits/chosen": 0.45677465200424194,
|
||
|
|
"logits/rejected": 0.5006839632987976,
|
||
|
|
"logps/chosen": -36.343292236328125,
|
||
|
|
"logps/rejected": -65.76275634765625,
|
||
|
|
"loss": 0.301,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.5972538590431213,
|
||
|
|
"rewards/margins": 2.6644716262817383,
|
||
|
|
"rewards/rejected": -2.0672178268432617,
|
||
|
|
"step": 214
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4985254519810232,
|
||
|
|
"grad_norm": 28.712191033509406,
|
||
|
|
"learning_rate": 4.669252163805919e-07,
|
||
|
|
"logits/chosen": 0.48203393816947937,
|
||
|
|
"logits/rejected": 0.5129568576812744,
|
||
|
|
"logps/chosen": -40.263328552246094,
|
||
|
|
"logps/rejected": -53.96393966674805,
|
||
|
|
"loss": 0.3434,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.3674449920654297,
|
||
|
|
"rewards/margins": 2.094463348388672,
|
||
|
|
"rewards/rejected": -1.7270184755325317,
|
||
|
|
"step": 216
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5031414283882549,
|
||
|
|
"grad_norm": 21.430060439194165,
|
||
|
|
"learning_rate": 4.65915832294809e-07,
|
||
|
|
"logits/chosen": 0.5647565722465515,
|
||
|
|
"logits/rejected": 0.6052375435829163,
|
||
|
|
"logps/chosen": -37.24385070800781,
|
||
|
|
"logps/rejected": -58.28202438354492,
|
||
|
|
"loss": 0.2945,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.5437911748886108,
|
||
|
|
"rewards/margins": 2.518171787261963,
|
||
|
|
"rewards/rejected": -1.9743802547454834,
|
||
|
|
"step": 218
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5077574047954866,
|
||
|
|
"grad_norm": 24.194015322932014,
|
||
|
|
"learning_rate": 4.6489240174680026e-07,
|
||
|
|
"logits/chosen": 0.5365298390388489,
|
||
|
|
"logits/rejected": 0.5451048612594604,
|
||
|
|
"logps/chosen": -40.26055145263672,
|
||
|
|
"logps/rejected": -40.11984634399414,
|
||
|
|
"loss": 0.4064,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": 0.4349411725997925,
|
||
|
|
"rewards/margins": 1.4253244400024414,
|
||
|
|
"rewards/rejected": -0.9903832674026489,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5077574047954866,
|
||
|
|
"eval_logits/chosen": 0.40611767768859863,
|
||
|
|
"eval_logits/rejected": 0.43133166432380676,
|
||
|
|
"eval_logps/chosen": -40.628150939941406,
|
||
|
|
"eval_logps/rejected": -51.22826385498047,
|
||
|
|
"eval_loss": 0.30713996291160583,
|
||
|
|
"eval_rewards/accuracies": 0.8018433451652527,
|
||
|
|
"eval_rewards/chosen": 0.5585668087005615,
|
||
|
|
"eval_rewards/margins": 2.153529644012451,
|
||
|
|
"eval_rewards/rejected": -1.5949628353118896,
|
||
|
|
"eval_runtime": 220.3416,
|
||
|
|
"eval_samples_per_second": 7.87,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5123733812027182,
|
||
|
|
"grad_norm": 23.39715012730976,
|
||
|
|
"learning_rate": 4.638549913161138e-07,
|
||
|
|
"logits/chosen": 0.5600088834762573,
|
||
|
|
"logits/rejected": 0.5736495852470398,
|
||
|
|
"logps/chosen": -46.20627212524414,
|
||
|
|
"logps/rejected": -47.1099739074707,
|
||
|
|
"loss": 0.2227,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.7162383794784546,
|
||
|
|
"rewards/margins": 2.4795196056365967,
|
||
|
|
"rewards/rejected": -1.763281226158142,
|
||
|
|
"step": 222
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.51698935760995,
|
||
|
|
"grad_norm": 23.70013936518676,
|
||
|
|
"learning_rate": 4.6280366849176267e-07,
|
||
|
|
"logits/chosen": 0.553576648235321,
|
||
|
|
"logits/rejected": 0.5800661444664001,
|
||
|
|
"logps/chosen": -41.73429870605469,
|
||
|
|
"logps/rejected": -47.09934997558594,
|
||
|
|
"loss": 0.2708,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.6174063682556152,
|
||
|
|
"rewards/margins": 2.10538649559021,
|
||
|
|
"rewards/rejected": -1.4879801273345947,
|
||
|
|
"step": 224
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5216053340171817,
|
||
|
|
"grad_norm": 19.39438827436505,
|
||
|
|
"learning_rate": 4.6173850166783446e-07,
|
||
|
|
"logits/chosen": 0.5699052810668945,
|
||
|
|
"logits/rejected": 0.5908712148666382,
|
||
|
|
"logps/chosen": -40.74462127685547,
|
||
|
|
"logps/rejected": -53.7403450012207,
|
||
|
|
"loss": 0.2716,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.5502187609672546,
|
||
|
|
"rewards/margins": 2.0002176761627197,
|
||
|
|
"rewards/rejected": -1.4499988555908203,
|
||
|
|
"step": 226
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5262213104244133,
|
||
|
|
"grad_norm": 24.49934372199594,
|
||
|
|
"learning_rate": 4.606595601390417e-07,
|
||
|
|
"logits/chosen": 0.46904435753822327,
|
||
|
|
"logits/rejected": 0.5106580257415771,
|
||
|
|
"logps/chosen": -39.85272979736328,
|
||
|
|
"logps/rejected": -61.70741653442383,
|
||
|
|
"loss": 0.2336,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.3319948613643646,
|
||
|
|
"rewards/margins": 2.6446897983551025,
|
||
|
|
"rewards/rejected": -2.312695026397705,
|
||
|
|
"step": 228
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5308372868316451,
|
||
|
|
"grad_norm": 28.165420212664795,
|
||
|
|
"learning_rate": 4.595669140962143e-07,
|
||
|
|
"logits/chosen": 0.4127655625343323,
|
||
|
|
"logits/rejected": 0.479299396276474,
|
||
|
|
"logps/chosen": -34.939422607421875,
|
||
|
|
"logps/rejected": -78.63516235351562,
|
||
|
|
"loss": 0.3107,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.18619215488433838,
|
||
|
|
"rewards/margins": 2.8822548389434814,
|
||
|
|
"rewards/rejected": -2.6960630416870117,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5308372868316451,
|
||
|
|
"eval_logits/chosen": 0.4082220494747162,
|
||
|
|
"eval_logits/rejected": 0.4335884749889374,
|
||
|
|
"eval_logps/chosen": -40.824676513671875,
|
||
|
|
"eval_logps/rejected": -51.529090881347656,
|
||
|
|
"eval_loss": 0.30161648988723755,
|
||
|
|
"eval_rewards/accuracies": 0.8104838728904724,
|
||
|
|
"eval_rewards/chosen": 0.4603023827075958,
|
||
|
|
"eval_rewards/margins": 2.205678939819336,
|
||
|
|
"eval_rewards/rejected": -1.745376706123352,
|
||
|
|
"eval_runtime": 220.269,
|
||
|
|
"eval_samples_per_second": 7.872,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5354532632388768,
|
||
|
|
"grad_norm": 16.564014282307436,
|
||
|
|
"learning_rate": 4.5846063462173284e-07,
|
||
|
|
"logits/chosen": 0.5141347050666809,
|
||
|
|
"logits/rejected": 0.5398997664451599,
|
||
|
|
"logps/chosen": -38.93478012084961,
|
||
|
|
"logps/rejected": -53.1637077331543,
|
||
|
|
"loss": 0.2932,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.3137105405330658,
|
||
|
|
"rewards/margins": 2.214162826538086,
|
||
|
|
"rewards/rejected": -1.9004522562026978,
|
||
|
|
"step": 232
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5400692396461084,
|
||
|
|
"grad_norm": 30.180896923031582,
|
||
|
|
"learning_rate": 4.573407936849044e-07,
|
||
|
|
"logits/chosen": 0.49748367071151733,
|
||
|
|
"logits/rejected": 0.502750039100647,
|
||
|
|
"logps/chosen": -46.67736053466797,
|
||
|
|
"logps/rejected": -48.594566345214844,
|
||
|
|
"loss": 0.3143,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.39324572682380676,
|
||
|
|
"rewards/margins": 1.9298076629638672,
|
||
|
|
"rewards/rejected": -1.5365619659423828,
|
||
|
|
"step": 234
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5446852160533402,
|
||
|
|
"grad_norm": 43.03719615392396,
|
||
|
|
"learning_rate": 4.5620746413728063e-07,
|
||
|
|
"logits/chosen": 0.5845724940299988,
|
||
|
|
"logits/rejected": 0.5915371775627136,
|
||
|
|
"logps/chosen": -52.0160026550293,
|
||
|
|
"logps/rejected": -49.12672805786133,
|
||
|
|
"loss": 0.2833,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.14373371005058289,
|
||
|
|
"rewards/margins": 2.1639184951782227,
|
||
|
|
"rewards/rejected": -2.0201845169067383,
|
||
|
|
"step": 236
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5493011924605719,
|
||
|
|
"grad_norm": 21.1030283537707,
|
||
|
|
"learning_rate": 4.550607197079185e-07,
|
||
|
|
"logits/chosen": 0.552834153175354,
|
||
|
|
"logits/rejected": 0.5818264484405518,
|
||
|
|
"logps/chosen": -38.04405212402344,
|
||
|
|
"logps/rejected": -46.87253189086914,
|
||
|
|
"loss": 0.2897,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.4428212344646454,
|
||
|
|
"rewards/margins": 1.7602063417434692,
|
||
|
|
"rewards/rejected": -1.317385196685791,
|
||
|
|
"step": 238
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5539171688678035,
|
||
|
|
"grad_norm": 14.340136381864786,
|
||
|
|
"learning_rate": 4.5390063499858353e-07,
|
||
|
|
"logits/chosen": 0.5454181432723999,
|
||
|
|
"logits/rejected": 0.5769542455673218,
|
||
|
|
"logps/chosen": -47.16811752319336,
|
||
|
|
"logps/rejected": -62.15293884277344,
|
||
|
|
"loss": 0.2046,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": 0.36544325947761536,
|
||
|
|
"rewards/margins": 2.6488418579101562,
|
||
|
|
"rewards/rejected": -2.2833986282348633,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5539171688678035,
|
||
|
|
"eval_logits/chosen": 0.41252779960632324,
|
||
|
|
"eval_logits/rejected": 0.4378991425037384,
|
||
|
|
"eval_logps/chosen": -40.974342346191406,
|
||
|
|
"eval_logps/rejected": -51.8930778503418,
|
||
|
|
"eval_loss": 0.2962896525859833,
|
||
|
|
"eval_rewards/accuracies": 0.8070276379585266,
|
||
|
|
"eval_rewards/chosen": 0.3854685127735138,
|
||
|
|
"eval_rewards/margins": 2.312840223312378,
|
||
|
|
"eval_rewards/rejected": -1.927371859550476,
|
||
|
|
"eval_runtime": 220.4271,
|
||
|
|
"eval_samples_per_second": 7.867,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5585331452750353,
|
||
|
|
"grad_norm": 20.121912107871452,
|
||
|
|
"learning_rate": 4.5272728547889687e-07,
|
||
|
|
"logits/chosen": 0.5017317533493042,
|
||
|
|
"logits/rejected": 0.5252359509468079,
|
||
|
|
"logps/chosen": -43.418678283691406,
|
||
|
|
"logps/rejected": -51.78999710083008,
|
||
|
|
"loss": 0.2157,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.29254353046417236,
|
||
|
|
"rewards/margins": 2.571570873260498,
|
||
|
|
"rewards/rejected": -2.2790274620056152,
|
||
|
|
"step": 242
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5631491216822669,
|
||
|
|
"grad_norm": 36.79556689673262,
|
||
|
|
"learning_rate": 4.5154074748142535e-07,
|
||
|
|
"logits/chosen": 0.5326908230781555,
|
||
|
|
"logits/rejected": 0.5592876672744751,
|
||
|
|
"logps/chosen": -45.176578521728516,
|
||
|
|
"logps/rejected": -55.26374053955078,
|
||
|
|
"loss": 0.2959,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.26444554328918457,
|
||
|
|
"rewards/margins": 2.2498509883880615,
|
||
|
|
"rewards/rejected": -1.985405445098877,
|
||
|
|
"step": 244
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5677650980894986,
|
||
|
|
"grad_norm": 30.279268688467162,
|
||
|
|
"learning_rate": 4.503410981967158e-07,
|
||
|
|
"logits/chosen": 0.508591890335083,
|
||
|
|
"logits/rejected": 0.5472189784049988,
|
||
|
|
"logps/chosen": -37.81255340576172,
|
||
|
|
"logps/rejected": -59.81355285644531,
|
||
|
|
"loss": 0.3387,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.324074387550354,
|
||
|
|
"rewards/margins": 2.479010581970215,
|
||
|
|
"rewards/rejected": -2.1549363136291504,
|
||
|
|
"step": 246
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5723810744967304,
|
||
|
|
"grad_norm": 32.696656835575155,
|
||
|
|
"learning_rate": 4.4912841566827333e-07,
|
||
|
|
"logits/chosen": 0.5358154773712158,
|
||
|
|
"logits/rejected": 0.572979211807251,
|
||
|
|
"logps/chosen": -40.84016799926758,
|
||
|
|
"logps/rejected": -57.57326889038086,
|
||
|
|
"loss": 0.2559,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.6461736559867859,
|
||
|
|
"rewards/margins": 2.717188835144043,
|
||
|
|
"rewards/rejected": -2.0710153579711914,
|
||
|
|
"step": 248
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.576997050903962,
|
||
|
|
"grad_norm": 26.864795137183627,
|
||
|
|
"learning_rate": 4.4790277878748415e-07,
|
||
|
|
"logits/chosen": 0.5129296779632568,
|
||
|
|
"logits/rejected": 0.543644368648529,
|
||
|
|
"logps/chosen": -36.90694046020508,
|
||
|
|
"logps/rejected": -51.41253662109375,
|
||
|
|
"loss": 0.2466,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": 0.4497109651565552,
|
||
|
|
"rewards/margins": 2.559537172317505,
|
||
|
|
"rewards/rejected": -2.1098265647888184,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.576997050903962,
|
||
|
|
"eval_logits/chosen": 0.4140053689479828,
|
||
|
|
"eval_logits/rejected": 0.43953680992126465,
|
||
|
|
"eval_logps/chosen": -40.92128372192383,
|
||
|
|
"eval_logps/rejected": -52.06728744506836,
|
||
|
|
"eval_loss": 0.29202744364738464,
|
||
|
|
"eval_rewards/accuracies": 0.8064516186714172,
|
||
|
|
"eval_rewards/chosen": 0.41199636459350586,
|
||
|
|
"eval_rewards/margins": 2.4264743328094482,
|
||
|
|
"eval_rewards/rejected": -2.0144779682159424,
|
||
|
|
"eval_runtime": 220.3958,
|
||
|
|
"eval_samples_per_second": 7.868,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5816130273111938,
|
||
|
|
"grad_norm": 34.34355868179491,
|
||
|
|
"learning_rate": 4.466642672884835e-07,
|
||
|
|
"logits/chosen": 0.5273095965385437,
|
||
|
|
"logits/rejected": 0.5604310631752014,
|
||
|
|
"logps/chosen": -39.039512634277344,
|
||
|
|
"logps/rejected": -52.470951080322266,
|
||
|
|
"loss": 0.2676,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.256040096282959,
|
||
|
|
"rewards/margins": 2.4306235313415527,
|
||
|
|
"rewards/rejected": -2.1745834350585938,
|
||
|
|
"step": 252
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5862290037184255,
|
||
|
|
"grad_norm": 27.545044099293104,
|
||
|
|
"learning_rate": 4.454129617429682e-07,
|
||
|
|
"logits/chosen": 0.515310525894165,
|
||
|
|
"logits/rejected": 0.5264334678649902,
|
||
|
|
"logps/chosen": -41.25297546386719,
|
||
|
|
"logps/rejected": -44.831031799316406,
|
||
|
|
"loss": 0.2921,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.2963744103908539,
|
||
|
|
"rewards/margins": 2.2201662063598633,
|
||
|
|
"rewards/rejected": -1.9237921237945557,
|
||
|
|
"step": 254
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5908449801256571,
|
||
|
|
"grad_norm": 16.22258168997157,
|
||
|
|
"learning_rate": 4.441489435549551e-07,
|
||
|
|
"logits/chosen": 0.5497354865074158,
|
||
|
|
"logits/rejected": 0.5820472240447998,
|
||
|
|
"logps/chosen": -45.16104507446289,
|
||
|
|
"logps/rejected": -60.09016799926758,
|
||
|
|
"loss": 0.2492,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.36222705245018005,
|
||
|
|
"rewards/margins": 2.6290435791015625,
|
||
|
|
"rewards/rejected": -2.2668166160583496,
|
||
|
|
"step": 256
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5954609565328889,
|
||
|
|
"grad_norm": 22.519317936372268,
|
||
|
|
"learning_rate": 4.4287229495548573e-07,
|
||
|
|
"logits/chosen": 0.5290111303329468,
|
||
|
|
"logits/rejected": 0.550987184047699,
|
||
|
|
"logps/chosen": -45.896942138671875,
|
||
|
|
"logps/rejected": -57.38431930541992,
|
||
|
|
"loss": 0.2158,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.3132680654525757,
|
||
|
|
"rewards/margins": 2.935549020767212,
|
||
|
|
"rewards/rejected": -2.622281074523926,
|
||
|
|
"step": 258
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6000769329401205,
|
||
|
|
"grad_norm": 33.27879387908239,
|
||
|
|
"learning_rate": 4.415830989972761e-07,
|
||
|
|
"logits/chosen": 0.613827645778656,
|
||
|
|
"logits/rejected": 0.6395273208618164,
|
||
|
|
"logps/chosen": -40.98984146118164,
|
||
|
|
"logps/rejected": -48.8809700012207,
|
||
|
|
"loss": 0.3209,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.3634183704853058,
|
||
|
|
"rewards/margins": 2.285569190979004,
|
||
|
|
"rewards/rejected": -1.922150731086731,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6000769329401205,
|
||
|
|
"eval_logits/chosen": 0.41586774587631226,
|
||
|
|
"eval_logits/rejected": 0.4413994550704956,
|
||
|
|
"eval_logps/chosen": -41.435340881347656,
|
||
|
|
"eval_logps/rejected": -52.66230773925781,
|
||
|
|
"eval_loss": 0.28806936740875244,
|
||
|
|
"eval_rewards/accuracies": 0.8116359710693359,
|
||
|
|
"eval_rewards/chosen": 0.15496963262557983,
|
||
|
|
"eval_rewards/margins": 2.4669582843780518,
|
||
|
|
"eval_rewards/rejected": -2.3119888305664062,
|
||
|
|
"eval_runtime": 220.1153,
|
||
|
|
"eval_samples_per_second": 7.878,
|
||
|
|
"eval_steps_per_second": 1.972,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6046929093473522,
|
||
|
|
"grad_norm": 28.090703957454657,
|
||
|
|
"learning_rate": 4.402814395493142e-07,
|
||
|
|
"logits/chosen": 0.49612462520599365,
|
||
|
|
"logits/rejected": 0.4979320168495178,
|
||
|
|
"logps/chosen": -40.7058219909668,
|
||
|
|
"logps/rejected": -38.908050537109375,
|
||
|
|
"loss": 0.3653,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.15811699628829956,
|
||
|
|
"rewards/margins": 1.8890395164489746,
|
||
|
|
"rewards/rejected": -1.7309226989746094,
|
||
|
|
"step": 262
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.609308885754584,
|
||
|
|
"grad_norm": 20.963207734816056,
|
||
|
|
"learning_rate": 4.3896740129140354e-07,
|
||
|
|
"logits/chosen": 0.49926820397377014,
|
||
|
|
"logits/rejected": 0.518930196762085,
|
||
|
|
"logps/chosen": -41.947425842285156,
|
||
|
|
"logps/rejected": -42.273597717285156,
|
||
|
|
"loss": 0.2493,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.2666120231151581,
|
||
|
|
"rewards/margins": 2.4279704093933105,
|
||
|
|
"rewards/rejected": -2.161358594894409,
|
||
|
|
"step": 264
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6139248621618156,
|
||
|
|
"grad_norm": 24.847993356607933,
|
||
|
|
"learning_rate": 4.3764106970865456e-07,
|
||
|
|
"logits/chosen": 0.5007407665252686,
|
||
|
|
"logits/rejected": 0.5330516695976257,
|
||
|
|
"logps/chosen": -36.07570266723633,
|
||
|
|
"logps/rejected": -50.92935562133789,
|
||
|
|
"loss": 0.3174,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.02925288677215576,
|
||
|
|
"rewards/margins": 2.231614589691162,
|
||
|
|
"rewards/rejected": -2.202361583709717,
|
||
|
|
"step": 266
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6185408385690473,
|
||
|
|
"grad_norm": 26.539349634561272,
|
||
|
|
"learning_rate": 4.3630253108592305e-07,
|
||
|
|
"logits/chosen": 0.5235443115234375,
|
||
|
|
"logits/rejected": 0.5463228821754456,
|
||
|
|
"logps/chosen": -48.52283477783203,
|
||
|
|
"logps/rejected": -54.78059387207031,
|
||
|
|
"loss": 0.2266,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.005189484916627407,
|
||
|
|
"rewards/margins": 2.9114773273468018,
|
||
|
|
"rewards/rejected": -2.9062881469726562,
|
||
|
|
"step": 268
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6231568149762791,
|
||
|
|
"grad_norm": 35.3397663590889,
|
||
|
|
"learning_rate": 4.3495187250219723e-07,
|
||
|
|
"logits/chosen": 0.4959086775779724,
|
||
|
|
"logits/rejected": 0.5330989360809326,
|
||
|
|
"logps/chosen": -37.50285339355469,
|
||
|
|
"logps/rejected": -56.99623489379883,
|
||
|
|
"loss": 0.2865,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.16485626995563507,
|
||
|
|
"rewards/margins": 2.9254465103149414,
|
||
|
|
"rewards/rejected": -3.0903029441833496,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6231568149762791,
|
||
|
|
"eval_logits/chosen": 0.4182251989841461,
|
||
|
|
"eval_logits/rejected": 0.44391536712646484,
|
||
|
|
"eval_logps/chosen": -41.51067352294922,
|
||
|
|
"eval_logps/rejected": -52.77988052368164,
|
||
|
|
"eval_loss": 0.2869359254837036,
|
||
|
|
"eval_rewards/accuracies": 0.8116359710693359,
|
||
|
|
"eval_rewards/chosen": 0.11730305105447769,
|
||
|
|
"eval_rewards/margins": 2.488077163696289,
|
||
|
|
"eval_rewards/rejected": -2.3707735538482666,
|
||
|
|
"eval_runtime": 220.1579,
|
||
|
|
"eval_samples_per_second": 7.876,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6277727913835107,
|
||
|
|
"grad_norm": 23.403340630174217,
|
||
|
|
"learning_rate": 4.3358918182493253e-07,
|
||
|
|
"logits/chosen": 0.5670427083969116,
|
||
|
|
"logits/rejected": 0.5846278071403503,
|
||
|
|
"logps/chosen": -41.197166442871094,
|
||
|
|
"logps/rejected": -48.75783920288086,
|
||
|
|
"loss": 0.229,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.05103777348995209,
|
||
|
|
"rewards/margins": 2.2875313758850098,
|
||
|
|
"rewards/rejected": -2.338569164276123,
|
||
|
|
"step": 272
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6323887677907424,
|
||
|
|
"grad_norm": 31.35543837574939,
|
||
|
|
"learning_rate": 4.3221454770433554e-07,
|
||
|
|
"logits/chosen": 0.5044899582862854,
|
||
|
|
"logits/rejected": 0.5252879858016968,
|
||
|
|
"logps/chosen": -46.43470764160156,
|
||
|
|
"logps/rejected": -50.872764587402344,
|
||
|
|
"loss": 0.2558,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.030280061066150665,
|
||
|
|
"rewards/margins": 2.529269218444824,
|
||
|
|
"rewards/rejected": -2.4989893436431885,
|
||
|
|
"step": 274
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6370047441979741,
|
||
|
|
"grad_norm": 27.239886684790495,
|
||
|
|
"learning_rate": 4.308280595675966e-07,
|
||
|
|
"logits/chosen": 0.5399680733680725,
|
||
|
|
"logits/rejected": 0.5539530515670776,
|
||
|
|
"logps/chosen": -45.22441101074219,
|
||
|
|
"logps/rejected": -51.61985397338867,
|
||
|
|
"loss": 0.3439,
|
||
|
|
"rewards/accuracies": 0.7638888955116272,
|
||
|
|
"rewards/chosen": -0.1256939023733139,
|
||
|
|
"rewards/margins": 2.2664339542388916,
|
||
|
|
"rewards/rejected": -2.392127752304077,
|
||
|
|
"step": 276
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6416207206052058,
|
||
|
|
"grad_norm": 29.254953852014435,
|
||
|
|
"learning_rate": 4.2942980761307227e-07,
|
||
|
|
"logits/chosen": 0.5513600707054138,
|
||
|
|
"logits/rejected": 0.5763798356056213,
|
||
|
|
"logps/chosen": -42.95576477050781,
|
||
|
|
"logps/rejected": -53.852542877197266,
|
||
|
|
"loss": 0.2795,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.3323157727718353,
|
||
|
|
"rewards/margins": 2.3478498458862305,
|
||
|
|
"rewards/rejected": -2.680166006088257,
|
||
|
|
"step": 278
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6462366970124375,
|
||
|
|
"grad_norm": 16.01715280590405,
|
||
|
|
"learning_rate": 4.2801988280441765e-07,
|
||
|
|
"logits/chosen": 0.5487841367721558,
|
||
|
|
"logits/rejected": 0.5692893862724304,
|
||
|
|
"logps/chosen": -45.817508697509766,
|
||
|
|
"logps/rejected": -54.61252975463867,
|
||
|
|
"loss": 0.2162,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.03073420189321041,
|
||
|
|
"rewards/margins": 2.809882402420044,
|
||
|
|
"rewards/rejected": -2.840616226196289,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6462366970124375,
|
||
|
|
"eval_logits/chosen": 0.41910773515701294,
|
||
|
|
"eval_logits/rejected": 0.44490164518356323,
|
||
|
|
"eval_logps/chosen": -41.43645477294922,
|
||
|
|
"eval_logps/rejected": -52.90102005004883,
|
||
|
|
"eval_loss": 0.2802717387676239,
|
||
|
|
"eval_rewards/accuracies": 0.8104838728904724,
|
||
|
|
"eval_rewards/chosen": 0.15440984070301056,
|
||
|
|
"eval_rewards/margins": 2.585754156112671,
|
||
|
|
"eval_rewards/rejected": -2.431344509124756,
|
||
|
|
"eval_runtime": 220.3099,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6508526734196692,
|
||
|
|
"grad_norm": 21.181113416054586,
|
||
|
|
"learning_rate": 4.2659837686466813e-07,
|
||
|
|
"logits/chosen": 0.498602032661438,
|
||
|
|
"logits/rejected": 0.5217832922935486,
|
||
|
|
"logps/chosen": -40.613285064697266,
|
||
|
|
"logps/rejected": -50.06806945800781,
|
||
|
|
"loss": 0.262,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.07628664374351501,
|
||
|
|
"rewards/margins": 2.542593240737915,
|
||
|
|
"rewards/rejected": -2.466306447982788,
|
||
|
|
"step": 282
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6554686498269009,
|
||
|
|
"grad_norm": 27.465624654814576,
|
||
|
|
"learning_rate": 4.25165382270273e-07,
|
||
|
|
"logits/chosen": 0.5099713206291199,
|
||
|
|
"logits/rejected": 0.5337219834327698,
|
||
|
|
"logps/chosen": -37.57986831665039,
|
||
|
|
"logps/rejected": -45.39601516723633,
|
||
|
|
"loss": 0.2483,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.15927743911743164,
|
||
|
|
"rewards/margins": 2.373776912689209,
|
||
|
|
"rewards/rejected": -2.2144994735717773,
|
||
|
|
"step": 284
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6600846262341326,
|
||
|
|
"grad_norm": 24.232084058794833,
|
||
|
|
"learning_rate": 4.2372099224507875e-07,
|
||
|
|
"logits/chosen": 0.47430500388145447,
|
||
|
|
"logits/rejected": 0.5168524980545044,
|
||
|
|
"logps/chosen": -34.61323547363281,
|
||
|
|
"logps/rejected": -60.36859130859375,
|
||
|
|
"loss": 0.2904,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.013289166614413261,
|
||
|
|
"rewards/margins": 2.84716534614563,
|
||
|
|
"rewards/rejected": -2.860454797744751,
|
||
|
|
"step": 286
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6647006026413643,
|
||
|
|
"grad_norm": 28.26074226923709,
|
||
|
|
"learning_rate": 4.2226530075426503e-07,
|
||
|
|
"logits/chosen": 0.5559656620025635,
|
||
|
|
"logits/rejected": 0.562049150466919,
|
||
|
|
"logps/chosen": -48.77291488647461,
|
||
|
|
"logps/rejected": -52.30695343017578,
|
||
|
|
"loss": 0.2904,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.016986362636089325,
|
||
|
|
"rewards/margins": 2.4160873889923096,
|
||
|
|
"rewards/rejected": -2.3991012573242188,
|
||
|
|
"step": 288
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.669316579048596,
|
||
|
|
"grad_norm": 25.964047989048964,
|
||
|
|
"learning_rate": 4.2079840249823106e-07,
|
||
|
|
"logits/chosen": 0.5188059210777283,
|
||
|
|
"logits/rejected": 0.5476034879684448,
|
||
|
|
"logps/chosen": -43.39430236816406,
|
||
|
|
"logps/rejected": -63.02970886230469,
|
||
|
|
"loss": 0.2964,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.22233732044696808,
|
||
|
|
"rewards/margins": 2.6584837436676025,
|
||
|
|
"rewards/rejected": -2.8808212280273438,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.669316579048596,
|
||
|
|
"eval_logits/chosen": 0.41873642802238464,
|
||
|
|
"eval_logits/rejected": 0.44454658031463623,
|
||
|
|
"eval_logps/chosen": -41.64173126220703,
|
||
|
|
"eval_logps/rejected": -53.234169006347656,
|
||
|
|
"eval_loss": 0.27578282356262207,
|
||
|
|
"eval_rewards/accuracies": 0.8127880096435547,
|
||
|
|
"eval_rewards/chosen": 0.05177304521203041,
|
||
|
|
"eval_rewards/margins": 2.6496896743774414,
|
||
|
|
"eval_rewards/rejected": -2.597916603088379,
|
||
|
|
"eval_runtime": 220.2319,
|
||
|
|
"eval_samples_per_second": 7.874,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6739325554558276,
|
||
|
|
"grad_norm": 28.11981406671555,
|
||
|
|
"learning_rate": 4.193203929064353e-07,
|
||
|
|
"logits/chosen": 0.5352766513824463,
|
||
|
|
"logits/rejected": 0.5633915066719055,
|
||
|
|
"logps/chosen": -43.08574676513672,
|
||
|
|
"logps/rejected": -63.65277099609375,
|
||
|
|
"loss": 0.292,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": -0.09769348800182343,
|
||
|
|
"rewards/margins": 2.7585980892181396,
|
||
|
|
"rewards/rejected": -2.8562917709350586,
|
||
|
|
"step": 292
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6785485318630594,
|
||
|
|
"grad_norm": 22.159785280949862,
|
||
|
|
"learning_rate": 4.1783136813118705e-07,
|
||
|
|
"logits/chosen": 0.5104035139083862,
|
||
|
|
"logits/rejected": 0.5326347947120667,
|
||
|
|
"logps/chosen": -44.235877990722656,
|
||
|
|
"logps/rejected": -53.24985885620117,
|
||
|
|
"loss": 0.2764,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.3315318822860718,
|
||
|
|
"rewards/margins": 2.574824810028076,
|
||
|
|
"rewards/rejected": -2.9063568115234375,
|
||
|
|
"step": 294
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6831645082702911,
|
||
|
|
"grad_norm": 16.58376439365046,
|
||
|
|
"learning_rate": 4.163314250413913e-07,
|
||
|
|
"logits/chosen": 0.5757681131362915,
|
||
|
|
"logits/rejected": 0.6053035855293274,
|
||
|
|
"logps/chosen": -40.00181579589844,
|
||
|
|
"logps/rejected": -50.29273986816406,
|
||
|
|
"loss": 0.193,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": 0.18691450357437134,
|
||
|
|
"rewards/margins": 2.6521503925323486,
|
||
|
|
"rewards/rejected": -2.465236186981201,
|
||
|
|
"step": 296
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6877804846775227,
|
||
|
|
"grad_norm": 32.319500846176076,
|
||
|
|
"learning_rate": 4.1482066121624716e-07,
|
||
|
|
"logits/chosen": 0.5265994668006897,
|
||
|
|
"logits/rejected": 0.5376725792884827,
|
||
|
|
"logps/chosen": -42.3819580078125,
|
||
|
|
"logps/rejected": -43.448524475097656,
|
||
|
|
"loss": 0.3285,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.1531985104084015,
|
||
|
|
"rewards/margins": 2.268404245376587,
|
||
|
|
"rewards/rejected": -2.115206003189087,
|
||
|
|
"step": 298
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6923964610847545,
|
||
|
|
"grad_norm": 23.349636529497012,
|
||
|
|
"learning_rate": 4.1329917493889933e-07,
|
||
|
|
"logits/chosen": 0.43518775701522827,
|
||
|
|
"logits/rejected": 0.46238911151885986,
|
||
|
|
"logps/chosen": -39.432003021240234,
|
||
|
|
"logps/rejected": -52.38154983520508,
|
||
|
|
"loss": 0.2382,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.1465599089860916,
|
||
|
|
"rewards/margins": 2.628819704055786,
|
||
|
|
"rewards/rejected": -2.7753796577453613,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6923964610847545,
|
||
|
|
"eval_logits/chosen": 0.4236195683479309,
|
||
|
|
"eval_logits/rejected": 0.4493381381034851,
|
||
|
|
"eval_logps/chosen": -41.62788009643555,
|
||
|
|
"eval_logps/rejected": -53.235809326171875,
|
||
|
|
"eval_loss": 0.2743636965751648,
|
||
|
|
"eval_rewards/accuracies": 0.8122119903564453,
|
||
|
|
"eval_rewards/chosen": 0.05869903042912483,
|
||
|
|
"eval_rewards/margins": 2.6574366092681885,
|
||
|
|
"eval_rewards/rejected": -2.5987374782562256,
|
||
|
|
"eval_runtime": 220.281,
|
||
|
|
"eval_samples_per_second": 7.872,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6970124374919862,
|
||
|
|
"grad_norm": 23.497513813632327,
|
||
|
|
"learning_rate": 4.117670651900446e-07,
|
||
|
|
"logits/chosen": 0.5692274570465088,
|
||
|
|
"logits/rejected": 0.5857737064361572,
|
||
|
|
"logps/chosen": -44.88375473022461,
|
||
|
|
"logps/rejected": -50.89904022216797,
|
||
|
|
"loss": 0.3059,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": -0.18370471894741058,
|
||
|
|
"rewards/margins": 2.1322684288024902,
|
||
|
|
"rewards/rejected": -2.3159730434417725,
|
||
|
|
"step": 302
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7016284138992178,
|
||
|
|
"grad_norm": 31.67224576363876,
|
||
|
|
"learning_rate": 4.1022443164149237e-07,
|
||
|
|
"logits/chosen": 0.48219427466392517,
|
||
|
|
"logits/rejected": 0.5107440948486328,
|
||
|
|
"logps/chosen": -46.37804412841797,
|
||
|
|
"logps/rejected": -62.33393859863281,
|
||
|
|
"loss": 0.2685,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.04567752406001091,
|
||
|
|
"rewards/margins": 2.84682559967041,
|
||
|
|
"rewards/rejected": -2.892503261566162,
|
||
|
|
"step": 304
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7062443903064496,
|
||
|
|
"grad_norm": 19.857257644454698,
|
||
|
|
"learning_rate": 4.086713746496808e-07,
|
||
|
|
"logits/chosen": 0.5637336373329163,
|
||
|
|
"logits/rejected": 0.588976263999939,
|
||
|
|
"logps/chosen": -39.28482437133789,
|
||
|
|
"logps/rejected": -50.71957778930664,
|
||
|
|
"loss": 0.2575,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.2317693531513214,
|
||
|
|
"rewards/margins": 2.6872549057006836,
|
||
|
|
"rewards/rejected": -2.4554860591888428,
|
||
|
|
"step": 306
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7108603667136812,
|
||
|
|
"grad_norm": 17.71463775233371,
|
||
|
|
"learning_rate": 4.0710799524914805e-07,
|
||
|
|
"logits/chosen": 0.5934479832649231,
|
||
|
|
"logits/rejected": 0.6081465482711792,
|
||
|
|
"logps/chosen": -50.33334732055664,
|
||
|
|
"logps/rejected": -55.25143814086914,
|
||
|
|
"loss": 0.2103,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.034923017024993896,
|
||
|
|
"rewards/margins": 2.6961231231689453,
|
||
|
|
"rewards/rejected": -2.731046199798584,
|
||
|
|
"step": 308
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7154763431209129,
|
||
|
|
"grad_norm": 19.132153588643654,
|
||
|
|
"learning_rate": 4.055343951459592e-07,
|
||
|
|
"logits/chosen": 0.5560102462768555,
|
||
|
|
"logits/rejected": 0.5947719812393188,
|
||
|
|
"logps/chosen": -37.43670654296875,
|
||
|
|
"logps/rejected": -57.06461715698242,
|
||
|
|
"loss": 0.226,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.07254935055971146,
|
||
|
|
"rewards/margins": 2.918682336807251,
|
||
|
|
"rewards/rejected": -2.991231918334961,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7154763431209129,
|
||
|
|
"eval_logits/chosen": 0.42303159832954407,
|
||
|
|
"eval_logits/rejected": 0.44889286160469055,
|
||
|
|
"eval_logps/chosen": -41.60685348510742,
|
||
|
|
"eval_logps/rejected": -53.284358978271484,
|
||
|
|
"eval_loss": 0.27253130078315735,
|
||
|
|
"eval_rewards/accuracies": 0.8133640289306641,
|
||
|
|
"eval_rewards/chosen": 0.06921074539422989,
|
||
|
|
"eval_rewards/margins": 2.692223072052002,
|
||
|
|
"eval_rewards/rejected": -2.6230127811431885,
|
||
|
|
"eval_runtime": 220.2961,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7200923195281447,
|
||
|
|
"grad_norm": 20.574269162073108,
|
||
|
|
"learning_rate": 4.0395067671108985e-07,
|
||
|
|
"logits/chosen": 0.47218936681747437,
|
||
|
|
"logits/rejected": 0.5014721155166626,
|
||
|
|
"logps/chosen": -35.916664123535156,
|
||
|
|
"logps/rejected": -44.856101989746094,
|
||
|
|
"loss": 0.2579,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.182376891374588,
|
||
|
|
"rewards/margins": 2.569021701812744,
|
||
|
|
"rewards/rejected": -2.3866446018218994,
|
||
|
|
"step": 312
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7247082959353763,
|
||
|
|
"grad_norm": 30.250167869534483,
|
||
|
|
"learning_rate": 4.0235694297376637e-07,
|
||
|
|
"logits/chosen": 0.5631113648414612,
|
||
|
|
"logits/rejected": 0.5769122242927551,
|
||
|
|
"logps/chosen": -49.87733459472656,
|
||
|
|
"logps/rejected": -55.8229866027832,
|
||
|
|
"loss": 0.2861,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.1988232433795929,
|
||
|
|
"rewards/margins": 2.635685443878174,
|
||
|
|
"rewards/rejected": -2.4368624687194824,
|
||
|
|
"step": 314
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.729324272342608,
|
||
|
|
"grad_norm": 32.09859733085628,
|
||
|
|
"learning_rate": 4.0075329761476347e-07,
|
||
|
|
"logits/chosen": 0.5582194924354553,
|
||
|
|
"logits/rejected": 0.5716796517372131,
|
||
|
|
"logps/chosen": -44.06077575683594,
|
||
|
|
"logps/rejected": -48.060577392578125,
|
||
|
|
"loss": 0.2637,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.14417774975299835,
|
||
|
|
"rewards/margins": 2.182429313659668,
|
||
|
|
"rewards/rejected": -2.3266072273254395,
|
||
|
|
"step": 316
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7339402487498398,
|
||
|
|
"grad_norm": 20.839702603979845,
|
||
|
|
"learning_rate": 3.991398449596588e-07,
|
||
|
|
"logits/chosen": 0.5104639530181885,
|
||
|
|
"logits/rejected": 0.5302228331565857,
|
||
|
|
"logps/chosen": -46.450565338134766,
|
||
|
|
"logps/rejected": -56.8250732421875,
|
||
|
|
"loss": 0.2178,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.05337013676762581,
|
||
|
|
"rewards/margins": 2.7899389266967773,
|
||
|
|
"rewards/rejected": -2.7365689277648926,
|
||
|
|
"step": 318
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7385562251570714,
|
||
|
|
"grad_norm": 35.607964067039056,
|
||
|
|
"learning_rate": 3.9751668997204647e-07,
|
||
|
|
"logits/chosen": 0.573165774345398,
|
||
|
|
"logits/rejected": 0.592732310295105,
|
||
|
|
"logps/chosen": -46.10280990600586,
|
||
|
|
"logps/rejected": -53.3104248046875,
|
||
|
|
"loss": 0.238,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.08940169960260391,
|
||
|
|
"rewards/margins": 2.5656909942626953,
|
||
|
|
"rewards/rejected": -2.655092716217041,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7385562251570714,
|
||
|
|
"eval_logits/chosen": 0.4224054217338562,
|
||
|
|
"eval_logits/rejected": 0.4482380449771881,
|
||
|
|
"eval_logps/chosen": -41.65960693359375,
|
||
|
|
"eval_logps/rejected": -53.47556686401367,
|
||
|
|
"eval_loss": 0.2701371908187866,
|
||
|
|
"eval_rewards/accuracies": 0.8185483813285828,
|
||
|
|
"eval_rewards/chosen": 0.04283595457673073,
|
||
|
|
"eval_rewards/margins": 2.761453866958618,
|
||
|
|
"eval_rewards/rejected": -2.718618154525757,
|
||
|
|
"eval_runtime": 220.4956,
|
||
|
|
"eval_samples_per_second": 7.864,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7431722015643031,
|
||
|
|
"grad_norm": 40.34998221595971,
|
||
|
|
"learning_rate": 3.958839382467084e-07,
|
||
|
|
"logits/chosen": 0.5077357888221741,
|
||
|
|
"logits/rejected": 0.5302278995513916,
|
||
|
|
"logps/chosen": -38.23583984375,
|
||
|
|
"logps/rejected": -49.62001037597656,
|
||
|
|
"loss": 0.2911,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.431808739900589,
|
||
|
|
"rewards/margins": 2.4383790493011475,
|
||
|
|
"rewards/rejected": -2.0065698623657227,
|
||
|
|
"step": 322
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7477881779715349,
|
||
|
|
"grad_norm": 37.34949673704143,
|
||
|
|
"learning_rate": 3.9424169600274494e-07,
|
||
|
|
"logits/chosen": 0.5166856646537781,
|
||
|
|
"logits/rejected": 0.5311781167984009,
|
||
|
|
"logps/chosen": -43.24025344848633,
|
||
|
|
"logps/rejected": -48.49333190917969,
|
||
|
|
"loss": 0.3054,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.1698003113269806,
|
||
|
|
"rewards/margins": 2.2522177696228027,
|
||
|
|
"rewards/rejected": -2.422018051147461,
|
||
|
|
"step": 324
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7524041543787665,
|
||
|
|
"grad_norm": 25.91010722050029,
|
||
|
|
"learning_rate": 3.9259007007666436e-07,
|
||
|
|
"logits/chosen": 0.5167285203933716,
|
||
|
|
"logits/rejected": 0.5338759422302246,
|
||
|
|
"logps/chosen": -44.82267761230469,
|
||
|
|
"logps/rejected": -55.40620803833008,
|
||
|
|
"loss": 0.2723,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.06528851389884949,
|
||
|
|
"rewards/margins": 2.759828805923462,
|
||
|
|
"rewards/rejected": -2.694540023803711,
|
||
|
|
"step": 326
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7570201307859982,
|
||
|
|
"grad_norm": 30.862683948057615,
|
||
|
|
"learning_rate": 3.909291679154332e-07,
|
||
|
|
"logits/chosen": 0.5040656328201294,
|
||
|
|
"logits/rejected": 0.5386430025100708,
|
||
|
|
"logps/chosen": -42.25190734863281,
|
||
|
|
"logps/rejected": -62.51930618286133,
|
||
|
|
"loss": 0.2759,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.2548324167728424,
|
||
|
|
"rewards/margins": 3.0783848762512207,
|
||
|
|
"rewards/rejected": -3.333217144012451,
|
||
|
|
"step": 328
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7616361071932299,
|
||
|
|
"grad_norm": 19.125155732205084,
|
||
|
|
"learning_rate": 3.892590975694858e-07,
|
||
|
|
"logits/chosen": 0.49563461542129517,
|
||
|
|
"logits/rejected": 0.539116621017456,
|
||
|
|
"logps/chosen": -39.31736755371094,
|
||
|
|
"logps/rejected": -60.45228576660156,
|
||
|
|
"loss": 0.2182,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.16796639561653137,
|
||
|
|
"rewards/margins": 3.4695467948913574,
|
||
|
|
"rewards/rejected": -3.301579713821411,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7616361071932299,
|
||
|
|
"eval_logits/chosen": 0.4225333333015442,
|
||
|
|
"eval_logits/rejected": 0.44842836260795593,
|
||
|
|
"eval_logps/chosen": -41.670494079589844,
|
||
|
|
"eval_logps/rejected": -53.553314208984375,
|
||
|
|
"eval_loss": 0.2688952684402466,
|
||
|
|
"eval_rewards/accuracies": 0.8145161271095276,
|
||
|
|
"eval_rewards/chosen": 0.037393342703580856,
|
||
|
|
"eval_rewards/margins": 2.7948849201202393,
|
||
|
|
"eval_rewards/rejected": -2.7574915885925293,
|
||
|
|
"eval_runtime": 220.4734,
|
||
|
|
"eval_samples_per_second": 7.865,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7662520836004616,
|
||
|
|
"grad_norm": 20.197390141727503,
|
||
|
|
"learning_rate": 3.875799676856952e-07,
|
||
|
|
"logits/chosen": 0.5481100082397461,
|
||
|
|
"logits/rejected": 0.5680783987045288,
|
||
|
|
"logps/chosen": -43.26856994628906,
|
||
|
|
"logps/rejected": -54.90293884277344,
|
||
|
|
"loss": 0.2148,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.2920362651348114,
|
||
|
|
"rewards/margins": 2.9112956523895264,
|
||
|
|
"rewards/rejected": -3.20333194732666,
|
||
|
|
"step": 332
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7708680600076933,
|
||
|
|
"grad_norm": 28.41138671183374,
|
||
|
|
"learning_rate": 3.858918875003053e-07,
|
||
|
|
"logits/chosen": 0.5375738143920898,
|
||
|
|
"logits/rejected": 0.5755133628845215,
|
||
|
|
"logps/chosen": -41.622859954833984,
|
||
|
|
"logps/rejected": -61.92311096191406,
|
||
|
|
"loss": 0.2733,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.09028860926628113,
|
||
|
|
"rewards/margins": 3.286768674850464,
|
||
|
|
"rewards/rejected": -3.3770573139190674,
|
||
|
|
"step": 334
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.775484036414925,
|
||
|
|
"grad_norm": 16.265551276537238,
|
||
|
|
"learning_rate": 3.8419496683182396e-07,
|
||
|
|
"logits/chosen": 0.5556432604789734,
|
||
|
|
"logits/rejected": 0.5942565202713013,
|
||
|
|
"logps/chosen": -41.74842071533203,
|
||
|
|
"logps/rejected": -57.50096893310547,
|
||
|
|
"loss": 0.1896,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.0623447448015213,
|
||
|
|
"rewards/margins": 2.878957748413086,
|
||
|
|
"rewards/rejected": -2.941302537918091,
|
||
|
|
"step": 336
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7801000128221567,
|
||
|
|
"grad_norm": 26.59915287717055,
|
||
|
|
"learning_rate": 3.824893160738792e-07,
|
||
|
|
"logits/chosen": 0.5246456861495972,
|
||
|
|
"logits/rejected": 0.553848385810852,
|
||
|
|
"logps/chosen": -42.39156723022461,
|
||
|
|
"logps/rejected": -57.20592498779297,
|
||
|
|
"loss": 0.2682,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": -0.07438618689775467,
|
||
|
|
"rewards/margins": 3.046879291534424,
|
||
|
|
"rewards/rejected": -3.1212656497955322,
|
||
|
|
"step": 338
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7847159892293883,
|
||
|
|
"grad_norm": 23.023616857684974,
|
||
|
|
"learning_rate": 3.8077504618803737e-07,
|
||
|
|
"logits/chosen": 0.580450713634491,
|
||
|
|
"logits/rejected": 0.5835237503051758,
|
||
|
|
"logps/chosen": -48.9189567565918,
|
||
|
|
"logps/rejected": -47.836578369140625,
|
||
|
|
"loss": 0.2668,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.06672815978527069,
|
||
|
|
"rewards/margins": 2.457933187484741,
|
||
|
|
"rewards/rejected": -2.5246615409851074,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7847159892293883,
|
||
|
|
"eval_logits/chosen": 0.4240727126598358,
|
||
|
|
"eval_logits/rejected": 0.4500102698802948,
|
||
|
|
"eval_logps/chosen": -41.714290618896484,
|
||
|
|
"eval_logps/rejected": -53.6696662902832,
|
||
|
|
"eval_loss": 0.2670309841632843,
|
||
|
|
"eval_rewards/accuracies": 0.8179723620414734,
|
||
|
|
"eval_rewards/chosen": 0.015493539161980152,
|
||
|
|
"eval_rewards/margins": 2.83115816116333,
|
||
|
|
"eval_rewards/rejected": -2.815664768218994,
|
||
|
|
"eval_runtime": 220.6721,
|
||
|
|
"eval_samples_per_second": 7.858,
|
||
|
|
"eval_steps_per_second": 1.967,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7893319656366201,
|
||
|
|
"grad_norm": 16.479244956266236,
|
||
|
|
"learning_rate": 3.7905226869658446e-07,
|
||
|
|
"logits/chosen": 0.4684799015522003,
|
||
|
|
"logits/rejected": 0.4874458909034729,
|
||
|
|
"logps/chosen": -43.62626647949219,
|
||
|
|
"logps/rejected": -55.70362854003906,
|
||
|
|
"loss": 0.2494,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.13190358877182007,
|
||
|
|
"rewards/margins": 2.8091206550598145,
|
||
|
|
"rewards/rejected": -2.6772167682647705,
|
||
|
|
"step": 342
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7939479420438518,
|
||
|
|
"grad_norm": 24.369877883114157,
|
||
|
|
"learning_rate": 3.773210956752709e-07,
|
||
|
|
"logits/chosen": 0.544243574142456,
|
||
|
|
"logits/rejected": 0.5578660368919373,
|
||
|
|
"logps/chosen": -40.1495246887207,
|
||
|
|
"logps/rejected": -44.17314910888672,
|
||
|
|
"loss": 0.2798,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.020625757053494453,
|
||
|
|
"rewards/margins": 2.502214193344116,
|
||
|
|
"rewards/rejected": -2.481588363647461,
|
||
|
|
"step": 344
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7985639184510834,
|
||
|
|
"grad_norm": 25.623903462647995,
|
||
|
|
"learning_rate": 3.7558163974602093e-07,
|
||
|
|
"logits/chosen": 0.474899560213089,
|
||
|
|
"logits/rejected": 0.5161857008934021,
|
||
|
|
"logps/chosen": -37.74607467651367,
|
||
|
|
"logps/rejected": -55.48906707763672,
|
||
|
|
"loss": 0.2419,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.001830246765166521,
|
||
|
|
"rewards/margins": 2.923034906387329,
|
||
|
|
"rewards/rejected": -2.924865245819092,
|
||
|
|
"step": 346
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031798948583152,
|
||
|
|
"grad_norm": 25.184522607734593,
|
||
|
|
"learning_rate": 3.73834014069605e-07,
|
||
|
|
"logits/chosen": 0.558302104473114,
|
||
|
|
"logits/rejected": 0.5833041667938232,
|
||
|
|
"logps/chosen": -48.4046630859375,
|
||
|
|
"logps/rejected": -61.20756149291992,
|
||
|
|
"loss": 0.2374,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.1346227377653122,
|
||
|
|
"rewards/margins": 2.8843278884887695,
|
||
|
|
"rewards/rejected": -3.0189502239227295,
|
||
|
|
"step": 348
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8077958712655469,
|
||
|
|
"grad_norm": 24.77024105098058,
|
||
|
|
"learning_rate": 3.7207833233827914e-07,
|
||
|
|
"logits/chosen": 0.4649287462234497,
|
||
|
|
"logits/rejected": 0.482571542263031,
|
||
|
|
"logps/chosen": -44.39389419555664,
|
||
|
|
"logps/rejected": -58.24624252319336,
|
||
|
|
"loss": 0.2534,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.14530682563781738,
|
||
|
|
"rewards/margins": 3.2228527069091797,
|
||
|
|
"rewards/rejected": -3.368159532546997,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8077958712655469,
|
||
|
|
"eval_logits/chosen": 0.42746230959892273,
|
||
|
|
"eval_logits/rejected": 0.45336535573005676,
|
||
|
|
"eval_logps/chosen": -42.037269592285156,
|
||
|
|
"eval_logps/rejected": -54.03358459472656,
|
||
|
|
"eval_loss": 0.2634715437889099,
|
||
|
|
"eval_rewards/accuracies": 0.8168202638626099,
|
||
|
|
"eval_rewards/chosen": -0.1459963023662567,
|
||
|
|
"eval_rewards/margins": 2.8516335487365723,
|
||
|
|
"eval_rewards/rejected": -2.9976296424865723,
|
||
|
|
"eval_runtime": 220.3701,
|
||
|
|
"eval_samples_per_second": 7.869,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8124118476727785,
|
||
|
|
"grad_norm": 26.201135314502036,
|
||
|
|
"learning_rate": 3.7031470876838786e-07,
|
||
|
|
"logits/chosen": 0.5293068289756775,
|
||
|
|
"logits/rejected": 0.5655782222747803,
|
||
|
|
"logps/chosen": -42.89842224121094,
|
||
|
|
"logps/rejected": -63.14483642578125,
|
||
|
|
"loss": 0.2516,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.4706004559993744,
|
||
|
|
"rewards/margins": 2.8817062377929688,
|
||
|
|
"rewards/rejected": -3.352307081222534,
|
||
|
|
"step": 352
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8170278240800103,
|
||
|
|
"grad_norm": 22.294887268242963,
|
||
|
|
"learning_rate": 3.6854325809293455e-07,
|
||
|
|
"logits/chosen": 0.49771615862846375,
|
||
|
|
"logits/rejected": 0.5413529276847839,
|
||
|
|
"logps/chosen": -36.90867233276367,
|
||
|
|
"logps/rejected": -64.4770278930664,
|
||
|
|
"loss": 0.2284,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.27428972721099854,
|
||
|
|
"rewards/margins": 3.501157522201538,
|
||
|
|
"rewards/rejected": -3.775447368621826,
|
||
|
|
"step": 354
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.821643800487242,
|
||
|
|
"grad_norm": 28.188753078893058,
|
||
|
|
"learning_rate": 3.6676409555411653e-07,
|
||
|
|
"logits/chosen": 0.5484297871589661,
|
||
|
|
"logits/rejected": 0.5813949704170227,
|
||
|
|
"logps/chosen": -45.460365295410156,
|
||
|
|
"logps/rejected": -60.86439895629883,
|
||
|
|
"loss": 0.2542,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.4246326684951782,
|
||
|
|
"rewards/margins": 3.2056918144226074,
|
||
|
|
"rewards/rejected": -3.630324363708496,
|
||
|
|
"step": 356
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8262597768944736,
|
||
|
|
"grad_norm": 17.14121226520804,
|
||
|
|
"learning_rate": 3.6497733689582866e-07,
|
||
|
|
"logits/chosen": 0.48876845836639404,
|
||
|
|
"logits/rejected": 0.5145962238311768,
|
||
|
|
"logps/chosen": -39.37761688232422,
|
||
|
|
"logps/rejected": -49.643211364746094,
|
||
|
|
"loss": 0.2016,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.21149006485939026,
|
||
|
|
"rewards/margins": 2.893353223800659,
|
||
|
|
"rewards/rejected": -3.1048433780670166,
|
||
|
|
"step": 358
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8308757533017054,
|
||
|
|
"grad_norm": 35.17955186267088,
|
||
|
|
"learning_rate": 3.631830983561335e-07,
|
||
|
|
"logits/chosen": 0.573662519454956,
|
||
|
|
"logits/rejected": 0.5948094725608826,
|
||
|
|
"logps/chosen": -47.85080337524414,
|
||
|
|
"logps/rejected": -52.225006103515625,
|
||
|
|
"loss": 0.2586,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.3559052646160126,
|
||
|
|
"rewards/margins": 2.786222219467163,
|
||
|
|
"rewards/rejected": -3.142127513885498,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8308757533017054,
|
||
|
|
"eval_logits/chosen": 0.42756161093711853,
|
||
|
|
"eval_logits/rejected": 0.45349106192588806,
|
||
|
|
"eval_logps/chosen": -42.38340759277344,
|
||
|
|
"eval_logps/rejected": -54.44844436645508,
|
||
|
|
"eval_loss": 0.2630784213542938,
|
||
|
|
"eval_rewards/accuracies": 0.8179723620414734,
|
||
|
|
"eval_rewards/chosen": -0.31906506419181824,
|
||
|
|
"eval_rewards/margins": 2.8859920501708984,
|
||
|
|
"eval_rewards/rejected": -3.205056667327881,
|
||
|
|
"eval_runtime": 220.2057,
|
||
|
|
"eval_samples_per_second": 7.874,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.835491729708937,
|
||
|
|
"grad_norm": 36.03053976982613,
|
||
|
|
"learning_rate": 3.613814966596991e-07,
|
||
|
|
"logits/chosen": 0.5263631343841553,
|
||
|
|
"logits/rejected": 0.5573300123214722,
|
||
|
|
"logps/chosen": -43.24696731567383,
|
||
|
|
"logps/rejected": -57.23331069946289,
|
||
|
|
"loss": 0.2526,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.4683598279953003,
|
||
|
|
"rewards/margins": 3.082267999649048,
|
||
|
|
"rewards/rejected": -3.5506277084350586,
|
||
|
|
"step": 362
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8401077061161687,
|
||
|
|
"grad_norm": 15.328563865471402,
|
||
|
|
"learning_rate": 3.595726490102059e-07,
|
||
|
|
"logits/chosen": 0.5707637071609497,
|
||
|
|
"logits/rejected": 0.6143693327903748,
|
||
|
|
"logps/chosen": -40.44147491455078,
|
||
|
|
"logps/rejected": -62.61209487915039,
|
||
|
|
"loss": 0.1294,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.3496915102005005,
|
||
|
|
"rewards/margins": 3.486618995666504,
|
||
|
|
"rewards/rejected": -3.836310863494873,
|
||
|
|
"step": 364
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8447236825234005,
|
||
|
|
"grad_norm": 15.002635114989888,
|
||
|
|
"learning_rate": 3.577566730827214e-07,
|
||
|
|
"logits/chosen": 0.5126733779907227,
|
||
|
|
"logits/rejected": 0.5439874529838562,
|
||
|
|
"logps/chosen": -40.29549789428711,
|
||
|
|
"logps/rejected": -56.204898834228516,
|
||
|
|
"loss": 0.2951,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": -0.3362084925174713,
|
||
|
|
"rewards/margins": 2.846021890640259,
|
||
|
|
"rewards/rejected": -3.182230234146118,
|
||
|
|
"step": 366
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8493396589306321,
|
||
|
|
"grad_norm": 25.52691859216037,
|
||
|
|
"learning_rate": 3.559336870160453e-07,
|
||
|
|
"logits/chosen": 0.5128374099731445,
|
||
|
|
"logits/rejected": 0.5424924492835999,
|
||
|
|
"logps/chosen": -38.71543884277344,
|
||
|
|
"logps/rejected": -52.61689758300781,
|
||
|
|
"loss": 0.2084,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.28658950328826904,
|
||
|
|
"rewards/margins": 3.0817792415618896,
|
||
|
|
"rewards/rejected": -3.368368625640869,
|
||
|
|
"step": 368
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8539556353378638,
|
||
|
|
"grad_norm": 30.283513234320385,
|
||
|
|
"learning_rate": 3.541038094050241e-07,
|
||
|
|
"logits/chosen": 0.515430212020874,
|
||
|
|
"logits/rejected": 0.5466374158859253,
|
||
|
|
"logps/chosen": -45.59136962890625,
|
||
|
|
"logps/rejected": -63.18849182128906,
|
||
|
|
"loss": 0.2378,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.5768634676933289,
|
||
|
|
"rewards/margins": 3.5630674362182617,
|
||
|
|
"rewards/rejected": -4.139930725097656,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8539556353378638,
|
||
|
|
"eval_logits/chosen": 0.4274056553840637,
|
||
|
|
"eval_logits/rejected": 0.45338377356529236,
|
||
|
|
"eval_logps/chosen": -43.063682556152344,
|
||
|
|
"eval_logps/rejected": -55.225093841552734,
|
||
|
|
"eval_loss": 0.2617854177951813,
|
||
|
|
"eval_rewards/accuracies": 0.817396342754364,
|
||
|
|
"eval_rewards/chosen": -0.659203290939331,
|
||
|
|
"eval_rewards/margins": 2.9341788291931152,
|
||
|
|
"eval_rewards/rejected": -3.5933821201324463,
|
||
|
|
"eval_runtime": 220.2088,
|
||
|
|
"eval_samples_per_second": 7.874,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8585716117450956,
|
||
|
|
"grad_norm": 30.9826241797592,
|
||
|
|
"learning_rate": 3.52267159292835e-07,
|
||
|
|
"logits/chosen": 0.4993041455745697,
|
||
|
|
"logits/rejected": 0.5248599052429199,
|
||
|
|
"logps/chosen": -44.83211898803711,
|
||
|
|
"logps/rejected": -61.29323959350586,
|
||
|
|
"loss": 0.2333,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.7047384977340698,
|
||
|
|
"rewards/margins": 3.358118772506714,
|
||
|
|
"rewards/rejected": -4.062856674194336,
|
||
|
|
"step": 372
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8631875881523272,
|
||
|
|
"grad_norm": 16.52463887201103,
|
||
|
|
"learning_rate": 3.5042385616324236e-07,
|
||
|
|
"logits/chosen": 0.4287330210208893,
|
||
|
|
"logits/rejected": 0.46707651019096375,
|
||
|
|
"logps/chosen": -36.363590240478516,
|
||
|
|
"logps/rejected": -59.82657241821289,
|
||
|
|
"loss": 0.22,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.8720024228096008,
|
||
|
|
"rewards/margins": 3.389249086380005,
|
||
|
|
"rewards/rejected": -4.261251449584961,
|
||
|
|
"step": 374
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8678035645595589,
|
||
|
|
"grad_norm": 15.500715269356169,
|
||
|
|
"learning_rate": 3.485740199328244e-07,
|
||
|
|
"logits/chosen": 0.5408291816711426,
|
||
|
|
"logits/rejected": 0.5578600764274597,
|
||
|
|
"logps/chosen": -50.285335540771484,
|
||
|
|
"logps/rejected": -54.07209014892578,
|
||
|
|
"loss": 0.1876,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.5448592305183411,
|
||
|
|
"rewards/margins": 3.2346181869506836,
|
||
|
|
"rewards/rejected": -3.779477119445801,
|
||
|
|
"step": 376
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8724195409667906,
|
||
|
|
"grad_norm": 12.222084345575727,
|
||
|
|
"learning_rate": 3.4671777094317196e-07,
|
||
|
|
"logits/chosen": 0.5013281106948853,
|
||
|
|
"logits/rejected": 0.5262949466705322,
|
||
|
|
"logps/chosen": -46.47956848144531,
|
||
|
|
"logps/rejected": -53.49814224243164,
|
||
|
|
"loss": 0.1677,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.7341945767402649,
|
||
|
|
"rewards/margins": 3.0543222427368164,
|
||
|
|
"rewards/rejected": -3.7885169982910156,
|
||
|
|
"step": 378
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8770355173740223,
|
||
|
|
"grad_norm": 22.531696347522484,
|
||
|
|
"learning_rate": 3.448552299530595e-07,
|
||
|
|
"logits/chosen": 0.5649933218955994,
|
||
|
|
"logits/rejected": 0.5860426425933838,
|
||
|
|
"logps/chosen": -42.52098083496094,
|
||
|
|
"logps/rejected": -52.308616638183594,
|
||
|
|
"loss": 0.3071,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": -0.9869860410690308,
|
||
|
|
"rewards/margins": 2.7113142013549805,
|
||
|
|
"rewards/rejected": -3.698300361633301,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8770355173740223,
|
||
|
|
"eval_logits/chosen": 0.4274827539920807,
|
||
|
|
"eval_logits/rejected": 0.45349830389022827,
|
||
|
|
"eval_logps/chosen": -43.129615783691406,
|
||
|
|
"eval_logps/rejected": -55.33893585205078,
|
||
|
|
"eval_loss": 0.2627149224281311,
|
||
|
|
"eval_rewards/accuracies": 0.8156682252883911,
|
||
|
|
"eval_rewards/chosen": -0.6921693086624146,
|
||
|
|
"eval_rewards/margins": 2.958131790161133,
|
||
|
|
"eval_rewards/rejected": -3.650301218032837,
|
||
|
|
"eval_runtime": 220.3046,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.881651493781254,
|
||
|
|
"grad_norm": 39.03269809250303,
|
||
|
|
"learning_rate": 3.429865181305894e-07,
|
||
|
|
"logits/chosen": 0.5594089031219482,
|
||
|
|
"logits/rejected": 0.5762946605682373,
|
||
|
|
"logps/chosen": -46.85918045043945,
|
||
|
|
"logps/rejected": -55.68655776977539,
|
||
|
|
"loss": 0.2915,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.9153691530227661,
|
||
|
|
"rewards/margins": 2.779404401779175,
|
||
|
|
"rewards/rejected": -3.694772958755493,
|
||
|
|
"step": 382
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8862674701884857,
|
||
|
|
"grad_norm": 25.617922410092657,
|
||
|
|
"learning_rate": 3.411117570453091e-07,
|
||
|
|
"logits/chosen": 0.5484945774078369,
|
||
|
|
"logits/rejected": 0.5738579034805298,
|
||
|
|
"logps/chosen": -42.73631286621094,
|
||
|
|
"logps/rejected": -53.853271484375,
|
||
|
|
"loss": 0.2369,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.7328565120697021,
|
||
|
|
"rewards/margins": 2.8266656398773193,
|
||
|
|
"rewards/rejected": -3.5595223903656006,
|
||
|
|
"step": 384
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8908834465957174,
|
||
|
|
"grad_norm": 30.869961559508535,
|
||
|
|
"learning_rate": 3.392310686603025e-07,
|
||
|
|
"logits/chosen": 0.534080982208252,
|
||
|
|
"logits/rejected": 0.5444844365119934,
|
||
|
|
"logps/chosen": -42.41215515136719,
|
||
|
|
"logps/rejected": -50.85294723510742,
|
||
|
|
"loss": 0.2909,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": -0.9006066918373108,
|
||
|
|
"rewards/margins": 2.361262559890747,
|
||
|
|
"rewards/rejected": -3.261868953704834,
|
||
|
|
"step": 386
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.895499423002949,
|
||
|
|
"grad_norm": 19.657432685783167,
|
||
|
|
"learning_rate": 3.3734457532425554e-07,
|
||
|
|
"logits/chosen": 0.5231594443321228,
|
||
|
|
"logits/rejected": 0.5530441403388977,
|
||
|
|
"logps/chosen": -42.48830795288086,
|
||
|
|
"logps/rejected": -57.00692367553711,
|
||
|
|
"loss": 0.2606,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.6170899271965027,
|
||
|
|
"rewards/margins": 3.237041711807251,
|
||
|
|
"rewards/rejected": -3.8541314601898193,
|
||
|
|
"step": 388
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9001153994101808,
|
||
|
|
"grad_norm": 24.399140672578795,
|
||
|
|
"learning_rate": 3.354523997634969e-07,
|
||
|
|
"logits/chosen": 0.540899932384491,
|
||
|
|
"logits/rejected": 0.5695917010307312,
|
||
|
|
"logps/chosen": -44.531185150146484,
|
||
|
|
"logps/rejected": -58.8494873046875,
|
||
|
|
"loss": 0.2251,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.7790883183479309,
|
||
|
|
"rewards/margins": 3.128167152404785,
|
||
|
|
"rewards/rejected": -3.9072554111480713,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9001153994101808,
|
||
|
|
"eval_logits/chosen": 0.42857107520103455,
|
||
|
|
"eval_logits/rejected": 0.4546278119087219,
|
||
|
|
"eval_logps/chosen": -43.16852951049805,
|
||
|
|
"eval_logps/rejected": -55.42344665527344,
|
||
|
|
"eval_loss": 0.2621525228023529,
|
||
|
|
"eval_rewards/accuracies": 0.8179723620414734,
|
||
|
|
"eval_rewards/chosen": -0.7116276621818542,
|
||
|
|
"eval_rewards/margins": 2.980929374694824,
|
||
|
|
"eval_rewards/rejected": -3.6925570964813232,
|
||
|
|
"eval_runtime": 220.3143,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9047313758174125,
|
||
|
|
"grad_norm": 35.01908054863291,
|
||
|
|
"learning_rate": 3.3355466507401374e-07,
|
||
|
|
"logits/chosen": 0.5315423607826233,
|
||
|
|
"logits/rejected": 0.5454668998718262,
|
||
|
|
"logps/chosen": -42.16218185424805,
|
||
|
|
"logps/rejected": -44.85585403442383,
|
||
|
|
"loss": 0.372,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": -0.805086612701416,
|
||
|
|
"rewards/margins": 2.338005542755127,
|
||
|
|
"rewards/rejected": -3.143092155456543,
|
||
|
|
"step": 392
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9093473522246441,
|
||
|
|
"grad_norm": 21.288998506479572,
|
||
|
|
"learning_rate": 3.3165149471344394e-07,
|
||
|
|
"logits/chosen": 0.5552914142608643,
|
||
|
|
"logits/rejected": 0.5818530321121216,
|
||
|
|
"logps/chosen": -42.95904541015625,
|
||
|
|
"logps/rejected": -52.76212692260742,
|
||
|
|
"loss": 0.2934,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.9580552577972412,
|
||
|
|
"rewards/margins": 2.6676671504974365,
|
||
|
|
"rewards/rejected": -3.6257221698760986,
|
||
|
|
"step": 394
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9139633286318759,
|
||
|
|
"grad_norm": 25.556003693396036,
|
||
|
|
"learning_rate": 3.297430124930444e-07,
|
||
|
|
"logits/chosen": 0.582655668258667,
|
||
|
|
"logits/rejected": 0.5952574014663696,
|
||
|
|
"logps/chosen": -48.771934509277344,
|
||
|
|
"logps/rejected": -54.426483154296875,
|
||
|
|
"loss": 0.3223,
|
||
|
|
"rewards/accuracies": 0.7361111044883728,
|
||
|
|
"rewards/chosen": -0.6146318912506104,
|
||
|
|
"rewards/margins": 2.4974234104156494,
|
||
|
|
"rewards/rejected": -3.112055540084839,
|
||
|
|
"step": 396
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9185793050391076,
|
||
|
|
"grad_norm": 23.905362174336005,
|
||
|
|
"learning_rate": 3.2782934256963647e-07,
|
||
|
|
"logits/chosen": 0.5089656114578247,
|
||
|
|
"logits/rejected": 0.5398065447807312,
|
||
|
|
"logps/chosen": -45.75530242919922,
|
||
|
|
"logps/rejected": -61.64253234863281,
|
||
|
|
"loss": 0.2549,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.6105983853340149,
|
||
|
|
"rewards/margins": 3.1589841842651367,
|
||
|
|
"rewards/rejected": -3.769582509994507,
|
||
|
|
"step": 398
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9231952814463392,
|
||
|
|
"grad_norm": 24.17532494020093,
|
||
|
|
"learning_rate": 3.259106094375289e-07,
|
||
|
|
"logits/chosen": 0.539167046546936,
|
||
|
|
"logits/rejected": 0.5812445282936096,
|
||
|
|
"logps/chosen": -39.31736755371094,
|
||
|
|
"logps/rejected": -63.33793640136719,
|
||
|
|
"loss": 0.2698,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.3948301374912262,
|
||
|
|
"rewards/margins": 3.442387819290161,
|
||
|
|
"rewards/rejected": -3.8372182846069336,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9231952814463392,
|
||
|
|
"eval_logits/chosen": 0.42656469345092773,
|
||
|
|
"eval_logits/rejected": 0.45276370644569397,
|
||
|
|
"eval_logps/chosen": -42.66855239868164,
|
||
|
|
"eval_logps/rejected": -55.0075798034668,
|
||
|
|
"eval_loss": 0.2560158371925354,
|
||
|
|
"eval_rewards/accuracies": 0.8231566548347473,
|
||
|
|
"eval_rewards/chosen": -0.46163854002952576,
|
||
|
|
"eval_rewards/margins": 3.0229856967926025,
|
||
|
|
"eval_rewards/rejected": -3.4846243858337402,
|
||
|
|
"eval_runtime": 220.2216,
|
||
|
|
"eval_samples_per_second": 7.874,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.927811257853571,
|
||
|
|
"grad_norm": 30.671620714098214,
|
||
|
|
"learning_rate": 3.239869379204189e-07,
|
||
|
|
"logits/chosen": 0.4974105656147003,
|
||
|
|
"logits/rejected": 0.5221477746963501,
|
||
|
|
"logps/chosen": -45.057281494140625,
|
||
|
|
"logps/rejected": -56.83816909790039,
|
||
|
|
"loss": 0.2017,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.5868238210678101,
|
||
|
|
"rewards/margins": 3.3964414596557617,
|
||
|
|
"rewards/rejected": -3.9832653999328613,
|
||
|
|
"step": 402
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9324272342608027,
|
||
|
|
"grad_norm": 24.915176146115876,
|
||
|
|
"learning_rate": 3.2205845316327144e-07,
|
||
|
|
"logits/chosen": 0.5429517030715942,
|
||
|
|
"logits/rejected": 0.5683455467224121,
|
||
|
|
"logps/chosen": -34.97327423095703,
|
||
|
|
"logps/rejected": -46.666717529296875,
|
||
|
|
"loss": 0.3399,
|
||
|
|
"rewards/accuracies": 0.7222222089767456,
|
||
|
|
"rewards/chosen": -0.43591320514678955,
|
||
|
|
"rewards/margins": 2.185106039047241,
|
||
|
|
"rewards/rejected": -2.6210196018218994,
|
||
|
|
"step": 404
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9370432106680343,
|
||
|
|
"grad_norm": 23.867375292949593,
|
||
|
|
"learning_rate": 3.2012528062417845e-07,
|
||
|
|
"logits/chosen": 0.5323294997215271,
|
||
|
|
"logits/rejected": 0.5459015369415283,
|
||
|
|
"logps/chosen": -43.10551071166992,
|
||
|
|
"logps/rejected": -47.71934127807617,
|
||
|
|
"loss": 0.2436,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.7240028977394104,
|
||
|
|
"rewards/margins": 2.4708030223846436,
|
||
|
|
"rewards/rejected": -3.1948060989379883,
|
||
|
|
"step": 406
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9416591870752661,
|
||
|
|
"grad_norm": 15.007721932706033,
|
||
|
|
"learning_rate": 3.1818754606619643e-07,
|
||
|
|
"logits/chosen": 0.5331852436065674,
|
||
|
|
"logits/rejected": 0.564946174621582,
|
||
|
|
"logps/chosen": -36.540283203125,
|
||
|
|
"logps/rejected": -57.03317642211914,
|
||
|
|
"loss": 0.2822,
|
||
|
|
"rewards/accuracies": 0.75,
|
||
|
|
"rewards/chosen": -0.16474466025829315,
|
||
|
|
"rewards/margins": 3.167923927307129,
|
||
|
|
"rewards/rejected": -3.3326683044433594,
|
||
|
|
"step": 408
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9462751634824977,
|
||
|
|
"grad_norm": 22.364487052769828,
|
||
|
|
"learning_rate": 3.162453755491655e-07,
|
||
|
|
"logits/chosen": 0.49684393405914307,
|
||
|
|
"logits/rejected": 0.5316374897956848,
|
||
|
|
"logps/chosen": -38.39241027832031,
|
||
|
|
"logps/rejected": -59.15244674682617,
|
||
|
|
"loss": 0.1874,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.322665810585022,
|
||
|
|
"rewards/margins": 3.4969892501831055,
|
||
|
|
"rewards/rejected": -3.819655179977417,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9462751634824977,
|
||
|
|
"eval_logits/chosen": 0.4290708899497986,
|
||
|
|
"eval_logits/rejected": 0.45515918731689453,
|
||
|
|
"eval_logps/chosen": -42.679603576660156,
|
||
|
|
"eval_logps/rejected": -55.10276412963867,
|
||
|
|
"eval_loss": 0.2565246820449829,
|
||
|
|
"eval_rewards/accuracies": 0.8191244006156921,
|
||
|
|
"eval_rewards/chosen": -0.467162162065506,
|
||
|
|
"eval_rewards/margins": 3.065053939819336,
|
||
|
|
"eval_rewards/rejected": -3.5322158336639404,
|
||
|
|
"eval_runtime": 220.2891,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9508911398897294,
|
||
|
|
"grad_norm": 25.19862106785063,
|
||
|
|
"learning_rate": 3.142988954215079e-07,
|
||
|
|
"logits/chosen": 0.5264102816581726,
|
||
|
|
"logits/rejected": 0.5622512698173523,
|
||
|
|
"logps/chosen": -43.48373794555664,
|
||
|
|
"logps/rejected": -66.42120361328125,
|
||
|
|
"loss": 0.2996,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.48827776312828064,
|
||
|
|
"rewards/margins": 3.3450686931610107,
|
||
|
|
"rewards/rejected": -3.833346128463745,
|
||
|
|
"step": 412
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9555071162969612,
|
||
|
|
"grad_norm": 21.74301345510537,
|
||
|
|
"learning_rate": 3.1234823231200925e-07,
|
||
|
|
"logits/chosen": 0.5031583309173584,
|
||
|
|
"logits/rejected": 0.5540390014648438,
|
||
|
|
"logps/chosen": -40.93600845336914,
|
||
|
|
"logps/rejected": -66.30878448486328,
|
||
|
|
"loss": 0.2428,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.5792509317398071,
|
||
|
|
"rewards/margins": 3.6368870735168457,
|
||
|
|
"rewards/rejected": -4.2161383628845215,
|
||
|
|
"step": 414
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9601230927041928,
|
||
|
|
"grad_norm": 22.436508219334904,
|
||
|
|
"learning_rate": 3.1039351312157993e-07,
|
||
|
|
"logits/chosen": 0.56053227186203,
|
||
|
|
"logits/rejected": 0.590539813041687,
|
||
|
|
"logps/chosen": -41.67660140991211,
|
||
|
|
"logps/rejected": -58.28109359741211,
|
||
|
|
"loss": 0.2048,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.6333367228507996,
|
||
|
|
"rewards/margins": 3.312451124191284,
|
||
|
|
"rewards/rejected": -3.9457881450653076,
|
||
|
|
"step": 416
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9647390691114246,
|
||
|
|
"grad_norm": 36.50210265432233,
|
||
|
|
"learning_rate": 3.0843486501499967e-07,
|
||
|
|
"logits/chosen": 0.508413553237915,
|
||
|
|
"logits/rejected": 0.5429882407188416,
|
||
|
|
"logps/chosen": -42.58755111694336,
|
||
|
|
"logps/rejected": -52.10399627685547,
|
||
|
|
"loss": 0.3069,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.4402269721031189,
|
||
|
|
"rewards/margins": 2.6428239345550537,
|
||
|
|
"rewards/rejected": -3.0830507278442383,
|
||
|
|
"step": 418
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9693550455186563,
|
||
|
|
"grad_norm": 19.432988353108243,
|
||
|
|
"learning_rate": 3.064724154126449e-07,
|
||
|
|
"logits/chosen": 0.48101869225502014,
|
||
|
|
"logits/rejected": 0.49470260739326477,
|
||
|
|
"logps/chosen": -43.99076461791992,
|
||
|
|
"logps/rejected": -47.8154411315918,
|
||
|
|
"loss": 0.2486,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.6770768761634827,
|
||
|
|
"rewards/margins": 2.6182446479797363,
|
||
|
|
"rewards/rejected": -3.2953217029571533,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9693550455186563,
|
||
|
|
"eval_logits/chosen": 0.4298844337463379,
|
||
|
|
"eval_logits/rejected": 0.45596131682395935,
|
||
|
|
"eval_logps/chosen": -42.74457550048828,
|
||
|
|
"eval_logps/rejected": -55.1827278137207,
|
||
|
|
"eval_loss": 0.2540464699268341,
|
||
|
|
"eval_rewards/accuracies": 0.8231566548347473,
|
||
|
|
"eval_rewards/chosen": -0.4996483027935028,
|
||
|
|
"eval_rewards/margins": 3.072551727294922,
|
||
|
|
"eval_rewards/rejected": -3.572199821472168,
|
||
|
|
"eval_runtime": 220.4655,
|
||
|
|
"eval_samples_per_second": 7.865,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9739710219258879,
|
||
|
|
"grad_norm": 21.396529357952137,
|
||
|
|
"learning_rate": 3.045062919821995e-07,
|
||
|
|
"logits/chosen": 0.5096142292022705,
|
||
|
|
"logits/rejected": 0.5509178638458252,
|
||
|
|
"logps/chosen": -40.65134811401367,
|
||
|
|
"logps/rejected": -64.13406372070312,
|
||
|
|
"loss": 0.2407,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.25429394841194153,
|
||
|
|
"rewards/margins": 3.5406899452209473,
|
||
|
|
"rewards/rejected": -3.7949838638305664,
|
||
|
|
"step": 422
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9785869983331197,
|
||
|
|
"grad_norm": 27.30197314549755,
|
||
|
|
"learning_rate": 3.0253662263034925e-07,
|
||
|
|
"logits/chosen": 0.5253940224647522,
|
||
|
|
"logits/rejected": 0.5617537498474121,
|
||
|
|
"logps/chosen": -44.63224792480469,
|
||
|
|
"logps/rejected": -62.29665756225586,
|
||
|
|
"loss": 0.2666,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.6128353476524353,
|
||
|
|
"rewards/margins": 3.4666247367858887,
|
||
|
|
"rewards/rejected": -4.079460144042969,
|
||
|
|
"step": 424
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9832029747403513,
|
||
|
|
"grad_norm": 40.51282949087652,
|
||
|
|
"learning_rate": 3.005635354944606e-07,
|
||
|
|
"logits/chosen": 0.5502428412437439,
|
||
|
|
"logits/rejected": 0.5616468787193298,
|
||
|
|
"logps/chosen": -46.97676467895508,
|
||
|
|
"logps/rejected": -46.36595153808594,
|
||
|
|
"loss": 0.2894,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.7273317575454712,
|
||
|
|
"rewards/margins": 2.6478114128112793,
|
||
|
|
"rewards/rejected": -3.375143051147461,
|
||
|
|
"step": 426
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.987818951147583,
|
||
|
|
"grad_norm": 23.92512657865844,
|
||
|
|
"learning_rate": 2.9858715893424504e-07,
|
||
|
|
"logits/chosen": 0.5228149890899658,
|
||
|
|
"logits/rejected": 0.5698718428611755,
|
||
|
|
"logps/chosen": -40.91889953613281,
|
||
|
|
"logps/rejected": -64.06893920898438,
|
||
|
|
"loss": 0.1794,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.32393407821655273,
|
||
|
|
"rewards/margins": 3.8048884868621826,
|
||
|
|
"rewards/rejected": -4.128821849822998,
|
||
|
|
"step": 428
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9924349275548148,
|
||
|
|
"grad_norm": 18.33017798245734,
|
||
|
|
"learning_rate": 2.966076215234082e-07,
|
||
|
|
"logits/chosen": 0.5833015441894531,
|
||
|
|
"logits/rejected": 0.6151509881019592,
|
||
|
|
"logps/chosen": -47.47243118286133,
|
||
|
|
"logps/rejected": -64.26097869873047,
|
||
|
|
"loss": 0.2098,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.2687421441078186,
|
||
|
|
"rewards/margins": 3.582411766052246,
|
||
|
|
"rewards/rejected": -3.85115385055542,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9924349275548148,
|
||
|
|
"eval_logits/chosen": 0.42911431193351746,
|
||
|
|
"eval_logits/rejected": 0.45535048842430115,
|
||
|
|
"eval_logps/chosen": -42.6432991027832,
|
||
|
|
"eval_logps/rejected": -55.0967903137207,
|
||
|
|
"eval_loss": 0.25298023223876953,
|
||
|
|
"eval_rewards/accuracies": 0.8237327337265015,
|
||
|
|
"eval_rewards/chosen": -0.4490084946155548,
|
||
|
|
"eval_rewards/margins": 3.0802206993103027,
|
||
|
|
"eval_rewards/rejected": -3.5292294025421143,
|
||
|
|
"eval_runtime": 220.5016,
|
||
|
|
"eval_samples_per_second": 7.864,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9970509039620464,
|
||
|
|
"grad_norm": 24.845062608395242,
|
||
|
|
"learning_rate": 2.94625052041286e-07,
|
||
|
|
"logits/chosen": 0.529398500919342,
|
||
|
|
"logits/rejected": 0.5461426377296448,
|
||
|
|
"logps/chosen": -42.26673889160156,
|
||
|
|
"logps/rejected": -52.43321228027344,
|
||
|
|
"loss": 0.2582,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.38506922125816345,
|
||
|
|
"rewards/margins": 2.947833299636841,
|
||
|
|
"rewards/rejected": -3.332902431488037,
|
||
|
|
"step": 432
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.001666880369278,
|
||
|
|
"grad_norm": 14.705625802608846,
|
||
|
|
"learning_rate": 2.926395794644665e-07,
|
||
|
|
"logits/chosen": 0.5060461759567261,
|
||
|
|
"logits/rejected": 0.5222041010856628,
|
||
|
|
"logps/chosen": -45.8979606628418,
|
||
|
|
"logps/rejected": -55.48097229003906,
|
||
|
|
"loss": 0.1798,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.3213649392127991,
|
||
|
|
"rewards/margins": 3.302720308303833,
|
||
|
|
"rewards/rejected": -3.6240854263305664,
|
||
|
|
"step": 434
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0062828567765099,
|
||
|
|
"grad_norm": 24.90302953634143,
|
||
|
|
"learning_rate": 2.906513329583991e-07,
|
||
|
|
"logits/chosen": 0.5120677351951599,
|
||
|
|
"logits/rejected": 0.5406749844551086,
|
||
|
|
"logps/chosen": -40.07225036621094,
|
||
|
|
"logps/rejected": -54.882259368896484,
|
||
|
|
"loss": 0.2186,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.5253066420555115,
|
||
|
|
"rewards/margins": 3.1281352043151855,
|
||
|
|
"rewards/rejected": -3.653441905975342,
|
||
|
|
"step": 436
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0108988331837414,
|
||
|
|
"grad_norm": 20.006366802619794,
|
||
|
|
"learning_rate": 2.886604418689921e-07,
|
||
|
|
"logits/chosen": 0.48885577917099,
|
||
|
|
"logits/rejected": 0.5327137112617493,
|
||
|
|
"logps/chosen": -38.752708435058594,
|
||
|
|
"logps/rejected": -66.8874740600586,
|
||
|
|
"loss": 0.2705,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": -0.5506837368011475,
|
||
|
|
"rewards/margins": 3.6388425827026367,
|
||
|
|
"rewards/rejected": -4.189526557922363,
|
||
|
|
"step": 438
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0155148095909732,
|
||
|
|
"grad_norm": 11.538422039384988,
|
||
|
|
"learning_rate": 2.866670357141979e-07,
|
||
|
|
"logits/chosen": 0.5471632480621338,
|
||
|
|
"logits/rejected": 0.5706813931465149,
|
||
|
|
"logps/chosen": -44.1706428527832,
|
||
|
|
"logps/rejected": -54.80915832519531,
|
||
|
|
"loss": 0.2123,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.5128348469734192,
|
||
|
|
"rewards/margins": 3.5640437602996826,
|
||
|
|
"rewards/rejected": -4.076879024505615,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0155148095909732,
|
||
|
|
"eval_logits/chosen": 0.42714568972587585,
|
||
|
|
"eval_logits/rejected": 0.4533489942550659,
|
||
|
|
"eval_logps/chosen": -42.395565032958984,
|
||
|
|
"eval_logps/rejected": -54.934104919433594,
|
||
|
|
"eval_loss": 0.2539977729320526,
|
||
|
|
"eval_rewards/accuracies": 0.8231566548347473,
|
||
|
|
"eval_rewards/chosen": -0.3251444697380066,
|
||
|
|
"eval_rewards/margins": 3.122741937637329,
|
||
|
|
"eval_rewards/rejected": -3.4478864669799805,
|
||
|
|
"eval_runtime": 220.3559,
|
||
|
|
"eval_samples_per_second": 7.869,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.020130785998205,
|
||
|
|
"grad_norm": 16.119320288131345,
|
||
|
|
"learning_rate": 2.8467124417558737e-07,
|
||
|
|
"logits/chosen": 0.5559278130531311,
|
||
|
|
"logits/rejected": 0.5782606601715088,
|
||
|
|
"logps/chosen": -43.08287048339844,
|
||
|
|
"logps/rejected": -55.4886474609375,
|
||
|
|
"loss": 0.2118,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.22590351104736328,
|
||
|
|
"rewards/margins": 3.3553009033203125,
|
||
|
|
"rewards/rejected": -3.581204414367676,
|
||
|
|
"step": 442
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0247467624054365,
|
||
|
|
"grad_norm": 21.10014479926061,
|
||
|
|
"learning_rate": 2.8267319708991253e-07,
|
||
|
|
"logits/chosen": 0.5570061206817627,
|
||
|
|
"logits/rejected": 0.5741885304450989,
|
||
|
|
"logps/chosen": -46.57928466796875,
|
||
|
|
"logps/rejected": -48.77629089355469,
|
||
|
|
"loss": 0.2203,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.2255779653787613,
|
||
|
|
"rewards/margins": 2.8583762645721436,
|
||
|
|
"rewards/rejected": -3.083954334259033,
|
||
|
|
"step": 444
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0293627388126683,
|
||
|
|
"grad_norm": 21.99323071947427,
|
||
|
|
"learning_rate": 2.806730244406612e-07,
|
||
|
|
"logits/chosen": 0.5444987416267395,
|
||
|
|
"logits/rejected": 0.5731097459793091,
|
||
|
|
"logps/chosen": -40.73080825805664,
|
||
|
|
"logps/rejected": -52.80342102050781,
|
||
|
|
"loss": 0.2407,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.2986847758293152,
|
||
|
|
"rewards/margins": 3.0820257663726807,
|
||
|
|
"rewards/rejected": -3.3807103633880615,
|
||
|
|
"step": 446
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0339787152199,
|
||
|
|
"grad_norm": 17.17450683483707,
|
||
|
|
"learning_rate": 2.786708563496001e-07,
|
||
|
|
"logits/chosen": 0.5541989207267761,
|
||
|
|
"logits/rejected": 0.5817456841468811,
|
||
|
|
"logps/chosen": -45.73213195800781,
|
||
|
|
"logps/rejected": -61.18666458129883,
|
||
|
|
"loss": 0.1772,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.05669987201690674,
|
||
|
|
"rewards/margins": 3.8165981769561768,
|
||
|
|
"rewards/rejected": -3.873298168182373,
|
||
|
|
"step": 448
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0385946916271316,
|
||
|
|
"grad_norm": 27.653708636239905,
|
||
|
|
"learning_rate": 2.7666682306830994e-07,
|
||
|
|
"logits/chosen": 0.5207394957542419,
|
||
|
|
"logits/rejected": 0.5322983860969543,
|
||
|
|
"logps/chosen": -41.09166717529297,
|
||
|
|
"logps/rejected": -43.31468200683594,
|
||
|
|
"loss": 0.2544,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.3381701707839966,
|
||
|
|
"rewards/margins": 2.6456761360168457,
|
||
|
|
"rewards/rejected": -2.9838459491729736,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0385946916271316,
|
||
|
|
"eval_logits/chosen": 0.43128177523612976,
|
||
|
|
"eval_logits/rejected": 0.4573296308517456,
|
||
|
|
"eval_logps/chosen": -42.16498565673828,
|
||
|
|
"eval_logps/rejected": -54.75392150878906,
|
||
|
|
"eval_loss": 0.2521970570087433,
|
||
|
|
"eval_rewards/accuracies": 0.8248847723007202,
|
||
|
|
"eval_rewards/chosen": -0.20985357463359833,
|
||
|
|
"eval_rewards/margins": 3.147939920425415,
|
||
|
|
"eval_rewards/rejected": -3.3577938079833984,
|
||
|
|
"eval_runtime": 220.2887,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0432106680343634,
|
||
|
|
"grad_norm": 26.863807248353726,
|
||
|
|
"learning_rate": 2.746610549697119e-07,
|
||
|
|
"logits/chosen": 0.5497666001319885,
|
||
|
|
"logits/rejected": 0.5746829509735107,
|
||
|
|
"logps/chosen": -42.95619583129883,
|
||
|
|
"logps/rejected": -57.17405700683594,
|
||
|
|
"loss": 0.2279,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.33137860894203186,
|
||
|
|
"rewards/margins": 3.0671894550323486,
|
||
|
|
"rewards/rejected": -3.3985676765441895,
|
||
|
|
"step": 452
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0478266444415951,
|
||
|
|
"grad_norm": 15.765922708965844,
|
||
|
|
"learning_rate": 2.7265368253958615e-07,
|
||
|
|
"logits/chosen": 0.5027904510498047,
|
||
|
|
"logits/rejected": 0.5187773108482361,
|
||
|
|
"logps/chosen": -40.01198959350586,
|
||
|
|
"logps/rejected": -49.16390609741211,
|
||
|
|
"loss": 0.1826,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.024355987086892128,
|
||
|
|
"rewards/margins": 3.001004219055176,
|
||
|
|
"rewards/rejected": -3.025360107421875,
|
||
|
|
"step": 454
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0524426208488267,
|
||
|
|
"grad_norm": 13.117750938407347,
|
||
|
|
"learning_rate": 2.706448363680831e-07,
|
||
|
|
"logits/chosen": 0.5505272746086121,
|
||
|
|
"logits/rejected": 0.592627763748169,
|
||
|
|
"logps/chosen": -40.86323928833008,
|
||
|
|
"logps/rejected": -65.0215072631836,
|
||
|
|
"loss": 0.1182,
|
||
|
|
"rewards/accuracies": 0.9722222089767456,
|
||
|
|
"rewards/chosen": -0.19750367105007172,
|
||
|
|
"rewards/margins": 4.092833995819092,
|
||
|
|
"rewards/rejected": -4.290337562561035,
|
||
|
|
"step": 456
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0570585972560584,
|
||
|
|
"grad_norm": 16.896591758231867,
|
||
|
|
"learning_rate": 2.686346471412277e-07,
|
||
|
|
"logits/chosen": 0.4872972071170807,
|
||
|
|
"logits/rejected": 0.5277370810508728,
|
||
|
|
"logps/chosen": -44.69199752807617,
|
||
|
|
"logps/rejected": -65.82919311523438,
|
||
|
|
"loss": 0.1481,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.253704696893692,
|
||
|
|
"rewards/margins": 3.8575947284698486,
|
||
|
|
"rewards/rejected": -4.111299514770508,
|
||
|
|
"step": 458
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0616745736632902,
|
||
|
|
"grad_norm": 20.974972760985903,
|
||
|
|
"learning_rate": 2.6662324563241805e-07,
|
||
|
|
"logits/chosen": 0.5082690119743347,
|
||
|
|
"logits/rejected": 0.5304160118103027,
|
||
|
|
"logps/chosen": -39.70173263549805,
|
||
|
|
"logps/rejected": -50.749732971191406,
|
||
|
|
"loss": 0.218,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.058096084743738174,
|
||
|
|
"rewards/margins": 2.925325632095337,
|
||
|
|
"rewards/rejected": -2.983421802520752,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0616745736632902,
|
||
|
|
"eval_logits/chosen": 0.42715081572532654,
|
||
|
|
"eval_logits/rejected": 0.45357510447502136,
|
||
|
|
"eval_logps/chosen": -41.917137145996094,
|
||
|
|
"eval_logps/rejected": -54.64493179321289,
|
||
|
|
"eval_loss": 0.2522634267807007,
|
||
|
|
"eval_rewards/accuracies": 0.8231566548347473,
|
||
|
|
"eval_rewards/chosen": -0.08592969179153442,
|
||
|
|
"eval_rewards/margins": 3.217369556427002,
|
||
|
|
"eval_rewards/rejected": -3.3032991886138916,
|
||
|
|
"eval_runtime": 220.2922,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0662905500705218,
|
||
|
|
"grad_norm": 14.344965515087893,
|
||
|
|
"learning_rate": 2.6461076269391713e-07,
|
||
|
|
"logits/chosen": 0.5723965167999268,
|
||
|
|
"logits/rejected": 0.6080074310302734,
|
||
|
|
"logps/chosen": -47.22536087036133,
|
||
|
|
"logps/rejected": -63.04933166503906,
|
||
|
|
"loss": 0.1633,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.08401741087436676,
|
||
|
|
"rewards/margins": 4.024357318878174,
|
||
|
|
"rewards/rejected": -4.10837459564209,
|
||
|
|
"step": 462
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0709065264777535,
|
||
|
|
"grad_norm": 22.161377940303407,
|
||
|
|
"learning_rate": 2.625973292483409e-07,
|
||
|
|
"logits/chosen": 0.49575677514076233,
|
||
|
|
"logits/rejected": 0.5175695419311523,
|
||
|
|
"logps/chosen": -49.86793518066406,
|
||
|
|
"logps/rejected": -61.0032958984375,
|
||
|
|
"loss": 0.2086,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.2437991052865982,
|
||
|
|
"rewards/margins": 3.3475723266601562,
|
||
|
|
"rewards/rejected": -3.5913712978363037,
|
||
|
|
"step": 464
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0755225028849853,
|
||
|
|
"grad_norm": 9.157546830395537,
|
||
|
|
"learning_rate": 2.6058307628014065e-07,
|
||
|
|
"logits/chosen": 0.5648156404495239,
|
||
|
|
"logits/rejected": 0.5903113484382629,
|
||
|
|
"logps/chosen": -47.16014099121094,
|
||
|
|
"logps/rejected": -58.00987243652344,
|
||
|
|
"loss": 0.1681,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.20527897775173187,
|
||
|
|
"rewards/margins": 3.885181427001953,
|
||
|
|
"rewards/rejected": -4.090460777282715,
|
||
|
|
"step": 466
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0801384792922168,
|
||
|
|
"grad_norm": 20.418800394750264,
|
||
|
|
"learning_rate": 2.5856813482708217e-07,
|
||
|
|
"logits/chosen": 0.5167273879051208,
|
||
|
|
"logits/rejected": 0.5341954827308655,
|
||
|
|
"logps/chosen": -44.03962707519531,
|
||
|
|
"logps/rejected": -48.64061737060547,
|
||
|
|
"loss": 0.205,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": 0.06323742121458054,
|
||
|
|
"rewards/margins": 3.104510545730591,
|
||
|
|
"rewards/rejected": -3.041273355484009,
|
||
|
|
"step": 468
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0847544556994486,
|
||
|
|
"grad_norm": 24.70628607742756,
|
||
|
|
"learning_rate": 2.565526359717206e-07,
|
||
|
|
"logits/chosen": 0.537581205368042,
|
||
|
|
"logits/rejected": 0.5596475005149841,
|
||
|
|
"logps/chosen": -37.46675109863281,
|
||
|
|
"logps/rejected": -45.9968147277832,
|
||
|
|
"loss": 0.3005,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.24194829165935516,
|
||
|
|
"rewards/margins": 2.6193909645080566,
|
||
|
|
"rewards/rejected": -2.8613390922546387,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0847544556994486,
|
||
|
|
"eval_logits/chosen": 0.4362466037273407,
|
||
|
|
"eval_logits/rejected": 0.4623866379261017,
|
||
|
|
"eval_logps/chosen": -42.15773010253906,
|
||
|
|
"eval_logps/rejected": -54.935401916503906,
|
||
|
|
"eval_loss": 0.24963192641735077,
|
||
|
|
"eval_rewards/accuracies": 0.8260368704795837,
|
||
|
|
"eval_rewards/chosen": -0.20622780919075012,
|
||
|
|
"eval_rewards/margins": 3.242306709289551,
|
||
|
|
"eval_rewards/rejected": -3.4485342502593994,
|
||
|
|
"eval_runtime": 220.4037,
|
||
|
|
"eval_samples_per_second": 7.867,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0893704321066804,
|
||
|
|
"grad_norm": 27.430779359112005,
|
||
|
|
"learning_rate": 2.545367108328731e-07,
|
||
|
|
"logits/chosen": 0.5652859807014465,
|
||
|
|
"logits/rejected": 0.591205358505249,
|
||
|
|
"logps/chosen": -43.71979904174805,
|
||
|
|
"logps/rejected": -53.00830841064453,
|
||
|
|
"loss": 0.2156,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.20083469152450562,
|
||
|
|
"rewards/margins": 3.2087488174438477,
|
||
|
|
"rewards/rejected": -3.409583330154419,
|
||
|
|
"step": 472
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0939864085139122,
|
||
|
|
"grad_norm": 13.134510140867176,
|
||
|
|
"learning_rate": 2.525204905570889e-07,
|
||
|
|
"logits/chosen": 0.5791910290718079,
|
||
|
|
"logits/rejected": 0.6038353443145752,
|
||
|
|
"logps/chosen": -46.998390197753906,
|
||
|
|
"logps/rejected": -59.18220520019531,
|
||
|
|
"loss": 0.1707,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.05355483293533325,
|
||
|
|
"rewards/margins": 3.5535666942596436,
|
||
|
|
"rewards/rejected": -3.607121706008911,
|
||
|
|
"step": 474
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0986023849211437,
|
||
|
|
"grad_norm": 19.90392742325827,
|
||
|
|
"learning_rate": 2.505041063101171e-07,
|
||
|
|
"logits/chosen": 0.5816848278045654,
|
||
|
|
"logits/rejected": 0.6008831858634949,
|
||
|
|
"logps/chosen": -47.19880676269531,
|
||
|
|
"logps/rejected": -51.822105407714844,
|
||
|
|
"loss": 0.2218,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.03883904218673706,
|
||
|
|
"rewards/margins": 3.348583221435547,
|
||
|
|
"rewards/rejected": -3.309744358062744,
|
||
|
|
"step": 476
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1032183613283755,
|
||
|
|
"grad_norm": 17.00116980477646,
|
||
|
|
"learning_rate": 2.4848768926837466e-07,
|
||
|
|
"logits/chosen": 0.5338962078094482,
|
||
|
|
"logits/rejected": 0.5906614065170288,
|
||
|
|
"logps/chosen": -40.04157257080078,
|
||
|
|
"logps/rejected": -76.84749603271484,
|
||
|
|
"loss": 0.1893,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.1601162701845169,
|
||
|
|
"rewards/margins": 4.218037128448486,
|
||
|
|
"rewards/rejected": -4.378152847290039,
|
||
|
|
"step": 478
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.107834337735607,
|
||
|
|
"grad_norm": 15.038557815597683,
|
||
|
|
"learning_rate": 2.464713706104113e-07,
|
||
|
|
"logits/chosen": 0.5352125763893127,
|
||
|
|
"logits/rejected": 0.5612537264823914,
|
||
|
|
"logps/chosen": -43.91660690307617,
|
||
|
|
"logps/rejected": -56.44979476928711,
|
||
|
|
"loss": 0.1633,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.2793487310409546,
|
||
|
|
"rewards/margins": 3.6175765991210938,
|
||
|
|
"rewards/rejected": -3.896925210952759,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.107834337735607,
|
||
|
|
"eval_logits/chosen": 0.43004509806632996,
|
||
|
|
"eval_logits/rejected": 0.4563468098640442,
|
||
|
|
"eval_logps/chosen": -42.171958923339844,
|
||
|
|
"eval_logps/rejected": -54.986507415771484,
|
||
|
|
"eval_loss": 0.24832715094089508,
|
||
|
|
"eval_rewards/accuracies": 0.8271889686584473,
|
||
|
|
"eval_rewards/chosen": -0.21334028244018555,
|
||
|
|
"eval_rewards/margins": 3.2607483863830566,
|
||
|
|
"eval_rewards/rejected": -3.474088668823242,
|
||
|
|
"eval_runtime": 220.2251,
|
||
|
|
"eval_samples_per_second": 7.874,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1124503141428388,
|
||
|
|
"grad_norm": 22.9744657106464,
|
||
|
|
"learning_rate": 2.444552815083767e-07,
|
||
|
|
"logits/chosen": 0.6254298686981201,
|
||
|
|
"logits/rejected": 0.6373676061630249,
|
||
|
|
"logps/chosen": -42.673282623291016,
|
||
|
|
"logps/rejected": -45.563087463378906,
|
||
|
|
"loss": 0.2114,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.057508740574121475,
|
||
|
|
"rewards/margins": 3.0235791206359863,
|
||
|
|
"rewards/rejected": -3.081087350845337,
|
||
|
|
"step": 482
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1170662905500706,
|
||
|
|
"grad_norm": 17.674691508042564,
|
||
|
|
"learning_rate": 2.4243955311948693e-07,
|
||
|
|
"logits/chosen": 0.5245480537414551,
|
||
|
|
"logits/rejected": 0.5648095011711121,
|
||
|
|
"logps/chosen": -39.3298225402832,
|
||
|
|
"logps/rejected": -61.31127166748047,
|
||
|
|
"loss": 0.2236,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.1908557116985321,
|
||
|
|
"rewards/margins": 3.677870512008667,
|
||
|
|
"rewards/rejected": -3.8687260150909424,
|
||
|
|
"step": 484
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1216822669573023,
|
||
|
|
"grad_norm": 19.4717194397301,
|
||
|
|
"learning_rate": 2.4042431657749115e-07,
|
||
|
|
"logits/chosen": 0.585620105266571,
|
||
|
|
"logits/rejected": 0.6345695853233337,
|
||
|
|
"logps/chosen": -41.645267486572266,
|
||
|
|
"logps/rejected": -72.78955078125,
|
||
|
|
"loss": 0.1703,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.20467931032180786,
|
||
|
|
"rewards/margins": 4.08174991607666,
|
||
|
|
"rewards/rejected": -4.286429405212402,
|
||
|
|
"step": 486
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1262982433645339,
|
||
|
|
"grad_norm": 30.909727917565508,
|
||
|
|
"learning_rate": 2.384097029841419e-07,
|
||
|
|
"logits/chosen": 0.4901224672794342,
|
||
|
|
"logits/rejected": 0.5071887969970703,
|
||
|
|
"logps/chosen": -43.30605697631836,
|
||
|
|
"logps/rejected": -50.992618560791016,
|
||
|
|
"loss": 0.2185,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.18728405237197876,
|
||
|
|
"rewards/margins": 2.9479784965515137,
|
||
|
|
"rewards/rejected": -3.1352624893188477,
|
||
|
|
"step": 488
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1309142197717656,
|
||
|
|
"grad_norm": 16.93415094151409,
|
||
|
|
"learning_rate": 2.3639584340066544e-07,
|
||
|
|
"logits/chosen": 0.5211553573608398,
|
||
|
|
"logits/rejected": 0.5518543124198914,
|
||
|
|
"logps/chosen": -37.83938980102539,
|
||
|
|
"logps/rejected": -53.91053009033203,
|
||
|
|
"loss": 0.234,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.05988183990120888,
|
||
|
|
"rewards/margins": 3.5345206260681152,
|
||
|
|
"rewards/rejected": -3.4746387004852295,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1309142197717656,
|
||
|
|
"eval_logits/chosen": 0.43326738476753235,
|
||
|
|
"eval_logits/rejected": 0.45958051085472107,
|
||
|
|
"eval_logps/chosen": -41.84520721435547,
|
||
|
|
"eval_logps/rejected": -54.6281852722168,
|
||
|
|
"eval_loss": 0.24792973697185516,
|
||
|
|
"eval_rewards/accuracies": 0.8220046162605286,
|
||
|
|
"eval_rewards/chosen": -0.04996471852064133,
|
||
|
|
"eval_rewards/margins": 3.244964361190796,
|
||
|
|
"eval_rewards/rejected": -3.294929265975952,
|
||
|
|
"eval_runtime": 220.3046,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1355301961789972,
|
||
|
|
"grad_norm": 16.790260075155444,
|
||
|
|
"learning_rate": 2.3438286883923539e-07,
|
||
|
|
"logits/chosen": 0.5881079435348511,
|
||
|
|
"logits/rejected": 0.6105315685272217,
|
||
|
|
"logps/chosen": -46.794837951660156,
|
||
|
|
"logps/rejected": -53.43986511230469,
|
||
|
|
"loss": 0.2269,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.08306831121444702,
|
||
|
|
"rewards/margins": 3.1719002723693848,
|
||
|
|
"rewards/rejected": -3.088831663131714,
|
||
|
|
"step": 492
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.140146172586229,
|
||
|
|
"grad_norm": 22.957641710400285,
|
||
|
|
"learning_rate": 2.323709102544506e-07,
|
||
|
|
"logits/chosen": 0.6002509593963623,
|
||
|
|
"logits/rejected": 0.6072889566421509,
|
||
|
|
"logps/chosen": -39.66600036621094,
|
||
|
|
"logps/rejected": -41.07653045654297,
|
||
|
|
"loss": 0.2857,
|
||
|
|
"rewards/accuracies": 0.8055555820465088,
|
||
|
|
"rewards/chosen": 0.20397840440273285,
|
||
|
|
"rewards/margins": 2.4769766330718994,
|
||
|
|
"rewards/rejected": -2.272998094558716,
|
||
|
|
"step": 494
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1447621489934607,
|
||
|
|
"grad_norm": 27.504424003065566,
|
||
|
|
"learning_rate": 2.3036009853481474e-07,
|
||
|
|
"logits/chosen": 0.5301830768585205,
|
||
|
|
"logits/rejected": 0.5608452558517456,
|
||
|
|
"logps/chosen": -39.39542770385742,
|
||
|
|
"logps/rejected": -58.36659622192383,
|
||
|
|
"loss": 0.2681,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.2189822793006897,
|
||
|
|
"rewards/margins": 3.4378933906555176,
|
||
|
|
"rewards/rejected": -3.6568756103515625,
|
||
|
|
"step": 496
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1493781254006925,
|
||
|
|
"grad_norm": 16.835368907101664,
|
||
|
|
"learning_rate": 2.283505644942223e-07,
|
||
|
|
"logits/chosen": 0.5190525054931641,
|
||
|
|
"logits/rejected": 0.5493537783622742,
|
||
|
|
"logps/chosen": -34.43808364868164,
|
||
|
|
"logps/rejected": -54.84063720703125,
|
||
|
|
"loss": 0.1937,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.13352231681346893,
|
||
|
|
"rewards/margins": 3.440141201019287,
|
||
|
|
"rewards/rejected": -3.3066186904907227,
|
||
|
|
"step": 498
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.153994101807924,
|
||
|
|
"grad_norm": 14.320814422051418,
|
||
|
|
"learning_rate": 2.2634243886344781e-07,
|
||
|
|
"logits/chosen": 0.5132643580436707,
|
||
|
|
"logits/rejected": 0.5381724834442139,
|
||
|
|
"logps/chosen": -41.94618225097656,
|
||
|
|
"logps/rejected": -54.74879455566406,
|
||
|
|
"loss": 0.243,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.1846380978822708,
|
||
|
|
"rewards/margins": 3.523959159851074,
|
||
|
|
"rewards/rejected": -3.3393211364746094,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.153994101807924,
|
||
|
|
"eval_logits/chosen": 0.43241602182388306,
|
||
|
|
"eval_logits/rejected": 0.45862025022506714,
|
||
|
|
"eval_logps/chosen": -41.512245178222656,
|
||
|
|
"eval_logps/rejected": -54.365325927734375,
|
||
|
|
"eval_loss": 0.24479356408119202,
|
||
|
|
"eval_rewards/accuracies": 0.8289170265197754,
|
||
|
|
"eval_rewards/chosen": 0.11651827394962311,
|
||
|
|
"eval_rewards/margins": 3.2800135612487793,
|
||
|
|
"eval_rewards/rejected": -3.1634950637817383,
|
||
|
|
"eval_runtime": 220.3257,
|
||
|
|
"eval_samples_per_second": 7.87,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1586100782151558,
|
||
|
|
"grad_norm": 17.24901468893502,
|
||
|
|
"learning_rate": 2.2433585228164115e-07,
|
||
|
|
"logits/chosen": 0.5386977791786194,
|
||
|
|
"logits/rejected": 0.5774834156036377,
|
||
|
|
"logps/chosen": -43.753910064697266,
|
||
|
|
"logps/rejected": -65.60494232177734,
|
||
|
|
"loss": 0.1918,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.19071653485298157,
|
||
|
|
"rewards/margins": 4.159061908721924,
|
||
|
|
"rewards/rejected": -3.9683446884155273,
|
||
|
|
"step": 502
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1632260546223874,
|
||
|
|
"grad_norm": 22.994462305856853,
|
||
|
|
"learning_rate": 2.2233093528782938e-07,
|
||
|
|
"logits/chosen": 0.5429908037185669,
|
||
|
|
"logits/rejected": 0.5663915872573853,
|
||
|
|
"logps/chosen": -49.295047760009766,
|
||
|
|
"logps/rejected": -58.83778381347656,
|
||
|
|
"loss": 0.1741,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": 0.27108439803123474,
|
||
|
|
"rewards/margins": 3.4974775314331055,
|
||
|
|
"rewards/rejected": -3.226392984390259,
|
||
|
|
"step": 504
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1678420310296191,
|
||
|
|
"grad_norm": 19.749474882703815,
|
||
|
|
"learning_rate": 2.2032781831242367e-07,
|
||
|
|
"logits/chosen": 0.5360143184661865,
|
||
|
|
"logits/rejected": 0.5641200542449951,
|
||
|
|
"logps/chosen": -35.82609558105469,
|
||
|
|
"logps/rejected": -44.779361724853516,
|
||
|
|
"loss": 0.2253,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.4115668535232544,
|
||
|
|
"rewards/margins": 2.9376118183135986,
|
||
|
|
"rewards/rejected": -2.526045083999634,
|
||
|
|
"step": 506
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.172458007436851,
|
||
|
|
"grad_norm": 29.881557534524536,
|
||
|
|
"learning_rate": 2.183266316687347e-07,
|
||
|
|
"logits/chosen": 0.5799429416656494,
|
||
|
|
"logits/rejected": 0.5963388681411743,
|
||
|
|
"logps/chosen": -42.11252975463867,
|
||
|
|
"logps/rejected": -44.56486511230469,
|
||
|
|
"loss": 0.2905,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": 0.2770005762577057,
|
||
|
|
"rewards/margins": 2.54060435295105,
|
||
|
|
"rewards/rejected": -2.263603687286377,
|
||
|
|
"step": 508
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1770739838440827,
|
||
|
|
"grad_norm": 11.72889590765659,
|
||
|
|
"learning_rate": 2.16327505544495e-07,
|
||
|
|
"logits/chosen": 0.5231108069419861,
|
||
|
|
"logits/rejected": 0.5499060153961182,
|
||
|
|
"logps/chosen": -43.436798095703125,
|
||
|
|
"logps/rejected": -57.92034912109375,
|
||
|
|
"loss": 0.1472,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": 0.47280406951904297,
|
||
|
|
"rewards/margins": 4.098244667053223,
|
||
|
|
"rewards/rejected": -3.625440835952759,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1770739838440827,
|
||
|
|
"eval_logits/chosen": 0.43323588371276855,
|
||
|
|
"eval_logits/rejected": 0.4594508111476898,
|
||
|
|
"eval_logps/chosen": -41.14154815673828,
|
||
|
|
"eval_logps/rejected": -54.075172424316406,
|
||
|
|
"eval_loss": 0.247583270072937,
|
||
|
|
"eval_rewards/accuracies": 0.828341007232666,
|
||
|
|
"eval_rewards/chosen": 0.30186572670936584,
|
||
|
|
"eval_rewards/margins": 3.3202853202819824,
|
||
|
|
"eval_rewards/rejected": -3.0184197425842285,
|
||
|
|
"eval_runtime": 220.3645,
|
||
|
|
"eval_samples_per_second": 7.869,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1816899602513142,
|
||
|
|
"grad_norm": 19.02915371887465,
|
||
|
|
"learning_rate": 2.143305699933892e-07,
|
||
|
|
"logits/chosen": 0.5309435725212097,
|
||
|
|
"logits/rejected": 0.5609121322631836,
|
||
|
|
"logps/chosen": -39.10821533203125,
|
||
|
|
"logps/rejected": -55.85133743286133,
|
||
|
|
"loss": 0.2148,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.35923802852630615,
|
||
|
|
"rewards/margins": 3.6412789821624756,
|
||
|
|
"rewards/rejected": -3.282041549682617,
|
||
|
|
"step": 512
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.186305936658546,
|
||
|
|
"grad_norm": 18.184730820886717,
|
||
|
|
"learning_rate": 2.1233595492659382e-07,
|
||
|
|
"logits/chosen": 0.6312618851661682,
|
||
|
|
"logits/rejected": 0.6453579068183899,
|
||
|
|
"logps/chosen": -48.93413543701172,
|
||
|
|
"logps/rejected": -50.58020782470703,
|
||
|
|
"loss": 0.1701,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.28959882259368896,
|
||
|
|
"rewards/margins": 3.4992854595184326,
|
||
|
|
"rewards/rejected": -3.209686040878296,
|
||
|
|
"step": 514
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1909219130657775,
|
||
|
|
"grad_norm": 21.115621604290848,
|
||
|
|
"learning_rate": 2.1034379010432542e-07,
|
||
|
|
"logits/chosen": 0.5738712549209595,
|
||
|
|
"logits/rejected": 0.5990296006202698,
|
||
|
|
"logps/chosen": -36.4149055480957,
|
||
|
|
"logps/rejected": -47.95274353027344,
|
||
|
|
"loss": 0.2192,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.35762646794319153,
|
||
|
|
"rewards/margins": 3.1450395584106445,
|
||
|
|
"rewards/rejected": -2.7874133586883545,
|
||
|
|
"step": 516
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1955378894730093,
|
||
|
|
"grad_norm": 18.313049973835163,
|
||
|
|
"learning_rate": 2.0835420512739957e-07,
|
||
|
|
"logits/chosen": 0.48849010467529297,
|
||
|
|
"logits/rejected": 0.5418619513511658,
|
||
|
|
"logps/chosen": -39.52627182006836,
|
||
|
|
"logps/rejected": -70.53701782226562,
|
||
|
|
"loss": 0.1678,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.39579084515571594,
|
||
|
|
"rewards/margins": 4.528857231140137,
|
||
|
|
"rewards/rejected": -4.133066654205322,
|
||
|
|
"step": 518
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.200153865880241,
|
||
|
|
"grad_norm": 18.512425100692376,
|
||
|
|
"learning_rate": 2.0636732942879917e-07,
|
||
|
|
"logits/chosen": 0.5643823146820068,
|
||
|
|
"logits/rejected": 0.5917804837226868,
|
||
|
|
"logps/chosen": -43.44633483886719,
|
||
|
|
"logps/rejected": -56.26163101196289,
|
||
|
|
"loss": 0.166,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": 0.33819130063056946,
|
||
|
|
"rewards/margins": 3.693488121032715,
|
||
|
|
"rewards/rejected": -3.3552963733673096,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.200153865880241,
|
||
|
|
"eval_logits/chosen": 0.4335879981517792,
|
||
|
|
"eval_logits/rejected": 0.45994046330451965,
|
||
|
|
"eval_logps/chosen": -41.402774810791016,
|
||
|
|
"eval_logps/rejected": -54.35234451293945,
|
||
|
|
"eval_loss": 0.2449788749217987,
|
||
|
|
"eval_rewards/accuracies": 0.8317972421646118,
|
||
|
|
"eval_rewards/chosen": 0.1712525486946106,
|
||
|
|
"eval_rewards/margins": 3.328258991241455,
|
||
|
|
"eval_rewards/rejected": -3.1570065021514893,
|
||
|
|
"eval_runtime": 220.2998,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2047698422874729,
|
||
|
|
"grad_norm": 11.696545134195471,
|
||
|
|
"learning_rate": 2.0438329226525415e-07,
|
||
|
|
"logits/chosen": 0.5642399787902832,
|
||
|
|
"logits/rejected": 0.587860643863678,
|
||
|
|
"logps/chosen": -41.212337493896484,
|
||
|
|
"logps/rejected": -43.521636962890625,
|
||
|
|
"loss": 0.2246,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.5518161058425903,
|
||
|
|
"rewards/margins": 2.9677634239196777,
|
||
|
|
"rewards/rejected": -2.415947675704956,
|
||
|
|
"step": 522
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2093858186947044,
|
||
|
|
"grad_norm": 24.196902238001236,
|
||
|
|
"learning_rate": 2.0240222270883288e-07,
|
||
|
|
"logits/chosen": 0.5227870941162109,
|
||
|
|
"logits/rejected": 0.5579611659049988,
|
||
|
|
"logps/chosen": -44.49864196777344,
|
||
|
|
"logps/rejected": -64.84123229980469,
|
||
|
|
"loss": 0.2314,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.16053809225559235,
|
||
|
|
"rewards/margins": 3.896054267883301,
|
||
|
|
"rewards/rejected": -3.73551607131958,
|
||
|
|
"step": 524
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2140017951019362,
|
||
|
|
"grad_norm": 12.971615376216704,
|
||
|
|
"learning_rate": 2.0042424963854542e-07,
|
||
|
|
"logits/chosen": 0.5063973665237427,
|
||
|
|
"logits/rejected": 0.5544097423553467,
|
||
|
|
"logps/chosen": -40.40736389160156,
|
||
|
|
"logps/rejected": -70.9152603149414,
|
||
|
|
"loss": 0.1526,
|
||
|
|
"rewards/accuracies": 0.9444444179534912,
|
||
|
|
"rewards/chosen": 0.3248124122619629,
|
||
|
|
"rewards/margins": 4.234506607055664,
|
||
|
|
"rewards/rejected": -3.9096946716308594,
|
||
|
|
"step": 526
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2186177715091677,
|
||
|
|
"grad_norm": 14.0866861852398,
|
||
|
|
"learning_rate": 1.9844950173195883e-07,
|
||
|
|
"logits/chosen": 0.5182596445083618,
|
||
|
|
"logits/rejected": 0.549498975276947,
|
||
|
|
"logps/chosen": -39.39563751220703,
|
||
|
|
"logps/rejected": -54.05485153198242,
|
||
|
|
"loss": 0.1818,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.22824376821517944,
|
||
|
|
"rewards/margins": 3.397740364074707,
|
||
|
|
"rewards/rejected": -3.169497013092041,
|
||
|
|
"step": 528
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2232337479163995,
|
||
|
|
"grad_norm": 13.76916365285817,
|
||
|
|
"learning_rate": 1.964781074568265e-07,
|
||
|
|
"logits/chosen": 0.5031299591064453,
|
||
|
|
"logits/rejected": 0.5121724009513855,
|
||
|
|
"logps/chosen": -41.18108367919922,
|
||
|
|
"logps/rejected": -45.627994537353516,
|
||
|
|
"loss": 0.1945,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": 0.03019801713526249,
|
||
|
|
"rewards/margins": 3.0934128761291504,
|
||
|
|
"rewards/rejected": -3.0632145404815674,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2232337479163995,
|
||
|
|
"eval_logits/chosen": 0.43405523896217346,
|
||
|
|
"eval_logits/rejected": 0.46039465069770813,
|
||
|
|
"eval_logps/chosen": -41.60369110107422,
|
||
|
|
"eval_logps/rejected": -54.51262664794922,
|
||
|
|
"eval_loss": 0.24258121848106384,
|
||
|
|
"eval_rewards/accuracies": 0.8335253596305847,
|
||
|
|
"eval_rewards/chosen": 0.07079467922449112,
|
||
|
|
"eval_rewards/margins": 3.3079416751861572,
|
||
|
|
"eval_rewards/rejected": -3.2371468544006348,
|
||
|
|
"eval_runtime": 220.2641,
|
||
|
|
"eval_samples_per_second": 7.872,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2278497243236313,
|
||
|
|
"grad_norm": 16.411903473780164,
|
||
|
|
"learning_rate": 1.9451019506273018e-07,
|
||
|
|
"logits/chosen": 0.541588306427002,
|
||
|
|
"logits/rejected": 0.5615941286087036,
|
||
|
|
"logps/chosen": -36.563297271728516,
|
||
|
|
"logps/rejected": -48.32072448730469,
|
||
|
|
"loss": 0.2351,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.17822687327861786,
|
||
|
|
"rewards/margins": 2.845065116882324,
|
||
|
|
"rewards/rejected": -2.6668384075164795,
|
||
|
|
"step": 532
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.232465700730863,
|
||
|
|
"grad_norm": 13.467269631637619,
|
||
|
|
"learning_rate": 1.9254589257273712e-07,
|
||
|
|
"logits/chosen": 0.5137292146682739,
|
||
|
|
"logits/rejected": 0.5505712032318115,
|
||
|
|
"logps/chosen": -36.598384857177734,
|
||
|
|
"logps/rejected": -57.48229217529297,
|
||
|
|
"loss": 0.1473,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.19568167626857758,
|
||
|
|
"rewards/margins": 4.128161907196045,
|
||
|
|
"rewards/rejected": -3.9324798583984375,
|
||
|
|
"step": 534
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2370816771380946,
|
||
|
|
"grad_norm": 24.645788661655104,
|
||
|
|
"learning_rate": 1.9058532777507141e-07,
|
||
|
|
"logits/chosen": 0.5294635891914368,
|
||
|
|
"logits/rejected": 0.5472697615623474,
|
||
|
|
"logps/chosen": -39.22220230102539,
|
||
|
|
"logps/rejected": -49.91395950317383,
|
||
|
|
"loss": 0.2172,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.11992057412862778,
|
||
|
|
"rewards/margins": 3.224815845489502,
|
||
|
|
"rewards/rejected": -3.1048953533172607,
|
||
|
|
"step": 536
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2416976535453264,
|
||
|
|
"grad_norm": 18.291984511184836,
|
||
|
|
"learning_rate": 1.886286282148002e-07,
|
||
|
|
"logits/chosen": 0.5298857688903809,
|
||
|
|
"logits/rejected": 0.5633623600006104,
|
||
|
|
"logps/chosen": -41.294647216796875,
|
||
|
|
"logps/rejected": -57.79304885864258,
|
||
|
|
"loss": 0.2731,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.11145944148302078,
|
||
|
|
"rewards/margins": 3.1801443099975586,
|
||
|
|
"rewards/rejected": -3.2916040420532227,
|
||
|
|
"step": 538
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.246313629952558,
|
||
|
|
"grad_norm": 17.71916747448851,
|
||
|
|
"learning_rate": 1.8667592118553693e-07,
|
||
|
|
"logits/chosen": 0.5349301099777222,
|
||
|
|
"logits/rejected": 0.5512058734893799,
|
||
|
|
"logps/chosen": -43.72676467895508,
|
||
|
|
"logps/rejected": -52.80296325683594,
|
||
|
|
"loss": 0.2216,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.03689540922641754,
|
||
|
|
"rewards/margins": 3.2271673679351807,
|
||
|
|
"rewards/rejected": -3.2640628814697266,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.246313629952558,
|
||
|
|
"eval_logits/chosen": 0.4325529932975769,
|
||
|
|
"eval_logits/rejected": 0.45892781019210815,
|
||
|
|
"eval_logps/chosen": -41.67875289916992,
|
||
|
|
"eval_logps/rejected": -54.59620666503906,
|
||
|
|
"eval_loss": 0.24205271899700165,
|
||
|
|
"eval_rewards/accuracies": 0.8277649879455566,
|
||
|
|
"eval_rewards/chosen": 0.03326287120580673,
|
||
|
|
"eval_rewards/margins": 3.312199115753174,
|
||
|
|
"eval_rewards/rejected": -3.2789359092712402,
|
||
|
|
"eval_runtime": 220.1774,
|
||
|
|
"eval_samples_per_second": 7.875,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2509296063597897,
|
||
|
|
"grad_norm": 15.1063531754732,
|
||
|
|
"learning_rate": 1.8472733372115956e-07,
|
||
|
|
"logits/chosen": 0.4958040416240692,
|
||
|
|
"logits/rejected": 0.5259097814559937,
|
||
|
|
"logps/chosen": -43.43186950683594,
|
||
|
|
"logps/rejected": -60.27039337158203,
|
||
|
|
"loss": 0.1823,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.40173831582069397,
|
||
|
|
"rewards/margins": 3.8025894165039062,
|
||
|
|
"rewards/rejected": -4.2043280601501465,
|
||
|
|
"step": 542
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2555455827670214,
|
||
|
|
"grad_norm": 23.60965925798032,
|
||
|
|
"learning_rate": 1.8278299258754692e-07,
|
||
|
|
"logits/chosen": 0.47050708532333374,
|
||
|
|
"logits/rejected": 0.5154716968536377,
|
||
|
|
"logps/chosen": -43.42805480957031,
|
||
|
|
"logps/rejected": -71.56327056884766,
|
||
|
|
"loss": 0.2284,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.35217729210853577,
|
||
|
|
"rewards/margins": 4.311697483062744,
|
||
|
|
"rewards/rejected": -4.663875102996826,
|
||
|
|
"step": 544
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2601615591742532,
|
||
|
|
"grad_norm": 11.785150141913245,
|
||
|
|
"learning_rate": 1.808430242743316e-07,
|
||
|
|
"logits/chosen": 0.46195343136787415,
|
||
|
|
"logits/rejected": 0.4784909784793854,
|
||
|
|
"logps/chosen": -42.974945068359375,
|
||
|
|
"logps/rejected": -54.21615219116211,
|
||
|
|
"loss": 0.1867,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": 0.31646448373794556,
|
||
|
|
"rewards/margins": 3.5641021728515625,
|
||
|
|
"rewards/rejected": -3.2476377487182617,
|
||
|
|
"step": 546
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2647775355814848,
|
||
|
|
"grad_norm": 13.346160813344762,
|
||
|
|
"learning_rate": 1.7890755498667104e-07,
|
||
|
|
"logits/chosen": 0.5626040101051331,
|
||
|
|
"logits/rejected": 0.5980097651481628,
|
||
|
|
"logps/chosen": -36.59039306640625,
|
||
|
|
"logps/rejected": -55.57601547241211,
|
||
|
|
"loss": 0.182,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.17459021508693695,
|
||
|
|
"rewards/margins": 3.451416015625,
|
||
|
|
"rewards/rejected": -3.2768259048461914,
|
||
|
|
"step": 548
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2693935119887165,
|
||
|
|
"grad_norm": 25.621843956328824,
|
||
|
|
"learning_rate": 1.7697671063703756e-07,
|
||
|
|
"logits/chosen": 0.5085393786430359,
|
||
|
|
"logits/rejected": 0.5440909266471863,
|
||
|
|
"logps/chosen": -39.27238464355469,
|
||
|
|
"logps/rejected": -59.40525817871094,
|
||
|
|
"loss": 0.2243,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.011964095756411552,
|
||
|
|
"rewards/margins": 3.6004185676574707,
|
||
|
|
"rewards/rejected": -3.588454246520996,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2693935119887165,
|
||
|
|
"eval_logits/chosen": 0.4355390965938568,
|
||
|
|
"eval_logits/rejected": 0.46181005239486694,
|
||
|
|
"eval_logps/chosen": -41.701602935791016,
|
||
|
|
"eval_logps/rejected": -54.663360595703125,
|
||
|
|
"eval_loss": 0.24010230600833893,
|
||
|
|
"eval_rewards/accuracies": 0.8260368704795837,
|
||
|
|
"eval_rewards/chosen": 0.0218377523124218,
|
||
|
|
"eval_rewards/margins": 3.3343515396118164,
|
||
|
|
"eval_rewards/rejected": -3.312513828277588,
|
||
|
|
"eval_runtime": 220.234,
|
||
|
|
"eval_samples_per_second": 7.873,
|
||
|
|
"eval_steps_per_second": 1.971,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.274009488395948,
|
||
|
|
"grad_norm": 29.85339571581757,
|
||
|
|
"learning_rate": 1.750506168370267e-07,
|
||
|
|
"logits/chosen": 0.5484946370124817,
|
||
|
|
"logits/rejected": 0.5642725229263306,
|
||
|
|
"logps/chosen": -40.738338470458984,
|
||
|
|
"logps/rejected": -47.2222900390625,
|
||
|
|
"loss": 0.2665,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.49148619174957275,
|
||
|
|
"rewards/margins": 3.0378835201263428,
|
||
|
|
"rewards/rejected": -2.5463972091674805,
|
||
|
|
"step": 552
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2786254648031798,
|
||
|
|
"grad_norm": 11.606234417897845,
|
||
|
|
"learning_rate": 1.7312939888918594e-07,
|
||
|
|
"logits/chosen": 0.5540368556976318,
|
||
|
|
"logits/rejected": 0.5830137729644775,
|
||
|
|
"logps/chosen": -43.42100143432617,
|
||
|
|
"logps/rejected": -63.07583999633789,
|
||
|
|
"loss": 0.1529,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": 0.060752179473638535,
|
||
|
|
"rewards/margins": 3.951368570327759,
|
||
|
|
"rewards/rejected": -3.8906164169311523,
|
||
|
|
"step": 554
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2832414412104116,
|
||
|
|
"grad_norm": 8.195981315855988,
|
||
|
|
"learning_rate": 1.712131817788628e-07,
|
||
|
|
"logits/chosen": 0.5598903298377991,
|
||
|
|
"logits/rejected": 0.582931637763977,
|
||
|
|
"logps/chosen": -39.05931854248047,
|
||
|
|
"logps/rejected": -49.5858154296875,
|
||
|
|
"loss": 0.2278,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.035774629563093185,
|
||
|
|
"rewards/margins": 3.2900662422180176,
|
||
|
|
"rewards/rejected": -3.325840950012207,
|
||
|
|
"step": 556
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2878574176176434,
|
||
|
|
"grad_norm": 10.58953396876903,
|
||
|
|
"learning_rate": 1.693020901660738e-07,
|
||
|
|
"logits/chosen": 0.5586022138595581,
|
||
|
|
"logits/rejected": 0.5835521221160889,
|
||
|
|
"logps/chosen": -46.566070556640625,
|
||
|
|
"logps/rejected": -56.1746940612793,
|
||
|
|
"loss": 0.1347,
|
||
|
|
"rewards/accuracies": 0.9583333134651184,
|
||
|
|
"rewards/chosen": 0.1323520541191101,
|
||
|
|
"rewards/margins": 3.951080322265625,
|
||
|
|
"rewards/rejected": -3.81872820854187,
|
||
|
|
"step": 558
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.292473394024875,
|
||
|
|
"grad_norm": 20.647672350132265,
|
||
|
|
"learning_rate": 1.6739624837739518e-07,
|
||
|
|
"logits/chosen": 0.4893258512020111,
|
||
|
|
"logits/rejected": 0.5065658092498779,
|
||
|
|
"logps/chosen": -46.70867919921875,
|
||
|
|
"logps/rejected": -53.02800369262695,
|
||
|
|
"loss": 0.2073,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.15436476469039917,
|
||
|
|
"rewards/margins": 3.050819158554077,
|
||
|
|
"rewards/rejected": -3.205183744430542,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.292473394024875,
|
||
|
|
"eval_logits/chosen": 0.4335208237171173,
|
||
|
|
"eval_logits/rejected": 0.45989227294921875,
|
||
|
|
"eval_logps/chosen": -41.82432556152344,
|
||
|
|
"eval_logps/rejected": -54.859825134277344,
|
||
|
|
"eval_loss": 0.23924875259399414,
|
||
|
|
"eval_rewards/accuracies": 0.8312212228775024,
|
||
|
|
"eval_rewards/chosen": -0.03952277451753616,
|
||
|
|
"eval_rewards/margins": 3.371224880218506,
|
||
|
|
"eval_rewards/rejected": -3.410747766494751,
|
||
|
|
"eval_runtime": 220.3082,
|
||
|
|
"eval_samples_per_second": 7.871,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2970893704321067,
|
||
|
|
"grad_norm": 15.328848187023517,
|
||
|
|
"learning_rate": 1.6549578039787434e-07,
|
||
|
|
"logits/chosen": 0.5223647356033325,
|
||
|
|
"logits/rejected": 0.5576710104942322,
|
||
|
|
"logps/chosen": -43.448875427246094,
|
||
|
|
"logps/rejected": -67.14339447021484,
|
||
|
|
"loss": 0.2405,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.1683083474636078,
|
||
|
|
"rewards/margins": 3.6626782417297363,
|
||
|
|
"rewards/rejected": -3.830986499786377,
|
||
|
|
"step": 562
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3017053468393383,
|
||
|
|
"grad_norm": 14.362719389125761,
|
||
|
|
"learning_rate": 1.6360080986296384e-07,
|
||
|
|
"logits/chosen": 0.5163556337356567,
|
||
|
|
"logits/rejected": 0.5569749474525452,
|
||
|
|
"logps/chosen": -37.78327941894531,
|
||
|
|
"logps/rejected": -64.23339080810547,
|
||
|
|
"loss": 0.186,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.047993943095207214,
|
||
|
|
"rewards/margins": 4.109629154205322,
|
||
|
|
"rewards/rejected": -4.157623291015625,
|
||
|
|
"step": 564
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.30632132324657,
|
||
|
|
"grad_norm": 8.849930925918736,
|
||
|
|
"learning_rate": 1.6171146005047894e-07,
|
||
|
|
"logits/chosen": 0.5622715353965759,
|
||
|
|
"logits/rejected": 0.5891626477241516,
|
||
|
|
"logps/chosen": -46.50107955932617,
|
||
|
|
"logps/rejected": -63.37003707885742,
|
||
|
|
"loss": 0.1689,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.1092919185757637,
|
||
|
|
"rewards/margins": 4.0769548416137695,
|
||
|
|
"rewards/rejected": -3.967662811279297,
|
||
|
|
"step": 566
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3109372996538018,
|
||
|
|
"grad_norm": 16.110148125770678,
|
||
|
|
"learning_rate": 1.5982785387257694e-07,
|
||
|
|
"logits/chosen": 0.5649956464767456,
|
||
|
|
"logits/rejected": 0.5782197117805481,
|
||
|
|
"logps/chosen": -43.4311408996582,
|
||
|
|
"logps/rejected": -49.03315734863281,
|
||
|
|
"loss": 0.2002,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.15342091023921967,
|
||
|
|
"rewards/margins": 2.909942150115967,
|
||
|
|
"rewards/rejected": -3.0633630752563477,
|
||
|
|
"step": 568
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3155532760610336,
|
||
|
|
"grad_norm": 23.725153045927403,
|
||
|
|
"learning_rate": 1.5795011386776159e-07,
|
||
|
|
"logits/chosen": 0.5103439688682556,
|
||
|
|
"logits/rejected": 0.5300507545471191,
|
||
|
|
"logps/chosen": -42.80021667480469,
|
||
|
|
"logps/rejected": -47.7119255065918,
|
||
|
|
"loss": 0.2255,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.14677530527114868,
|
||
|
|
"rewards/margins": 3.0557618141174316,
|
||
|
|
"rewards/rejected": -3.2025370597839355,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3155532760610336,
|
||
|
|
"eval_logits/chosen": 0.43335986137390137,
|
||
|
|
"eval_logits/rejected": 0.4598417580127716,
|
||
|
|
"eval_logps/chosen": -41.851680755615234,
|
||
|
|
"eval_logps/rejected": -54.97309112548828,
|
||
|
|
"eval_loss": 0.23906731605529785,
|
||
|
|
"eval_rewards/accuracies": 0.835829496383667,
|
||
|
|
"eval_rewards/chosen": -0.05320117622613907,
|
||
|
|
"eval_rewards/margins": 3.4141783714294434,
|
||
|
|
"eval_rewards/rejected": -3.467379570007324,
|
||
|
|
"eval_runtime": 220.3588,
|
||
|
|
"eval_samples_per_second": 7.869,
|
||
|
|
"eval_steps_per_second": 1.97,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.320169252468265,
|
||
|
|
"grad_norm": 16.172756609459842,
|
||
|
|
"learning_rate": 1.560783621929113e-07,
|
||
|
|
"logits/chosen": 0.5175637006759644,
|
||
|
|
"logits/rejected": 0.5229324102401733,
|
||
|
|
"logps/chosen": -49.446102142333984,
|
||
|
|
"logps/rejected": -55.164894104003906,
|
||
|
|
"loss": 0.1869,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": 0.08015252649784088,
|
||
|
|
"rewards/margins": 3.3609066009521484,
|
||
|
|
"rewards/rejected": -3.2807538509368896,
|
||
|
|
"step": 572
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3247852288754969,
|
||
|
|
"grad_norm": 24.72268513177688,
|
||
|
|
"learning_rate": 1.5421272061533177e-07,
|
||
|
|
"logits/chosen": 0.5066720247268677,
|
||
|
|
"logits/rejected": 0.5451788306236267,
|
||
|
|
"logps/chosen": -37.343570709228516,
|
||
|
|
"logps/rejected": -60.23046112060547,
|
||
|
|
"loss": 0.2949,
|
||
|
|
"rewards/accuracies": 0.7777777910232544,
|
||
|
|
"rewards/chosen": 0.1486133188009262,
|
||
|
|
"rewards/margins": 3.3898818492889404,
|
||
|
|
"rewards/rejected": -3.2412681579589844,
|
||
|
|
"step": 574
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3294012052827284,
|
||
|
|
"grad_norm": 18.734543272703554,
|
||
|
|
"learning_rate": 1.5235331050483513e-07,
|
||
|
|
"logits/chosen": 0.5524860620498657,
|
||
|
|
"logits/rejected": 0.5772072672843933,
|
||
|
|
"logps/chosen": -43.33749771118164,
|
||
|
|
"logps/rejected": -56.5976676940918,
|
||
|
|
"loss": 0.2367,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.3428743779659271,
|
||
|
|
"rewards/margins": 3.3112895488739014,
|
||
|
|
"rewards/rejected": -3.6541638374328613,
|
||
|
|
"step": 576
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3340171816899602,
|
||
|
|
"grad_norm": 15.636365920242639,
|
||
|
|
"learning_rate": 1.5050025282584327e-07,
|
||
|
|
"logits/chosen": 0.5805926322937012,
|
||
|
|
"logits/rejected": 0.6090676188468933,
|
||
|
|
"logps/chosen": -49.13417434692383,
|
||
|
|
"logps/rejected": -64.1076431274414,
|
||
|
|
"loss": 0.1791,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.08167193830013275,
|
||
|
|
"rewards/margins": 3.955726146697998,
|
||
|
|
"rewards/rejected": -4.037397861480713,
|
||
|
|
"step": 578
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.338633158097192,
|
||
|
|
"grad_norm": 15.524132351808905,
|
||
|
|
"learning_rate": 1.4865366812951921e-07,
|
||
|
|
"logits/chosen": 0.598872721195221,
|
||
|
|
"logits/rejected": 0.62497878074646,
|
||
|
|
"logps/chosen": -36.58146667480469,
|
||
|
|
"logps/rejected": -46.25484085083008,
|
||
|
|
"loss": 0.1893,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.01747778430581093,
|
||
|
|
"rewards/margins": 3.4903595447540283,
|
||
|
|
"rewards/rejected": -3.5078377723693848,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.338633158097192,
|
||
|
|
"eval_logits/chosen": 0.4342789053916931,
|
||
|
|
"eval_logits/rejected": 0.46078288555145264,
|
||
|
|
"eval_logps/chosen": -42.1205940246582,
|
||
|
|
"eval_logps/rejected": -55.25835418701172,
|
||
|
|
"eval_loss": 0.2389531433582306,
|
||
|
|
"eval_rewards/accuracies": 0.8352534770965576,
|
||
|
|
"eval_rewards/chosen": -0.18765874207019806,
|
||
|
|
"eval_rewards/margins": 3.4223523139953613,
|
||
|
|
"eval_rewards/rejected": -3.610011339187622,
|
||
|
|
"eval_runtime": 220.361,
|
||
|
|
"eval_samples_per_second": 7.869,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3432491345044237,
|
||
|
|
"grad_norm": 22.418640332294185,
|
||
|
|
"learning_rate": 1.4681367654592446e-07,
|
||
|
|
"logits/chosen": 0.583182692527771,
|
||
|
|
"logits/rejected": 0.596510112285614,
|
||
|
|
"logps/chosen": -45.08745574951172,
|
||
|
|
"logps/rejected": -52.57502746582031,
|
||
|
|
"loss": 0.1635,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.24202129244804382,
|
||
|
|
"rewards/margins": 3.0601682662963867,
|
||
|
|
"rewards/rejected": -3.302189826965332,
|
||
|
|
"step": 582
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3478651109116553,
|
||
|
|
"grad_norm": 16.477398466397805,
|
||
|
|
"learning_rate": 1.4498039777620353e-07,
|
||
|
|
"logits/chosen": 0.5257098078727722,
|
||
|
|
"logits/rejected": 0.5561378598213196,
|
||
|
|
"logps/chosen": -49.92831039428711,
|
||
|
|
"logps/rejected": -66.70814514160156,
|
||
|
|
"loss": 0.1983,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": 0.07970259338617325,
|
||
|
|
"rewards/margins": 4.159069538116455,
|
||
|
|
"rewards/rejected": -4.079366683959961,
|
||
|
|
"step": 584
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.352481087318887,
|
||
|
|
"grad_norm": 21.638282072644653,
|
||
|
|
"learning_rate": 1.4315395108479728e-07,
|
||
|
|
"logits/chosen": 0.5448426008224487,
|
||
|
|
"logits/rejected": 0.5733739733695984,
|
||
|
|
"logps/chosen": -42.567203521728516,
|
||
|
|
"logps/rejected": -59.23841094970703,
|
||
|
|
"loss": 0.1872,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.3566977083683014,
|
||
|
|
"rewards/margins": 3.441741943359375,
|
||
|
|
"rewards/rejected": -3.7984399795532227,
|
||
|
|
"step": 586
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3570970637261186,
|
||
|
|
"grad_norm": 22.386629994354788,
|
||
|
|
"learning_rate": 1.4133445529168365e-07,
|
||
|
|
"logits/chosen": 0.5482079982757568,
|
||
|
|
"logits/rejected": 0.5674624443054199,
|
||
|
|
"logps/chosen": -47.31834030151367,
|
||
|
|
"logps/rejected": -59.47747802734375,
|
||
|
|
"loss": 0.1735,
|
||
|
|
"rewards/accuracies": 0.9444444179534912,
|
||
|
|
"rewards/chosen": -0.25350263714790344,
|
||
|
|
"rewards/margins": 3.711785316467285,
|
||
|
|
"rewards/rejected": -3.965287923812866,
|
||
|
|
"step": 588
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3617130401333504,
|
||
|
|
"grad_norm": 14.716672759245373,
|
||
|
|
"learning_rate": 1.395220287646483e-07,
|
||
|
|
"logits/chosen": 0.5413531064987183,
|
||
|
|
"logits/rejected": 0.5619943141937256,
|
||
|
|
"logps/chosen": -45.74396514892578,
|
||
|
|
"logps/rejected": -54.50990295410156,
|
||
|
|
"loss": 0.1609,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.3855374753475189,
|
||
|
|
"rewards/margins": 3.439289093017578,
|
||
|
|
"rewards/rejected": -3.82482647895813,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3617130401333504,
|
||
|
|
"eval_logits/chosen": 0.43462061882019043,
|
||
|
|
"eval_logits/rejected": 0.461076945066452,
|
||
|
|
"eval_logps/chosen": -42.448509216308594,
|
||
|
|
"eval_logps/rejected": -55.58904266357422,
|
||
|
|
"eval_loss": 0.2393806427717209,
|
||
|
|
"eval_rewards/accuracies": 0.8317972421646118,
|
||
|
|
"eval_rewards/chosen": -0.3516136407852173,
|
||
|
|
"eval_rewards/margins": 3.4237425327301025,
|
||
|
|
"eval_rewards/rejected": -3.7753562927246094,
|
||
|
|
"eval_runtime": 220.4141,
|
||
|
|
"eval_samples_per_second": 7.867,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3663290165405821,
|
||
|
|
"grad_norm": 21.200823940085225,
|
||
|
|
"learning_rate": 1.377167894115837e-07,
|
||
|
|
"logits/chosen": 0.562565803527832,
|
||
|
|
"logits/rejected": 0.6183031797409058,
|
||
|
|
"logps/chosen": -38.32450866699219,
|
||
|
|
"logps/rejected": -68.53689575195312,
|
||
|
|
"loss": 0.179,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.12118612229824066,
|
||
|
|
"rewards/margins": 4.04473876953125,
|
||
|
|
"rewards/rejected": -4.165925025939941,
|
||
|
|
"step": 592
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.370944992947814,
|
||
|
|
"grad_norm": 13.082922810935031,
|
||
|
|
"learning_rate": 1.3591885467281877e-07,
|
||
|
|
"logits/chosen": 0.4695725440979004,
|
||
|
|
"logits/rejected": 0.4965362548828125,
|
||
|
|
"logps/chosen": -39.13195037841797,
|
||
|
|
"logps/rejected": -58.23176574707031,
|
||
|
|
"loss": 0.1861,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.30562350153923035,
|
||
|
|
"rewards/margins": 3.781522035598755,
|
||
|
|
"rewards/rejected": -4.0871453285217285,
|
||
|
|
"step": 594
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3755609693550455,
|
||
|
|
"grad_norm": 34.97692684836387,
|
||
|
|
"learning_rate": 1.3412834151347896e-07,
|
||
|
|
"logits/chosen": 0.5469548106193542,
|
||
|
|
"logits/rejected": 0.5717971324920654,
|
||
|
|
"logps/chosen": -44.02994155883789,
|
||
|
|
"logps/rejected": -57.28227996826172,
|
||
|
|
"loss": 0.2084,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.3421705365180969,
|
||
|
|
"rewards/margins": 3.692906379699707,
|
||
|
|
"rewards/rejected": -4.035076141357422,
|
||
|
|
"step": 596
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3801769457622772,
|
||
|
|
"grad_norm": 14.254996050777464,
|
||
|
|
"learning_rate": 1.323453664158769e-07,
|
||
|
|
"logits/chosen": 0.5193799138069153,
|
||
|
|
"logits/rejected": 0.5635771155357361,
|
||
|
|
"logps/chosen": -40.06482696533203,
|
||
|
|
"logps/rejected": -67.0745620727539,
|
||
|
|
"loss": 0.2322,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.5795915126800537,
|
||
|
|
"rewards/margins": 3.6668989658355713,
|
||
|
|
"rewards/rejected": -4.246490001678467,
|
||
|
|
"step": 598
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3847929221695088,
|
||
|
|
"grad_norm": 18.46063830068681,
|
||
|
|
"learning_rate": 1.3057004537193422e-07,
|
||
|
|
"logits/chosen": 0.5273723602294922,
|
||
|
|
"logits/rejected": 0.5402401685714722,
|
||
|
|
"logps/chosen": -45.491241455078125,
|
||
|
|
"logps/rejected": -53.827972412109375,
|
||
|
|
"loss": 0.185,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.32591530680656433,
|
||
|
|
"rewards/margins": 3.758335590362549,
|
||
|
|
"rewards/rejected": -4.084251403808594,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3847929221695088,
|
||
|
|
"eval_logits/chosen": 0.4299531877040863,
|
||
|
|
"eval_logits/rejected": 0.45649805665016174,
|
||
|
|
"eval_logps/chosen": -42.37248992919922,
|
||
|
|
"eval_logps/rejected": -55.565975189208984,
|
||
|
|
"eval_loss": 0.23996217548847198,
|
||
|
|
"eval_rewards/accuracies": 0.8300691246986389,
|
||
|
|
"eval_rewards/chosen": -0.31360533833503723,
|
||
|
|
"eval_rewards/margins": 3.450216054916382,
|
||
|
|
"eval_rewards/rejected": -3.7638211250305176,
|
||
|
|
"eval_runtime": 220.4449,
|
||
|
|
"eval_samples_per_second": 7.866,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3894088985767405,
|
||
|
|
"grad_norm": 24.193490725343704,
|
||
|
|
"learning_rate": 1.2880249387563662e-07,
|
||
|
|
"logits/chosen": 0.5480252504348755,
|
||
|
|
"logits/rejected": 0.5805102586746216,
|
||
|
|
"logps/chosen": -43.4918098449707,
|
||
|
|
"logps/rejected": -62.1549072265625,
|
||
|
|
"loss": 0.1713,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.4662397801876068,
|
||
|
|
"rewards/margins": 3.974961280822754,
|
||
|
|
"rewards/rejected": -4.441201210021973,
|
||
|
|
"step": 602
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3940248749839723,
|
||
|
|
"grad_norm": 8.975682909766576,
|
||
|
|
"learning_rate": 1.2704282691551938e-07,
|
||
|
|
"logits/chosen": 0.45732539892196655,
|
||
|
|
"logits/rejected": 0.5041163563728333,
|
||
|
|
"logps/chosen": -40.32965850830078,
|
||
|
|
"logps/rejected": -67.52854919433594,
|
||
|
|
"loss": 0.1754,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.014653444290161133,
|
||
|
|
"rewards/margins": 4.295289993286133,
|
||
|
|
"rewards/rejected": -4.280636787414551,
|
||
|
|
"step": 604
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.398640851391204,
|
||
|
|
"grad_norm": 27.018968489026342,
|
||
|
|
"learning_rate": 1.2529115896718714e-07,
|
||
|
|
"logits/chosen": 0.5242836475372314,
|
||
|
|
"logits/rejected": 0.5399221777915955,
|
||
|
|
"logps/chosen": -45.72035217285156,
|
||
|
|
"logps/rejected": -52.612548828125,
|
||
|
|
"loss": 0.2076,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.37530067563056946,
|
||
|
|
"rewards/margins": 3.2071659564971924,
|
||
|
|
"rewards/rejected": -3.5824666023254395,
|
||
|
|
"step": 606
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4032568277984356,
|
||
|
|
"grad_norm": 13.414881670063712,
|
||
|
|
"learning_rate": 1.2354760398586708e-07,
|
||
|
|
"logits/chosen": 0.5383539199829102,
|
||
|
|
"logits/rejected": 0.5773718953132629,
|
||
|
|
"logps/chosen": -48.75130081176758,
|
||
|
|
"logps/rejected": -72.36872863769531,
|
||
|
|
"loss": 0.1511,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.44930893182754517,
|
||
|
|
"rewards/margins": 4.512818336486816,
|
||
|
|
"rewards/rejected": -4.962126731872559,
|
||
|
|
"step": 608
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4078728042056674,
|
||
|
|
"grad_norm": 7.330900567316457,
|
||
|
|
"learning_rate": 1.2181227539899468e-07,
|
||
|
|
"logits/chosen": 0.5381309986114502,
|
||
|
|
"logits/rejected": 0.5586973428726196,
|
||
|
|
"logps/chosen": -45.09908676147461,
|
||
|
|
"logps/rejected": -58.20050811767578,
|
||
|
|
"loss": 0.1744,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.2882673442363739,
|
||
|
|
"rewards/margins": 3.7085728645324707,
|
||
|
|
"rewards/rejected": -3.996840238571167,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4078728042056674,
|
||
|
|
"eval_logits/chosen": 0.4304519295692444,
|
||
|
|
"eval_logits/rejected": 0.45695292949676514,
|
||
|
|
"eval_logps/chosen": -42.44300842285156,
|
||
|
|
"eval_logps/rejected": -55.6538200378418,
|
||
|
|
"eval_loss": 0.238841712474823,
|
||
|
|
"eval_rewards/accuracies": 0.8352534770965576,
|
||
|
|
"eval_rewards/chosen": -0.34886524081230164,
|
||
|
|
"eval_rewards/margins": 3.4588773250579834,
|
||
|
|
"eval_rewards/rejected": -3.8077423572540283,
|
||
|
|
"eval_runtime": 220.5308,
|
||
|
|
"eval_samples_per_second": 7.863,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.412488780612899,
|
||
|
|
"grad_norm": 10.681757170937171,
|
||
|
|
"learning_rate": 1.2008528609883557e-07,
|
||
|
|
"logits/chosen": 0.5007774233818054,
|
||
|
|
"logits/rejected": 0.5296944379806519,
|
||
|
|
"logps/chosen": -47.22381591796875,
|
||
|
|
"logps/rejected": -64.06365966796875,
|
||
|
|
"loss": 0.1531,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.030609939247369766,
|
||
|
|
"rewards/margins": 4.320724010467529,
|
||
|
|
"rewards/rejected": -4.351334571838379,
|
||
|
|
"step": 612
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4171047570201307,
|
||
|
|
"grad_norm": 10.655182313924602,
|
||
|
|
"learning_rate": 1.1836674843514042e-07,
|
||
|
|
"logits/chosen": 0.5347999930381775,
|
||
|
|
"logits/rejected": 0.564474880695343,
|
||
|
|
"logps/chosen": -37.77484893798828,
|
||
|
|
"logps/rejected": -54.86954879760742,
|
||
|
|
"loss": 0.175,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.38236376643180847,
|
||
|
|
"rewards/margins": 3.763371706008911,
|
||
|
|
"rewards/rejected": -4.145735263824463,
|
||
|
|
"step": 614
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4217207334273625,
|
||
|
|
"grad_norm": 4.808937007878847,
|
||
|
|
"learning_rate": 1.1665677420783671e-07,
|
||
|
|
"logits/chosen": 0.5504859089851379,
|
||
|
|
"logits/rejected": 0.5750877261161804,
|
||
|
|
"logps/chosen": -43.14183807373047,
|
||
|
|
"logps/rejected": -53.28805160522461,
|
||
|
|
"loss": 0.1417,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.12460337579250336,
|
||
|
|
"rewards/margins": 3.7694272994995117,
|
||
|
|
"rewards/rejected": -3.894031047821045,
|
||
|
|
"step": 616
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4263367098345943,
|
||
|
|
"grad_norm": 25.84566759360446,
|
||
|
|
"learning_rate": 1.149554746597553e-07,
|
||
|
|
"logits/chosen": 0.5723487734794617,
|
||
|
|
"logits/rejected": 0.6003535389900208,
|
||
|
|
"logps/chosen": -45.33318328857422,
|
||
|
|
"logps/rejected": -59.90052795410156,
|
||
|
|
"loss": 0.262,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.3526383936405182,
|
||
|
|
"rewards/margins": 3.843003988265991,
|
||
|
|
"rewards/rejected": -4.195642471313477,
|
||
|
|
"step": 618
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4309526862418258,
|
||
|
|
"grad_norm": 16.545628594299828,
|
||
|
|
"learning_rate": 1.1326296046939333e-07,
|
||
|
|
"logits/chosen": 0.5338951945304871,
|
||
|
|
"logits/rejected": 0.5544497966766357,
|
||
|
|
"logps/chosen": -39.78907775878906,
|
||
|
|
"logps/rejected": -49.23013687133789,
|
||
|
|
"loss": 0.2511,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.12468406558036804,
|
||
|
|
"rewards/margins": 3.2776834964752197,
|
||
|
|
"rewards/rejected": -3.402367353439331,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4309526862418258,
|
||
|
|
"eval_logits/chosen": 0.43395209312438965,
|
||
|
|
"eval_logits/rejected": 0.46030664443969727,
|
||
|
|
"eval_logps/chosen": -42.41356658935547,
|
||
|
|
"eval_logps/rejected": -55.699623107910156,
|
||
|
|
"eval_loss": 0.23819313943386078,
|
||
|
|
"eval_rewards/accuracies": 0.8317972421646118,
|
||
|
|
"eval_rewards/chosen": -0.3341463804244995,
|
||
|
|
"eval_rewards/margins": 3.4965004920959473,
|
||
|
|
"eval_rewards/rejected": -3.8306467533111572,
|
||
|
|
"eval_runtime": 220.4261,
|
||
|
|
"eval_samples_per_second": 7.867,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4355686626490576,
|
||
|
|
"grad_norm": 18.451086465748666,
|
||
|
|
"learning_rate": 1.1157934174371413e-07,
|
||
|
|
"logits/chosen": 0.497620165348053,
|
||
|
|
"logits/rejected": 0.5271977782249451,
|
||
|
|
"logps/chosen": -44.88563919067383,
|
||
|
|
"logps/rejected": -63.52084732055664,
|
||
|
|
"loss": 0.1973,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.4545660614967346,
|
||
|
|
"rewards/margins": 4.014831066131592,
|
||
|
|
"rewards/rejected": -4.469396591186523,
|
||
|
|
"step": 622
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4401846390562894,
|
||
|
|
"grad_norm": 15.41826391561629,
|
||
|
|
"learning_rate": 1.0990472801098419e-07,
|
||
|
|
"logits/chosen": 0.49964290857315063,
|
||
|
|
"logits/rejected": 0.5341427326202393,
|
||
|
|
"logps/chosen": -39.38306427001953,
|
||
|
|
"logps/rejected": -59.41951370239258,
|
||
|
|
"loss": 0.1465,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.07668253034353256,
|
||
|
|
"rewards/margins": 4.010004043579102,
|
||
|
|
"rewards/rejected": -4.086687088012695,
|
||
|
|
"step": 624
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.444800615463521,
|
||
|
|
"grad_norm": 13.657128245878823,
|
||
|
|
"learning_rate": 1.0823922821364795e-07,
|
||
|
|
"logits/chosen": 0.5488825440406799,
|
||
|
|
"logits/rejected": 0.5648425221443176,
|
||
|
|
"logps/chosen": -49.72515869140625,
|
||
|
|
"logps/rejected": -57.29216766357422,
|
||
|
|
"loss": 0.1844,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.15428660809993744,
|
||
|
|
"rewards/margins": 3.7048492431640625,
|
||
|
|
"rewards/rejected": -3.859135627746582,
|
||
|
|
"step": 626
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4494165918707527,
|
||
|
|
"grad_norm": 17.171702939592354,
|
||
|
|
"learning_rate": 1.0658295070124026e-07,
|
||
|
|
"logits/chosen": 0.5274313688278198,
|
||
|
|
"logits/rejected": 0.540188729763031,
|
||
|
|
"logps/chosen": -47.955406188964844,
|
||
|
|
"logps/rejected": -54.03617477416992,
|
||
|
|
"loss": 0.2187,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.16990727186203003,
|
||
|
|
"rewards/margins": 3.60162091255188,
|
||
|
|
"rewards/rejected": -3.7715280055999756,
|
||
|
|
"step": 628
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4540325682779844,
|
||
|
|
"grad_norm": 25.795693399142227,
|
||
|
|
"learning_rate": 1.0493600322333762e-07,
|
||
|
|
"logits/chosen": 0.5215524435043335,
|
||
|
|
"logits/rejected": 0.5590708255767822,
|
||
|
|
"logps/chosen": -44.3021354675293,
|
||
|
|
"logps/rejected": -73.55774688720703,
|
||
|
|
"loss": 0.141,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.3967975080013275,
|
||
|
|
"rewards/margins": 4.7301740646362305,
|
||
|
|
"rewards/rejected": -5.12697172164917,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4540325682779844,
|
||
|
|
"eval_logits/chosen": 0.43174034357070923,
|
||
|
|
"eval_logits/rejected": 0.4582732319831848,
|
||
|
|
"eval_logps/chosen": -42.194610595703125,
|
||
|
|
"eval_logps/rejected": -55.55934524536133,
|
||
|
|
"eval_loss": 0.23693177103996277,
|
||
|
|
"eval_rewards/accuracies": 0.8317972421646118,
|
||
|
|
"eval_rewards/chosen": -0.22466643154621124,
|
||
|
|
"eval_rewards/margins": 3.535839080810547,
|
||
|
|
"eval_rewards/rejected": -3.7605059146881104,
|
||
|
|
"eval_runtime": 220.3801,
|
||
|
|
"eval_samples_per_second": 7.868,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.458648544685216,
|
||
|
|
"grad_norm": 14.475820972948407,
|
||
|
|
"learning_rate": 1.0329849292254883e-07,
|
||
|
|
"logits/chosen": 0.596792995929718,
|
||
|
|
"logits/rejected": 0.624647855758667,
|
||
|
|
"logps/chosen": -45.63186264038086,
|
||
|
|
"logps/rejected": -62.25794982910156,
|
||
|
|
"loss": 0.1936,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.2872418463230133,
|
||
|
|
"rewards/margins": 3.9080302715301514,
|
||
|
|
"rewards/rejected": -4.195271968841553,
|
||
|
|
"step": 632
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4632645210924478,
|
||
|
|
"grad_norm": 26.862980766739724,
|
||
|
|
"learning_rate": 1.0167052632754458e-07,
|
||
|
|
"logits/chosen": 0.5725838541984558,
|
||
|
|
"logits/rejected": 0.5932745337486267,
|
||
|
|
"logps/chosen": -41.20800018310547,
|
||
|
|
"logps/rejected": -51.21732711791992,
|
||
|
|
"loss": 0.227,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.39695149660110474,
|
||
|
|
"rewards/margins": 2.928715229034424,
|
||
|
|
"rewards/rejected": -3.325666666030884,
|
||
|
|
"step": 634
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4678804974996795,
|
||
|
|
"grad_norm": 13.962052681918495,
|
||
|
|
"learning_rate": 1.0005220934612713e-07,
|
||
|
|
"logits/chosen": 0.6229636669158936,
|
||
|
|
"logits/rejected": 0.6402004361152649,
|
||
|
|
"logps/chosen": -46.95052719116211,
|
||
|
|
"logps/rejected": -53.86199951171875,
|
||
|
|
"loss": 0.1824,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.3401051461696625,
|
||
|
|
"rewards/margins": 3.6175549030303955,
|
||
|
|
"rewards/rejected": -3.95766019821167,
|
||
|
|
"step": 636
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.472496473906911,
|
||
|
|
"grad_norm": 9.092245687630806,
|
||
|
|
"learning_rate": 9.844364725834056e-08,
|
||
|
|
"logits/chosen": 0.48213544487953186,
|
||
|
|
"logits/rejected": 0.5316063761711121,
|
||
|
|
"logps/chosen": -45.23646545410156,
|
||
|
|
"logps/rejected": -75.49991607666016,
|
||
|
|
"loss": 0.0997,
|
||
|
|
"rewards/accuracies": 0.9583333134651184,
|
||
|
|
"rewards/chosen": -0.1606331765651703,
|
||
|
|
"rewards/margins": 5.202739238739014,
|
||
|
|
"rewards/rejected": -5.363372802734375,
|
||
|
|
"step": 638
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4771124503141428,
|
||
|
|
"grad_norm": 18.96340702396886,
|
||
|
|
"learning_rate": 9.68449447096217e-08,
|
||
|
|
"logits/chosen": 0.4373500943183899,
|
||
|
|
"logits/rejected": 0.4579113721847534,
|
||
|
|
"logps/chosen": -39.44499588012695,
|
||
|
|
"logps/rejected": -51.54633712768555,
|
||
|
|
"loss": 0.3299,
|
||
|
|
"rewards/accuracies": 0.7916666865348816,
|
||
|
|
"rewards/chosen": -0.2867163419723511,
|
||
|
|
"rewards/margins": 3.076793670654297,
|
||
|
|
"rewards/rejected": -3.3635098934173584,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4771124503141428,
|
||
|
|
"eval_logits/chosen": 0.4346330463886261,
|
||
|
|
"eval_logits/rejected": 0.461146742105484,
|
||
|
|
"eval_logps/chosen": -42.071449279785156,
|
||
|
|
"eval_logps/rejected": -55.46683883666992,
|
||
|
|
"eval_loss": 0.23784740269184113,
|
||
|
|
"eval_rewards/accuracies": 0.835829496383667,
|
||
|
|
"eval_rewards/chosen": -0.16308562457561493,
|
||
|
|
"eval_rewards/margins": 3.551164388656616,
|
||
|
|
"eval_rewards/rejected": -3.714250087738037,
|
||
|
|
"eval_runtime": 220.3881,
|
||
|
|
"eval_samples_per_second": 7.868,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4817284267213746,
|
||
|
|
"grad_norm": 22.570461867090884,
|
||
|
|
"learning_rate": 9.525620570399259e-08,
|
||
|
|
"logits/chosen": 0.5038811564445496,
|
||
|
|
"logits/rejected": 0.5432533025741577,
|
||
|
|
"logps/chosen": -44.41080856323242,
|
||
|
|
"logps/rejected": -65.23593139648438,
|
||
|
|
"loss": 0.1275,
|
||
|
|
"rewards/accuracies": 0.9444444179534912,
|
||
|
|
"rewards/chosen": -0.2485545426607132,
|
||
|
|
"rewards/margins": 4.013004779815674,
|
||
|
|
"rewards/rejected": -4.261559009552002,
|
||
|
|
"step": 642
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4863444031286062,
|
||
|
|
"grad_norm": 11.127499049370783,
|
||
|
|
"learning_rate": 9.36775335972943e-08,
|
||
|
|
"logits/chosen": 0.4518318772315979,
|
||
|
|
"logits/rejected": 0.531367838382721,
|
||
|
|
"logps/chosen": -39.415767669677734,
|
||
|
|
"logps/rejected": -98.71846771240234,
|
||
|
|
"loss": 0.1566,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.000497970322612673,
|
||
|
|
"rewards/margins": 6.575231075286865,
|
||
|
|
"rewards/rejected": -6.575727939605713,
|
||
|
|
"step": 644
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.490960379535838,
|
||
|
|
"grad_norm": 24.53509661266678,
|
||
|
|
"learning_rate": 9.210903109046284e-08,
|
||
|
|
"logits/chosen": 0.46663856506347656,
|
||
|
|
"logits/rejected": 0.5147727727890015,
|
||
|
|
"logps/chosen": -43.30581283569336,
|
||
|
|
"logps/rejected": -63.16206741333008,
|
||
|
|
"loss": 0.1684,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.5338683128356934,
|
||
|
|
"rewards/margins": 4.3571882247924805,
|
||
|
|
"rewards/rejected": -4.89105749130249,
|
||
|
|
"step": 646
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4955763559430697,
|
||
|
|
"grad_norm": 11.303027411423997,
|
||
|
|
"learning_rate": 9.05508002228485e-08,
|
||
|
|
"logits/chosen": 0.529050350189209,
|
||
|
|
"logits/rejected": 0.5628350377082825,
|
||
|
|
"logps/chosen": -38.363826751708984,
|
||
|
|
"logps/rejected": -53.06625747680664,
|
||
|
|
"loss": 0.2071,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.031818799674510956,
|
||
|
|
"rewards/margins": 3.961611032485962,
|
||
|
|
"rewards/rejected": -3.929792642593384,
|
||
|
|
"step": 648
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5001923323503012,
|
||
|
|
"grad_norm": 10.500286558923209,
|
||
|
|
"learning_rate": 8.900294236557707e-08,
|
||
|
|
"logits/chosen": 0.49337685108184814,
|
||
|
|
"logits/rejected": 0.5243138074874878,
|
||
|
|
"logps/chosen": -37.17765808105469,
|
||
|
|
"logps/rejected": -49.10523986816406,
|
||
|
|
"loss": 0.2143,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": 0.008912450633943081,
|
||
|
|
"rewards/margins": 3.240175485610962,
|
||
|
|
"rewards/rejected": -3.2312631607055664,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5001923323503012,
|
||
|
|
"eval_logits/chosen": 0.4313080310821533,
|
||
|
|
"eval_logits/rejected": 0.45790737867355347,
|
||
|
|
"eval_logps/chosen": -42.17680740356445,
|
||
|
|
"eval_logps/rejected": -55.66178894042969,
|
||
|
|
"eval_loss": 0.2398524433374405,
|
||
|
|
"eval_rewards/accuracies": 0.8306451439857483,
|
||
|
|
"eval_rewards/chosen": -0.21576282382011414,
|
||
|
|
"eval_rewards/margins": 3.59596586227417,
|
||
|
|
"eval_rewards/rejected": -3.8117284774780273,
|
||
|
|
"eval_runtime": 220.4293,
|
||
|
|
"eval_samples_per_second": 7.866,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.504808308757533,
|
||
|
|
"grad_norm": 21.390880404408534,
|
||
|
|
"learning_rate": 8.746555821495561e-08,
|
||
|
|
"logits/chosen": 0.4801899492740631,
|
||
|
|
"logits/rejected": 0.5136987566947937,
|
||
|
|
"logps/chosen": -43.907596588134766,
|
||
|
|
"logps/rejected": -62.06863021850586,
|
||
|
|
"loss": 0.1972,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.21802374720573425,
|
||
|
|
"rewards/margins": 4.019637584686279,
|
||
|
|
"rewards/rejected": -4.237661361694336,
|
||
|
|
"step": 652
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5094242851647648,
|
||
|
|
"grad_norm": 17.814740010117944,
|
||
|
|
"learning_rate": 8.593874778592122e-08,
|
||
|
|
"logits/chosen": 0.4772498309612274,
|
||
|
|
"logits/rejected": 0.5082363486289978,
|
||
|
|
"logps/chosen": -36.85258483886719,
|
||
|
|
"logps/rejected": -49.34876251220703,
|
||
|
|
"loss": 0.1537,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.038329627364873886,
|
||
|
|
"rewards/margins": 3.5393142700195312,
|
||
|
|
"rewards/rejected": -3.577643394470215,
|
||
|
|
"step": 654
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5140402615719966,
|
||
|
|
"grad_norm": 24.684686325904988,
|
||
|
|
"learning_rate": 8.442261040553472e-08,
|
||
|
|
"logits/chosen": 0.5512763857841492,
|
||
|
|
"logits/rejected": 0.5618037581443787,
|
||
|
|
"logps/chosen": -44.694515228271484,
|
||
|
|
"logps/rejected": -49.48525619506836,
|
||
|
|
"loss": 0.1683,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": 0.0919620469212532,
|
||
|
|
"rewards/margins": 3.498401403427124,
|
||
|
|
"rewards/rejected": -3.406439781188965,
|
||
|
|
"step": 656
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.518656237979228,
|
||
|
|
"grad_norm": 21.50701378180569,
|
||
|
|
"learning_rate": 8.291724470651903e-08,
|
||
|
|
"logits/chosen": 0.49069249629974365,
|
||
|
|
"logits/rejected": 0.5210825800895691,
|
||
|
|
"logps/chosen": -44.639766693115234,
|
||
|
|
"logps/rejected": -57.28916549682617,
|
||
|
|
"loss": 0.2335,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.4066303074359894,
|
||
|
|
"rewards/margins": 3.4069387912750244,
|
||
|
|
"rewards/rejected": -3.813568592071533,
|
||
|
|
"step": 658
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5232722143864597,
|
||
|
|
"grad_norm": 11.082339838552715,
|
||
|
|
"learning_rate": 8.14227486208423e-08,
|
||
|
|
"logits/chosen": 0.4665941596031189,
|
||
|
|
"logits/rejected": 0.4930134415626526,
|
||
|
|
"logps/chosen": -37.94073486328125,
|
||
|
|
"logps/rejected": -53.0433464050293,
|
||
|
|
"loss": 0.1797,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": 0.09727773815393448,
|
||
|
|
"rewards/margins": 3.91404128074646,
|
||
|
|
"rewards/rejected": -3.8167638778686523,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5232722143864597,
|
||
|
|
"eval_logits/chosen": 0.43500614166259766,
|
||
|
|
"eval_logits/rejected": 0.4616233706474304,
|
||
|
|
"eval_logps/chosen": -42.075767517089844,
|
||
|
|
"eval_logps/rejected": -55.58706283569336,
|
||
|
|
"eval_loss": 0.2391819953918457,
|
||
|
|
"eval_rewards/accuracies": 0.8306451439857483,
|
||
|
|
"eval_rewards/chosen": -0.1652439683675766,
|
||
|
|
"eval_rewards/margins": 3.609118938446045,
|
||
|
|
"eval_rewards/rejected": -3.774362802505493,
|
||
|
|
"eval_runtime": 220.4966,
|
||
|
|
"eval_samples_per_second": 7.864,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5278881907936914,
|
||
|
|
"grad_norm": 17.884909353386927,
|
||
|
|
"learning_rate": 7.993921937334716e-08,
|
||
|
|
"logits/chosen": 0.5584304332733154,
|
||
|
|
"logits/rejected": 0.5700749754905701,
|
||
|
|
"logps/chosen": -41.323944091796875,
|
||
|
|
"logps/rejected": -49.892147064208984,
|
||
|
|
"loss": 0.2096,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.27705806493759155,
|
||
|
|
"rewards/margins": 3.536667823791504,
|
||
|
|
"rewards/rejected": -3.813725709915161,
|
||
|
|
"step": 662
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5325041672009232,
|
||
|
|
"grad_norm": 6.982953174746173,
|
||
|
|
"learning_rate": 7.846675347542578e-08,
|
||
|
|
"logits/chosen": 0.5807335376739502,
|
||
|
|
"logits/rejected": 0.6132792234420776,
|
||
|
|
"logps/chosen": -37.81986999511719,
|
||
|
|
"logps/rejected": -49.71797180175781,
|
||
|
|
"loss": 0.1272,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": 0.3861154019832611,
|
||
|
|
"rewards/margins": 4.170031547546387,
|
||
|
|
"rewards/rejected": -3.783916473388672,
|
||
|
|
"step": 664
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.537120143608155,
|
||
|
|
"grad_norm": 18.18022469520284,
|
||
|
|
"learning_rate": 7.700544671874079e-08,
|
||
|
|
"logits/chosen": 0.6006969213485718,
|
||
|
|
"logits/rejected": 0.6162829995155334,
|
||
|
|
"logps/chosen": -47.33814239501953,
|
||
|
|
"logps/rejected": -52.70623016357422,
|
||
|
|
"loss": 0.1962,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.2818297743797302,
|
||
|
|
"rewards/margins": 3.495248317718506,
|
||
|
|
"rewards/rejected": -3.7770779132843018,
|
||
|
|
"step": 666
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5417361200153867,
|
||
|
|
"grad_norm": 17.752568042598934,
|
||
|
|
"learning_rate": 7.555539416899437e-08,
|
||
|
|
"logits/chosen": 0.5043608546257019,
|
||
|
|
"logits/rejected": 0.535383939743042,
|
||
|
|
"logps/chosen": -37.40916442871094,
|
||
|
|
"logps/rejected": -52.42148971557617,
|
||
|
|
"loss": 0.2323,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.4006814658641815,
|
||
|
|
"rewards/margins": 3.385708808898926,
|
||
|
|
"rewards/rejected": -3.7863900661468506,
|
||
|
|
"step": 668
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5463520964226183,
|
||
|
|
"grad_norm": 14.165329854797266,
|
||
|
|
"learning_rate": 7.41166901597429e-08,
|
||
|
|
"logits/chosen": 0.5081818699836731,
|
||
|
|
"logits/rejected": 0.5341579914093018,
|
||
|
|
"logps/chosen": -42.154205322265625,
|
||
|
|
"logps/rejected": -55.97992706298828,
|
||
|
|
"loss": 0.1774,
|
||
|
|
"rewards/accuracies": 0.9027777910232544,
|
||
|
|
"rewards/chosen": -0.05981425940990448,
|
||
|
|
"rewards/margins": 3.988154172897339,
|
||
|
|
"rewards/rejected": -4.047967910766602,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5463520964226183,
|
||
|
|
"eval_logits/chosen": 0.4372006952762604,
|
||
|
|
"eval_logits/rejected": 0.46362602710723877,
|
||
|
|
"eval_logps/chosen": -42.13774490356445,
|
||
|
|
"eval_logps/rejected": -55.63636779785156,
|
||
|
|
"eval_loss": 0.23786574602127075,
|
||
|
|
"eval_rewards/accuracies": 0.8329492807388306,
|
||
|
|
"eval_rewards/chosen": -0.19623348116874695,
|
||
|
|
"eval_rewards/margins": 3.602783203125,
|
||
|
|
"eval_rewards/rejected": -3.7990164756774902,
|
||
|
|
"eval_runtime": 220.5205,
|
||
|
|
"eval_samples_per_second": 7.863,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5509680728298498,
|
||
|
|
"grad_norm": 22.84931762442886,
|
||
|
|
"learning_rate": 7.268942828626046e-08,
|
||
|
|
"logits/chosen": 0.5015777349472046,
|
||
|
|
"logits/rejected": 0.5260412096977234,
|
||
|
|
"logps/chosen": -39.39936828613281,
|
||
|
|
"logps/rejected": -50.80826950073242,
|
||
|
|
"loss": 0.2259,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.02117648348212242,
|
||
|
|
"rewards/margins": 3.6156790256500244,
|
||
|
|
"rewards/rejected": -3.6368556022644043,
|
||
|
|
"step": 672
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5555840492370816,
|
||
|
|
"grad_norm": 10.729660784502734,
|
||
|
|
"learning_rate": 7.127370139945018e-08,
|
||
|
|
"logits/chosen": 0.5064399242401123,
|
||
|
|
"logits/rejected": 0.542765736579895,
|
||
|
|
"logps/chosen": -41.118350982666016,
|
||
|
|
"logps/rejected": -57.55162048339844,
|
||
|
|
"loss": 0.1581,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.18698811531066895,
|
||
|
|
"rewards/margins": 4.028824806213379,
|
||
|
|
"rewards/rejected": -4.215813159942627,
|
||
|
|
"step": 674
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5602000256443134,
|
||
|
|
"grad_norm": 12.758336439580667,
|
||
|
|
"learning_rate": 6.986960159980326e-08,
|
||
|
|
"logits/chosen": 0.5471921563148499,
|
||
|
|
"logits/rejected": 0.5656020045280457,
|
||
|
|
"logps/chosen": -44.28984069824219,
|
||
|
|
"logps/rejected": -53.67868423461914,
|
||
|
|
"loss": 0.1621,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.007483018562197685,
|
||
|
|
"rewards/margins": 3.514232873916626,
|
||
|
|
"rewards/rejected": -3.5217158794403076,
|
||
|
|
"step": 676
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5648160020515451,
|
||
|
|
"grad_norm": 25.743372698631337,
|
||
|
|
"learning_rate": 6.847722023140776e-08,
|
||
|
|
"logits/chosen": 0.5099420547485352,
|
||
|
|
"logits/rejected": 0.5306479930877686,
|
||
|
|
"logps/chosen": -38.24551773071289,
|
||
|
|
"logps/rejected": -46.37004470825195,
|
||
|
|
"loss": 0.2453,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": -0.13890628516674042,
|
||
|
|
"rewards/margins": 3.242166757583618,
|
||
|
|
"rewards/rejected": -3.381072998046875,
|
||
|
|
"step": 678
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.569431978458777,
|
||
|
|
"grad_norm": 29.001544411683714,
|
||
|
|
"learning_rate": 6.709664787600616e-08,
|
||
|
|
"logits/chosen": 0.5341071486473083,
|
||
|
|
"logits/rejected": 0.549387514591217,
|
||
|
|
"logps/chosen": -38.39107131958008,
|
||
|
|
"logps/rejected": -45.22284698486328,
|
||
|
|
"loss": 0.2519,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": -0.32282909750938416,
|
||
|
|
"rewards/margins": 2.876624822616577,
|
||
|
|
"rewards/rejected": -3.1994540691375732,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.569431978458777,
|
||
|
|
"eval_logits/chosen": 0.4367799460887909,
|
||
|
|
"eval_logits/rejected": 0.46335569024086,
|
||
|
|
"eval_logps/chosen": -42.14803695678711,
|
||
|
|
"eval_logps/rejected": -55.68684005737305,
|
||
|
|
"eval_loss": 0.23701736330986023,
|
||
|
|
"eval_rewards/accuracies": 0.8335253596305847,
|
||
|
|
"eval_rewards/chosen": -0.20137952268123627,
|
||
|
|
"eval_rewards/margins": 3.622871160507202,
|
||
|
|
"eval_rewards/rejected": -3.8242506980895996,
|
||
|
|
"eval_runtime": 220.405,
|
||
|
|
"eval_samples_per_second": 7.867,
|
||
|
|
"eval_steps_per_second": 1.969,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5740479548660085,
|
||
|
|
"grad_norm": 26.57226192590101,
|
||
|
|
"learning_rate": 6.572797434710219e-08,
|
||
|
|
"logits/chosen": 0.47764989733695984,
|
||
|
|
"logits/rejected": 0.5231152772903442,
|
||
|
|
"logps/chosen": -39.2479362487793,
|
||
|
|
"logps/rejected": -67.22251892089844,
|
||
|
|
"loss": 0.1985,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.03589929640293121,
|
||
|
|
"rewards/margins": 4.406409740447998,
|
||
|
|
"rewards/rejected": -4.370510578155518,
|
||
|
|
"step": 682
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.57866393127324,
|
||
|
|
"grad_norm": 7.8158043752344115,
|
||
|
|
"learning_rate": 6.437128868411856e-08,
|
||
|
|
"logits/chosen": 0.5327097177505493,
|
||
|
|
"logits/rejected": 0.5473262071609497,
|
||
|
|
"logps/chosen": -38.83921813964844,
|
||
|
|
"logps/rejected": -47.30848693847656,
|
||
|
|
"loss": 0.212,
|
||
|
|
"rewards/accuracies": 0.8333333134651184,
|
||
|
|
"rewards/chosen": 0.002615167060866952,
|
||
|
|
"rewards/margins": 3.3942179679870605,
|
||
|
|
"rewards/rejected": -3.3916027545928955,
|
||
|
|
"step": 684
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5832799076804718,
|
||
|
|
"grad_norm": 11.10012939486401,
|
||
|
|
"learning_rate": 6.302667914660384e-08,
|
||
|
|
"logits/chosen": 0.5219799280166626,
|
||
|
|
"logits/rejected": 0.55839604139328,
|
||
|
|
"logps/chosen": -37.46578598022461,
|
||
|
|
"logps/rejected": -54.46531295776367,
|
||
|
|
"loss": 0.2233,
|
||
|
|
"rewards/accuracies": 0.8472222089767456,
|
||
|
|
"rewards/chosen": 0.02391706593334675,
|
||
|
|
"rewards/margins": 3.7034101486206055,
|
||
|
|
"rewards/rejected": -3.679492950439453,
|
||
|
|
"step": 686
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5878958840877035,
|
||
|
|
"grad_norm": 19.67549763311113,
|
||
|
|
"learning_rate": 6.169423320849112e-08,
|
||
|
|
"logits/chosen": 0.5211795568466187,
|
||
|
|
"logits/rejected": 0.5298517346382141,
|
||
|
|
"logps/chosen": -45.8150520324707,
|
||
|
|
"logps/rejected": -47.33256149291992,
|
||
|
|
"loss": 0.2021,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.2716074287891388,
|
||
|
|
"rewards/margins": 3.559727191925049,
|
||
|
|
"rewards/rejected": -3.831334352493286,
|
||
|
|
"step": 688
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5925118604949353,
|
||
|
|
"grad_norm": 15.711220514951888,
|
||
|
|
"learning_rate": 6.037403755240748e-08,
|
||
|
|
"logits/chosen": 0.5544189810752869,
|
||
|
|
"logits/rejected": 0.5787670612335205,
|
||
|
|
"logps/chosen": -45.216304779052734,
|
||
|
|
"logps/rejected": -59.76258850097656,
|
||
|
|
"loss": 0.1572,
|
||
|
|
"rewards/accuracies": 0.9305555820465088,
|
||
|
|
"rewards/chosen": -0.14509858191013336,
|
||
|
|
"rewards/margins": 3.88366436958313,
|
||
|
|
"rewards/rejected": -4.0287628173828125,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5925118604949353,
|
||
|
|
"eval_logits/chosen": 0.43276646733283997,
|
||
|
|
"eval_logits/rejected": 0.45934849977493286,
|
||
|
|
"eval_logps/chosen": -42.20445251464844,
|
||
|
|
"eval_logps/rejected": -55.753753662109375,
|
||
|
|
"eval_loss": 0.23724210262298584,
|
||
|
|
"eval_rewards/accuracies": 0.8317972421646118,
|
||
|
|
"eval_rewards/chosen": -0.2295861542224884,
|
||
|
|
"eval_rewards/margins": 3.6281206607818604,
|
||
|
|
"eval_rewards/rejected": -3.8577067852020264,
|
||
|
|
"eval_runtime": 220.4833,
|
||
|
|
"eval_samples_per_second": 7.865,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.597127836902167,
|
||
|
|
"grad_norm": 14.487508826565733,
|
||
|
|
"learning_rate": 5.9066178064034326e-08,
|
||
|
|
"logits/chosen": 0.4430210590362549,
|
||
|
|
"logits/rejected": 0.4965353012084961,
|
||
|
|
"logps/chosen": -33.27760696411133,
|
||
|
|
"logps/rejected": -71.74127197265625,
|
||
|
|
"loss": 0.2328,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.2861379384994507,
|
||
|
|
"rewards/margins": 4.55012321472168,
|
||
|
|
"rewards/rejected": -4.836262226104736,
|
||
|
|
"step": 692
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6017438133093986,
|
||
|
|
"grad_norm": 23.580990452467088,
|
||
|
|
"learning_rate": 5.777073982652064e-08,
|
||
|
|
"logits/chosen": 0.5170236825942993,
|
||
|
|
"logits/rejected": 0.5521243214607239,
|
||
|
|
"logps/chosen": -35.71030044555664,
|
||
|
|
"logps/rejected": -52.74575424194336,
|
||
|
|
"loss": 0.2247,
|
||
|
|
"rewards/accuracies": 0.8611111044883728,
|
||
|
|
"rewards/chosen": -0.3935600519180298,
|
||
|
|
"rewards/margins": 3.574741840362549,
|
||
|
|
"rewards/rejected": -3.96830153465271,
|
||
|
|
"step": 694
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6063597897166302,
|
||
|
|
"grad_norm": 13.54068941517088,
|
||
|
|
"learning_rate": 5.6487807114947325e-08,
|
||
|
|
"logits/chosen": 0.551853358745575,
|
||
|
|
"logits/rejected": 0.5928479433059692,
|
||
|
|
"logps/chosen": -42.63957214355469,
|
||
|
|
"logps/rejected": -70.68295288085938,
|
||
|
|
"loss": 0.1803,
|
||
|
|
"rewards/accuracies": 0.8888888955116272,
|
||
|
|
"rewards/chosen": -0.2251981645822525,
|
||
|
|
"rewards/margins": 4.277625560760498,
|
||
|
|
"rewards/rejected": -4.502823352813721,
|
||
|
|
"step": 696
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.610975766123862,
|
||
|
|
"grad_norm": 27.742044897151906,
|
||
|
|
"learning_rate": 5.521746339084532e-08,
|
||
|
|
"logits/chosen": 0.5765677094459534,
|
||
|
|
"logits/rejected": 0.5921374559402466,
|
||
|
|
"logps/chosen": -47.175655364990234,
|
||
|
|
"logps/rejected": -58.09642028808594,
|
||
|
|
"loss": 0.2516,
|
||
|
|
"rewards/accuracies": 0.8194444179534912,
|
||
|
|
"rewards/chosen": -0.3188338875770569,
|
||
|
|
"rewards/margins": 3.57645583152771,
|
||
|
|
"rewards/rejected": -3.895289897918701,
|
||
|
|
"step": 698
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6155917425310937,
|
||
|
|
"grad_norm": 13.652878320465026,
|
||
|
|
"learning_rate": 5.39597912967652e-08,
|
||
|
|
"logits/chosen": 0.5359885692596436,
|
||
|
|
"logits/rejected": 0.575743556022644,
|
||
|
|
"logps/chosen": -38.843807220458984,
|
||
|
|
"logps/rejected": -61.49338150024414,
|
||
|
|
"loss": 0.1886,
|
||
|
|
"rewards/accuracies": 0.9166666865348816,
|
||
|
|
"rewards/chosen": -0.01514108944684267,
|
||
|
|
"rewards/margins": 4.108646392822266,
|
||
|
|
"rewards/rejected": -4.1237874031066895,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6155917425310937,
|
||
|
|
"eval_logits/chosen": 0.43191081285476685,
|
||
|
|
"eval_logits/rejected": 0.4585791528224945,
|
||
|
|
"eval_logps/chosen": -42.20844268798828,
|
||
|
|
"eval_logps/rejected": -55.773094177246094,
|
||
|
|
"eval_loss": 0.23592650890350342,
|
||
|
|
"eval_rewards/accuracies": 0.8364055156707764,
|
||
|
|
"eval_rewards/chosen": -0.23158276081085205,
|
||
|
|
"eval_rewards/margins": 3.635798692703247,
|
||
|
|
"eval_rewards/rejected": -3.8673815727233887,
|
||
|
|
"eval_runtime": 220.5019,
|
||
|
|
"eval_samples_per_second": 7.864,
|
||
|
|
"eval_steps_per_second": 1.968,
|
||
|
|
"step": 700
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 2,
|
||
|
|
"max_steps": 866,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 2,
|
||
|
|
"save_steps": 10,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|