3309 lines
112 KiB
JSON
3309 lines
112 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 100,
|
||
|
|
"global_step": 1984,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.005040957781978576,
|
||
|
|
"grad_norm": 25.75,
|
||
|
|
"learning_rate": 3e-08,
|
||
|
|
"logits/chosen": -0.07891461253166199,
|
||
|
|
"logits/rejected": 0.004119270481169224,
|
||
|
|
"logps/chosen": -62.022430419921875,
|
||
|
|
"logps/rejected": -65.60428619384766,
|
||
|
|
"loss": 0.6949,
|
||
|
|
"rewards/accuracies": 0.3687500059604645,
|
||
|
|
"rewards/chosen": 0.0029429681599140167,
|
||
|
|
"rewards/margins": -0.0032222606241703033,
|
||
|
|
"rewards/rejected": 0.006165228318423033,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.010081915563957152,
|
||
|
|
"grad_norm": 19.5,
|
||
|
|
"learning_rate": 6.333333333333333e-08,
|
||
|
|
"logits/chosen": -0.058114223182201385,
|
||
|
|
"logits/rejected": 0.02890823781490326,
|
||
|
|
"logps/chosen": -61.93231201171875,
|
||
|
|
"logps/rejected": -65.11712646484375,
|
||
|
|
"loss": 0.6907,
|
||
|
|
"rewards/accuracies": 0.53125,
|
||
|
|
"rewards/chosen": 0.008925376459956169,
|
||
|
|
"rewards/margins": 0.005283808801323175,
|
||
|
|
"rewards/rejected": 0.0036415669601410627,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.015122873345935728,
|
||
|
|
"grad_norm": 25.0,
|
||
|
|
"learning_rate": 9.666666666666666e-08,
|
||
|
|
"logits/chosen": -0.04496127367019653,
|
||
|
|
"logits/rejected": 0.017042722553014755,
|
||
|
|
"logps/chosen": -61.5279541015625,
|
||
|
|
"logps/rejected": -65.24421691894531,
|
||
|
|
"loss": 0.691,
|
||
|
|
"rewards/accuracies": 0.6000000238418579,
|
||
|
|
"rewards/chosen": 0.0035877160262316465,
|
||
|
|
"rewards/margins": 0.004559223540127277,
|
||
|
|
"rewards/rejected": -0.0009715079213492572,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.020163831127914304,
|
||
|
|
"grad_norm": 29.25,
|
||
|
|
"learning_rate": 1.3e-07,
|
||
|
|
"logits/chosen": -0.0775897353887558,
|
||
|
|
"logits/rejected": -0.00422413507476449,
|
||
|
|
"logps/chosen": -61.86296463012695,
|
||
|
|
"logps/rejected": -65.46966552734375,
|
||
|
|
"loss": 0.6852,
|
||
|
|
"rewards/accuracies": 0.668749988079071,
|
||
|
|
"rewards/chosen": 0.006382000632584095,
|
||
|
|
"rewards/margins": 0.016284234821796417,
|
||
|
|
"rewards/rejected": -0.009902234189212322,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.02520478890989288,
|
||
|
|
"grad_norm": 32.0,
|
||
|
|
"learning_rate": 1.6333333333333331e-07,
|
||
|
|
"logits/chosen": -0.10396875441074371,
|
||
|
|
"logits/rejected": -0.028726909309625626,
|
||
|
|
"logps/chosen": -61.53949737548828,
|
||
|
|
"logps/rejected": -65.4787826538086,
|
||
|
|
"loss": 0.6777,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 0.0072999573312699795,
|
||
|
|
"rewards/margins": 0.03146541863679886,
|
||
|
|
"rewards/rejected": -0.02416546270251274,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.030245746691871456,
|
||
|
|
"grad_norm": 28.125,
|
||
|
|
"learning_rate": 1.9666666666666665e-07,
|
||
|
|
"logits/chosen": -0.15684179961681366,
|
||
|
|
"logits/rejected": -0.08579285442829132,
|
||
|
|
"logps/chosen": -62.388755798339844,
|
||
|
|
"logps/rejected": -65.70417785644531,
|
||
|
|
"loss": 0.6749,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": -0.004609875846654177,
|
||
|
|
"rewards/margins": 0.03752168267965317,
|
||
|
|
"rewards/rejected": -0.04213155806064606,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03528670447385003,
|
||
|
|
"grad_norm": 32.75,
|
||
|
|
"learning_rate": 2.3e-07,
|
||
|
|
"logits/chosen": -0.14031846821308136,
|
||
|
|
"logits/rejected": -0.054399728775024414,
|
||
|
|
"logps/chosen": -62.0086669921875,
|
||
|
|
"logps/rejected": -66.37232971191406,
|
||
|
|
"loss": 0.6629,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": -0.01638166978955269,
|
||
|
|
"rewards/margins": 0.0622851625084877,
|
||
|
|
"rewards/rejected": -0.07866682857275009,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04032766225582861,
|
||
|
|
"grad_norm": 30.25,
|
||
|
|
"learning_rate": 2.633333333333333e-07,
|
||
|
|
"logits/chosen": -0.19501006603240967,
|
||
|
|
"logits/rejected": -0.1102059856057167,
|
||
|
|
"logps/chosen": -62.68883514404297,
|
||
|
|
"logps/rejected": -66.286865234375,
|
||
|
|
"loss": 0.6564,
|
||
|
|
"rewards/accuracies": 0.8062499761581421,
|
||
|
|
"rewards/chosen": -0.03311951085925102,
|
||
|
|
"rewards/margins": 0.07680130004882812,
|
||
|
|
"rewards/rejected": -0.10992081463336945,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.045368620037807186,
|
||
|
|
"grad_norm": 32.0,
|
||
|
|
"learning_rate": 2.966666666666667e-07,
|
||
|
|
"logits/chosen": -0.24648718535900116,
|
||
|
|
"logits/rejected": -0.16621707379817963,
|
||
|
|
"logps/chosen": -62.35799026489258,
|
||
|
|
"logps/rejected": -67.0835189819336,
|
||
|
|
"loss": 0.6371,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.039066143333911896,
|
||
|
|
"rewards/margins": 0.1181197538971901,
|
||
|
|
"rewards/rejected": -0.1571858823299408,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05040957781978576,
|
||
|
|
"grad_norm": 26.0,
|
||
|
|
"learning_rate": 3.3e-07,
|
||
|
|
"logits/chosen": -0.2515576481819153,
|
||
|
|
"logits/rejected": -0.19324862957000732,
|
||
|
|
"logps/chosen": -62.55232620239258,
|
||
|
|
"logps/rejected": -67.55314636230469,
|
||
|
|
"loss": 0.6217,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": -0.06582323461771011,
|
||
|
|
"rewards/margins": 0.15158048272132874,
|
||
|
|
"rewards/rejected": -0.21740372478961945,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05040957781978576,
|
||
|
|
"eval_logits/chosen": -0.3329714834690094,
|
||
|
|
"eval_logits/rejected": -0.27068039774894714,
|
||
|
|
"eval_logps/chosen": -62.82643508911133,
|
||
|
|
"eval_logps/rejected": -67.86115264892578,
|
||
|
|
"eval_loss": 0.6163578629493713,
|
||
|
|
"eval_rewards/accuracies": 0.8772454857826233,
|
||
|
|
"eval_rewards/chosen": -0.08201639354228973,
|
||
|
|
"eval_rewards/margins": 0.16533808410167694,
|
||
|
|
"eval_rewards/rejected": -0.24735447764396667,
|
||
|
|
"eval_runtime": 71.6288,
|
||
|
|
"eval_samples_per_second": 23.315,
|
||
|
|
"eval_steps_per_second": 23.315,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.055450535601764335,
|
||
|
|
"grad_norm": 34.5,
|
||
|
|
"learning_rate": 3.6333333333333333e-07,
|
||
|
|
"logits/chosen": -0.38161173462867737,
|
||
|
|
"logits/rejected": -0.3071076273918152,
|
||
|
|
"logps/chosen": -62.72125244140625,
|
||
|
|
"logps/rejected": -68.94508361816406,
|
||
|
|
"loss": 0.5933,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.07404644042253494,
|
||
|
|
"rewards/margins": 0.2169165164232254,
|
||
|
|
"rewards/rejected": -0.29096299409866333,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06049149338374291,
|
||
|
|
"grad_norm": 25.375,
|
||
|
|
"learning_rate": 3.9666666666666665e-07,
|
||
|
|
"logits/chosen": -0.40286582708358765,
|
||
|
|
"logits/rejected": -0.34470418095588684,
|
||
|
|
"logps/chosen": -63.3223762512207,
|
||
|
|
"logps/rejected": -68.89364624023438,
|
||
|
|
"loss": 0.5858,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": -0.1222512498497963,
|
||
|
|
"rewards/margins": 0.23771443963050842,
|
||
|
|
"rewards/rejected": -0.3599656820297241,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06553245116572148,
|
||
|
|
"grad_norm": 23.125,
|
||
|
|
"learning_rate": 4.2999999999999996e-07,
|
||
|
|
"logits/chosen": -0.4781390130519867,
|
||
|
|
"logits/rejected": -0.4242860674858093,
|
||
|
|
"logps/chosen": -63.421775817871094,
|
||
|
|
"logps/rejected": -69.99397277832031,
|
||
|
|
"loss": 0.5631,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.15090402960777283,
|
||
|
|
"rewards/margins": 0.29352977871894836,
|
||
|
|
"rewards/rejected": -0.4444337785243988,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07057340894770006,
|
||
|
|
"grad_norm": 23.25,
|
||
|
|
"learning_rate": 4.633333333333333e-07,
|
||
|
|
"logits/chosen": -0.5767303705215454,
|
||
|
|
"logits/rejected": -0.5158231854438782,
|
||
|
|
"logps/chosen": -64.02821350097656,
|
||
|
|
"logps/rejected": -70.48841857910156,
|
||
|
|
"loss": 0.5487,
|
||
|
|
"rewards/accuracies": 0.831250011920929,
|
||
|
|
"rewards/chosen": -0.18629249930381775,
|
||
|
|
"rewards/margins": 0.34241610765457153,
|
||
|
|
"rewards/rejected": -0.5287086367607117,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07561436672967864,
|
||
|
|
"grad_norm": 23.125,
|
||
|
|
"learning_rate": 4.966666666666666e-07,
|
||
|
|
"logits/chosen": -0.6403996348381042,
|
||
|
|
"logits/rejected": -0.6139777898788452,
|
||
|
|
"logps/chosen": -63.662750244140625,
|
||
|
|
"logps/rejected": -71.99102020263672,
|
||
|
|
"loss": 0.5035,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": -0.180726557970047,
|
||
|
|
"rewards/margins": 0.4561690390110016,
|
||
|
|
"rewards/rejected": -0.6368955373764038,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08065532451165722,
|
||
|
|
"grad_norm": 24.0,
|
||
|
|
"learning_rate": 4.975463467829879e-07,
|
||
|
|
"logits/chosen": -0.6926871538162231,
|
||
|
|
"logits/rejected": -0.6587594747543335,
|
||
|
|
"logps/chosen": -63.4420051574707,
|
||
|
|
"logps/rejected": -72.86981201171875,
|
||
|
|
"loss": 0.4668,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.1938735544681549,
|
||
|
|
"rewards/margins": 0.5592324137687683,
|
||
|
|
"rewards/rejected": -0.7531059980392456,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0856962822936358,
|
||
|
|
"grad_norm": 23.125,
|
||
|
|
"learning_rate": 4.948200654307524e-07,
|
||
|
|
"logits/chosen": -0.7586608529090881,
|
||
|
|
"logits/rejected": -0.737440824508667,
|
||
|
|
"logps/chosen": -64.16552734375,
|
||
|
|
"logps/rejected": -72.98350524902344,
|
||
|
|
"loss": 0.4687,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.22095069289207458,
|
||
|
|
"rewards/margins": 0.5723496675491333,
|
||
|
|
"rewards/rejected": -0.7933003306388855,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09073724007561437,
|
||
|
|
"grad_norm": 22.5,
|
||
|
|
"learning_rate": 4.920937840785169e-07,
|
||
|
|
"logits/chosen": -0.8342425227165222,
|
||
|
|
"logits/rejected": -0.8266761898994446,
|
||
|
|
"logps/chosen": -64.02726745605469,
|
||
|
|
"logps/rejected": -74.54716491699219,
|
||
|
|
"loss": 0.438,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": -0.22078147530555725,
|
||
|
|
"rewards/margins": 0.6804903745651245,
|
||
|
|
"rewards/rejected": -0.9012719392776489,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09577819785759294,
|
||
|
|
"grad_norm": 22.125,
|
||
|
|
"learning_rate": 4.893675027262814e-07,
|
||
|
|
"logits/chosen": -0.8242881894111633,
|
||
|
|
"logits/rejected": -0.8162325620651245,
|
||
|
|
"logps/chosen": -64.03907012939453,
|
||
|
|
"logps/rejected": -75.18043518066406,
|
||
|
|
"loss": 0.4107,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.21485848724842072,
|
||
|
|
"rewards/margins": 0.7709285616874695,
|
||
|
|
"rewards/rejected": -0.985787034034729,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10081915563957151,
|
||
|
|
"grad_norm": 19.375,
|
||
|
|
"learning_rate": 4.866412213740458e-07,
|
||
|
|
"logits/chosen": -0.8690522909164429,
|
||
|
|
"logits/rejected": -0.8740390539169312,
|
||
|
|
"logps/chosen": -63.641624450683594,
|
||
|
|
"logps/rejected": -75.82588195800781,
|
||
|
|
"loss": 0.3957,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.1833191215991974,
|
||
|
|
"rewards/margins": 0.8537132143974304,
|
||
|
|
"rewards/rejected": -1.0370323657989502,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10081915563957151,
|
||
|
|
"eval_logits/chosen": -0.8788526058197021,
|
||
|
|
"eval_logits/rejected": -0.8778823018074036,
|
||
|
|
"eval_logps/chosen": -64.24597930908203,
|
||
|
|
"eval_logps/rejected": -75.78997802734375,
|
||
|
|
"eval_loss": 0.41245341300964355,
|
||
|
|
"eval_rewards/accuracies": 0.8772454857826233,
|
||
|
|
"eval_rewards/chosen": -0.22397060692310333,
|
||
|
|
"eval_rewards/margins": 0.8162661790847778,
|
||
|
|
"eval_rewards/rejected": -1.0402368307113647,
|
||
|
|
"eval_runtime": 73.4377,
|
||
|
|
"eval_samples_per_second": 22.74,
|
||
|
|
"eval_steps_per_second": 22.74,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10586011342155009,
|
||
|
|
"grad_norm": 16.375,
|
||
|
|
"learning_rate": 4.839149400218102e-07,
|
||
|
|
"logits/chosen": -0.9034843444824219,
|
||
|
|
"logits/rejected": -0.9223787188529968,
|
||
|
|
"logps/chosen": -63.76435089111328,
|
||
|
|
"logps/rejected": -76.31708526611328,
|
||
|
|
"loss": 0.3748,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": -0.18892225623130798,
|
||
|
|
"rewards/margins": 0.9179438352584839,
|
||
|
|
"rewards/rejected": -1.1068661212921143,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11090107120352867,
|
||
|
|
"grad_norm": 19.25,
|
||
|
|
"learning_rate": 4.811886586695747e-07,
|
||
|
|
"logits/chosen": -0.9078966379165649,
|
||
|
|
"logits/rejected": -0.9262178540229797,
|
||
|
|
"logps/chosen": -63.7225456237793,
|
||
|
|
"logps/rejected": -76.8105239868164,
|
||
|
|
"loss": 0.3902,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": -0.18831631541252136,
|
||
|
|
"rewards/margins": 0.9241956472396851,
|
||
|
|
"rewards/rejected": -1.1125118732452393,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11594202898550725,
|
||
|
|
"grad_norm": 20.5,
|
||
|
|
"learning_rate": 4.784623773173392e-07,
|
||
|
|
"logits/chosen": -0.9634265899658203,
|
||
|
|
"logits/rejected": -0.9702705144882202,
|
||
|
|
"logps/chosen": -63.542388916015625,
|
||
|
|
"logps/rejected": -77.10391998291016,
|
||
|
|
"loss": 0.3663,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.19163846969604492,
|
||
|
|
"rewards/margins": 0.9870219230651855,
|
||
|
|
"rewards/rejected": -1.1786603927612305,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12098298676748583,
|
||
|
|
"grad_norm": 19.125,
|
||
|
|
"learning_rate": 4.7573609596510354e-07,
|
||
|
|
"logits/chosen": -0.920444130897522,
|
||
|
|
"logits/rejected": -0.9512295722961426,
|
||
|
|
"logps/chosen": -63.34272384643555,
|
||
|
|
"logps/rejected": -77.4976577758789,
|
||
|
|
"loss": 0.3607,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": -0.14525040984153748,
|
||
|
|
"rewards/margins": 1.072211503982544,
|
||
|
|
"rewards/rejected": -1.2174618244171143,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1260239445494644,
|
||
|
|
"grad_norm": 19.125,
|
||
|
|
"learning_rate": 4.7300981461286803e-07,
|
||
|
|
"logits/chosen": -0.9372714757919312,
|
||
|
|
"logits/rejected": -0.9608826637268066,
|
||
|
|
"logps/chosen": -63.630760192871094,
|
||
|
|
"logps/rejected": -77.2890625,
|
||
|
|
"loss": 0.3813,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": -0.14888228476047516,
|
||
|
|
"rewards/margins": 1.0466125011444092,
|
||
|
|
"rewards/rejected": -1.1954947710037231,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13106490233144297,
|
||
|
|
"grad_norm": 16.125,
|
||
|
|
"learning_rate": 4.7028353326063247e-07,
|
||
|
|
"logits/chosen": -0.9241989254951477,
|
||
|
|
"logits/rejected": -0.9565703272819519,
|
||
|
|
"logps/chosen": -62.576759338378906,
|
||
|
|
"logps/rejected": -78.99192810058594,
|
||
|
|
"loss": 0.2995,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": -0.06501423567533493,
|
||
|
|
"rewards/margins": 1.2511422634124756,
|
||
|
|
"rewards/rejected": -1.3161563873291016,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13610586011342155,
|
||
|
|
"grad_norm": 18.375,
|
||
|
|
"learning_rate": 4.675572519083969e-07,
|
||
|
|
"logits/chosen": -0.9289599657058716,
|
||
|
|
"logits/rejected": -0.959551990032196,
|
||
|
|
"logps/chosen": -64.06565856933594,
|
||
|
|
"logps/rejected": -78.49452209472656,
|
||
|
|
"loss": 0.365,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": -0.16143986582756042,
|
||
|
|
"rewards/margins": 1.1138832569122314,
|
||
|
|
"rewards/rejected": -1.2753230333328247,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14114681789540012,
|
||
|
|
"grad_norm": 14.75,
|
||
|
|
"learning_rate": 4.648309705561614e-07,
|
||
|
|
"logits/chosen": -0.9531835317611694,
|
||
|
|
"logits/rejected": -0.9772550463676453,
|
||
|
|
"logps/chosen": -63.0753173828125,
|
||
|
|
"logps/rejected": -78.2010726928711,
|
||
|
|
"loss": 0.3479,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": -0.13170328736305237,
|
||
|
|
"rewards/margins": 1.1704909801483154,
|
||
|
|
"rewards/rejected": -1.3021942377090454,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1461877756773787,
|
||
|
|
"grad_norm": 14.125,
|
||
|
|
"learning_rate": 4.6210468920392583e-07,
|
||
|
|
"logits/chosen": -0.9216547012329102,
|
||
|
|
"logits/rejected": -0.9838453531265259,
|
||
|
|
"logps/chosen": -62.03778076171875,
|
||
|
|
"logps/rejected": -79.33902740478516,
|
||
|
|
"loss": 0.2986,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": -0.029102539643645287,
|
||
|
|
"rewards/margins": 1.3270413875579834,
|
||
|
|
"rewards/rejected": -1.3561439514160156,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15122873345935728,
|
||
|
|
"grad_norm": 18.375,
|
||
|
|
"learning_rate": 4.5937840785169027e-07,
|
||
|
|
"logits/chosen": -0.9648447036743164,
|
||
|
|
"logits/rejected": -1.0225803852081299,
|
||
|
|
"logps/chosen": -61.83771514892578,
|
||
|
|
"logps/rejected": -79.71820831298828,
|
||
|
|
"loss": 0.2767,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": -0.035991497337818146,
|
||
|
|
"rewards/margins": 1.3972457647323608,
|
||
|
|
"rewards/rejected": -1.4332373142242432,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15122873345935728,
|
||
|
|
"eval_logits/chosen": -0.977104663848877,
|
||
|
|
"eval_logits/rejected": -1.0141139030456543,
|
||
|
|
"eval_logps/chosen": -63.36873245239258,
|
||
|
|
"eval_logps/rejected": -79.43486022949219,
|
||
|
|
"eval_loss": 0.3420425057411194,
|
||
|
|
"eval_rewards/accuracies": 0.8808383345603943,
|
||
|
|
"eval_rewards/chosen": -0.1362457126379013,
|
||
|
|
"eval_rewards/margins": 1.2684792280197144,
|
||
|
|
"eval_rewards/rejected": -1.4047249555587769,
|
||
|
|
"eval_runtime": 70.6339,
|
||
|
|
"eval_samples_per_second": 23.643,
|
||
|
|
"eval_steps_per_second": 23.643,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15626969124133586,
|
||
|
|
"grad_norm": 13.75,
|
||
|
|
"learning_rate": 4.566521264994547e-07,
|
||
|
|
"logits/chosen": -0.9742618799209595,
|
||
|
|
"logits/rejected": -1.0176128149032593,
|
||
|
|
"logps/chosen": -63.59131622314453,
|
||
|
|
"logps/rejected": -79.64852905273438,
|
||
|
|
"loss": 0.3397,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": -0.14291712641716003,
|
||
|
|
"rewards/margins": 1.2807643413543701,
|
||
|
|
"rewards/rejected": -1.4236814975738525,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16131064902331443,
|
||
|
|
"grad_norm": 21.625,
|
||
|
|
"learning_rate": 4.5392584514721915e-07,
|
||
|
|
"logits/chosen": -0.9295794367790222,
|
||
|
|
"logits/rejected": -0.9652513265609741,
|
||
|
|
"logps/chosen": -62.136070251464844,
|
||
|
|
"logps/rejected": -80.7836685180664,
|
||
|
|
"loss": 0.2774,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": -0.03530962020158768,
|
||
|
|
"rewards/margins": 1.468548059463501,
|
||
|
|
"rewards/rejected": -1.5038578510284424,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.166351606805293,
|
||
|
|
"grad_norm": 14.4375,
|
||
|
|
"learning_rate": 4.5119956379498364e-07,
|
||
|
|
"logits/chosen": -0.9235115051269531,
|
||
|
|
"logits/rejected": -0.9803950190544128,
|
||
|
|
"logps/chosen": -62.91417694091797,
|
||
|
|
"logps/rejected": -80.86149597167969,
|
||
|
|
"loss": 0.292,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.07058247923851013,
|
||
|
|
"rewards/margins": 1.4306252002716064,
|
||
|
|
"rewards/rejected": -1.5012075901031494,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1713925645872716,
|
||
|
|
"grad_norm": 22.875,
|
||
|
|
"learning_rate": 4.484732824427481e-07,
|
||
|
|
"logits/chosen": -0.9881145358085632,
|
||
|
|
"logits/rejected": -1.0382649898529053,
|
||
|
|
"logps/chosen": -62.81303787231445,
|
||
|
|
"logps/rejected": -81.26265716552734,
|
||
|
|
"loss": 0.278,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": -0.061654943972826004,
|
||
|
|
"rewards/margins": 1.5001782178878784,
|
||
|
|
"rewards/rejected": -1.561833143234253,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17643352236925017,
|
||
|
|
"grad_norm": 10.25,
|
||
|
|
"learning_rate": 4.457470010905125e-07,
|
||
|
|
"logits/chosen": -0.9716188311576843,
|
||
|
|
"logits/rejected": -1.0232843160629272,
|
||
|
|
"logps/chosen": -63.168006896972656,
|
||
|
|
"logps/rejected": -79.84440612792969,
|
||
|
|
"loss": 0.3661,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": -0.10739569365978241,
|
||
|
|
"rewards/margins": 1.3512216806411743,
|
||
|
|
"rewards/rejected": -1.4586174488067627,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18147448015122875,
|
||
|
|
"grad_norm": 16.125,
|
||
|
|
"learning_rate": 4.43020719738277e-07,
|
||
|
|
"logits/chosen": -0.9138787388801575,
|
||
|
|
"logits/rejected": -0.9851690530776978,
|
||
|
|
"logps/chosen": -61.79378128051758,
|
||
|
|
"logps/rejected": -81.6797866821289,
|
||
|
|
"loss": 0.2695,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.00970448087900877,
|
||
|
|
"rewards/margins": 1.6192023754119873,
|
||
|
|
"rewards/rejected": -1.6094980239868164,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18651543793320732,
|
||
|
|
"grad_norm": 12.0625,
|
||
|
|
"learning_rate": 4.402944383860414e-07,
|
||
|
|
"logits/chosen": -0.932845950126648,
|
||
|
|
"logits/rejected": -0.98590487241745,
|
||
|
|
"logps/chosen": -62.03528594970703,
|
||
|
|
"logps/rejected": -81.13326263427734,
|
||
|
|
"loss": 0.2783,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.0034210742451250553,
|
||
|
|
"rewards/margins": 1.5679662227630615,
|
||
|
|
"rewards/rejected": -1.5713872909545898,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19155639571518587,
|
||
|
|
"grad_norm": 9.625,
|
||
|
|
"learning_rate": 4.375681570338059e-07,
|
||
|
|
"logits/chosen": -0.9335094690322876,
|
||
|
|
"logits/rejected": -0.991866946220398,
|
||
|
|
"logps/chosen": -61.888038635253906,
|
||
|
|
"logps/rejected": -80.99397277832031,
|
||
|
|
"loss": 0.3039,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.0033101304434239864,
|
||
|
|
"rewards/margins": 1.5600807666778564,
|
||
|
|
"rewards/rejected": -1.5567705631256104,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19659735349716445,
|
||
|
|
"grad_norm": 24.5,
|
||
|
|
"learning_rate": 4.348418756815703e-07,
|
||
|
|
"logits/chosen": -0.9722970724105835,
|
||
|
|
"logits/rejected": -1.0437185764312744,
|
||
|
|
"logps/chosen": -62.06909942626953,
|
||
|
|
"logps/rejected": -80.99043273925781,
|
||
|
|
"loss": 0.2994,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": -0.017526980489492416,
|
||
|
|
"rewards/margins": 1.57229483127594,
|
||
|
|
"rewards/rejected": -1.5898219347000122,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20163831127914303,
|
||
|
|
"grad_norm": 14.3125,
|
||
|
|
"learning_rate": 4.3211559432933476e-07,
|
||
|
|
"logits/chosen": -0.9651460647583008,
|
||
|
|
"logits/rejected": -1.0433061122894287,
|
||
|
|
"logps/chosen": -61.74712371826172,
|
||
|
|
"logps/rejected": -82.36299133300781,
|
||
|
|
"loss": 0.2577,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.03247659280896187,
|
||
|
|
"rewards/margins": 1.6827892065048218,
|
||
|
|
"rewards/rejected": -1.6503126621246338,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20163831127914303,
|
||
|
|
"eval_logits/chosen": -0.9533343315124512,
|
||
|
|
"eval_logits/rejected": -1.0109245777130127,
|
||
|
|
"eval_logps/chosen": -62.41316604614258,
|
||
|
|
"eval_logps/rejected": -81.344482421875,
|
||
|
|
"eval_loss": 0.3170239329338074,
|
||
|
|
"eval_rewards/accuracies": 0.8856287598609924,
|
||
|
|
"eval_rewards/chosen": -0.040688931941986084,
|
||
|
|
"eval_rewards/margins": 1.5549986362457275,
|
||
|
|
"eval_rewards/rejected": -1.5956875085830688,
|
||
|
|
"eval_runtime": 71.3317,
|
||
|
|
"eval_samples_per_second": 23.412,
|
||
|
|
"eval_steps_per_second": 23.412,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2066792690611216,
|
||
|
|
"grad_norm": 24.25,
|
||
|
|
"learning_rate": 4.2938931297709925e-07,
|
||
|
|
"logits/chosen": -0.9435351490974426,
|
||
|
|
"logits/rejected": -1.0250452756881714,
|
||
|
|
"logps/chosen": -61.98693084716797,
|
||
|
|
"logps/rejected": -82.12709045410156,
|
||
|
|
"loss": 0.2696,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.0036094195675104856,
|
||
|
|
"rewards/margins": 1.654160737991333,
|
||
|
|
"rewards/rejected": -1.650551199913025,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21172022684310018,
|
||
|
|
"grad_norm": 11.5625,
|
||
|
|
"learning_rate": 4.266630316248637e-07,
|
||
|
|
"logits/chosen": -0.9397176504135132,
|
||
|
|
"logits/rejected": -1.0133641958236694,
|
||
|
|
"logps/chosen": -61.4076042175293,
|
||
|
|
"logps/rejected": -83.43376159667969,
|
||
|
|
"loss": 0.258,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 0.0406290665268898,
|
||
|
|
"rewards/margins": 1.7928526401519775,
|
||
|
|
"rewards/rejected": -1.7522236108779907,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21676118462507876,
|
||
|
|
"grad_norm": 17.75,
|
||
|
|
"learning_rate": 4.239367502726281e-07,
|
||
|
|
"logits/chosen": -0.9263733625411987,
|
||
|
|
"logits/rejected": -1.007187843322754,
|
||
|
|
"logps/chosen": -60.838951110839844,
|
||
|
|
"logps/rejected": -82.61154174804688,
|
||
|
|
"loss": 0.2287,
|
||
|
|
"rewards/accuracies": 0.9437500238418579,
|
||
|
|
"rewards/chosen": 0.09760002046823502,
|
||
|
|
"rewards/margins": 1.8123610019683838,
|
||
|
|
"rewards/rejected": -1.7147607803344727,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22180214240705734,
|
||
|
|
"grad_norm": 18.375,
|
||
|
|
"learning_rate": 4.2121046892039257e-07,
|
||
|
|
"logits/chosen": -0.9172664880752563,
|
||
|
|
"logits/rejected": -0.98748779296875,
|
||
|
|
"logps/chosen": -61.5716667175293,
|
||
|
|
"logps/rejected": -82.24566650390625,
|
||
|
|
"loss": 0.2758,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.008749181404709816,
|
||
|
|
"rewards/margins": 1.7142765522003174,
|
||
|
|
"rewards/rejected": -1.7055273056030273,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22684310018903592,
|
||
|
|
"grad_norm": 28.75,
|
||
|
|
"learning_rate": 4.18484187568157e-07,
|
||
|
|
"logits/chosen": -0.9780591726303101,
|
||
|
|
"logits/rejected": -1.02444326877594,
|
||
|
|
"logps/chosen": -62.16161346435547,
|
||
|
|
"logps/rejected": -81.88623046875,
|
||
|
|
"loss": 0.2933,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": -0.024681804701685905,
|
||
|
|
"rewards/margins": 1.6520192623138428,
|
||
|
|
"rewards/rejected": -1.6767011880874634,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2318840579710145,
|
||
|
|
"grad_norm": 15.375,
|
||
|
|
"learning_rate": 4.1575790621592144e-07,
|
||
|
|
"logits/chosen": -0.9287646412849426,
|
||
|
|
"logits/rejected": -0.9916049838066101,
|
||
|
|
"logps/chosen": -61.4310188293457,
|
||
|
|
"logps/rejected": -82.45499420166016,
|
||
|
|
"loss": 0.2939,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.04269082099199295,
|
||
|
|
"rewards/margins": 1.7475837469100952,
|
||
|
|
"rewards/rejected": -1.704892873764038,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23692501575299307,
|
||
|
|
"grad_norm": 13.75,
|
||
|
|
"learning_rate": 4.1303162486368593e-07,
|
||
|
|
"logits/chosen": -0.9281194806098938,
|
||
|
|
"logits/rejected": -1.0078294277191162,
|
||
|
|
"logps/chosen": -60.9696159362793,
|
||
|
|
"logps/rejected": -82.53529357910156,
|
||
|
|
"loss": 0.2433,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": 0.07616719603538513,
|
||
|
|
"rewards/margins": 1.8099626302719116,
|
||
|
|
"rewards/rejected": -1.7337955236434937,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24196597353497165,
|
||
|
|
"grad_norm": 24.125,
|
||
|
|
"learning_rate": 4.1030534351145037e-07,
|
||
|
|
"logits/chosen": -0.9563441276550293,
|
||
|
|
"logits/rejected": -1.0490442514419556,
|
||
|
|
"logps/chosen": -61.71998977661133,
|
||
|
|
"logps/rejected": -81.29328918457031,
|
||
|
|
"loss": 0.3422,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 0.017765596508979797,
|
||
|
|
"rewards/margins": 1.6064462661743164,
|
||
|
|
"rewards/rejected": -1.5886808633804321,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24700693131695023,
|
||
|
|
"grad_norm": 18.75,
|
||
|
|
"learning_rate": 4.075790621592148e-07,
|
||
|
|
"logits/chosen": -0.9509885907173157,
|
||
|
|
"logits/rejected": -1.0152177810668945,
|
||
|
|
"logps/chosen": -60.49787139892578,
|
||
|
|
"logps/rejected": -82.62560272216797,
|
||
|
|
"loss": 0.2712,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.13019904494285583,
|
||
|
|
"rewards/margins": 1.8366725444793701,
|
||
|
|
"rewards/rejected": -1.7064735889434814,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2520478890989288,
|
||
|
|
"grad_norm": 20.0,
|
||
|
|
"learning_rate": 4.0485278080697925e-07,
|
||
|
|
"logits/chosen": -0.9022938013076782,
|
||
|
|
"logits/rejected": -0.9922693371772766,
|
||
|
|
"logps/chosen": -61.09675216674805,
|
||
|
|
"logps/rejected": -82.68385314941406,
|
||
|
|
"loss": 0.2781,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.08039947599172592,
|
||
|
|
"rewards/margins": 1.8100239038467407,
|
||
|
|
"rewards/rejected": -1.7296245098114014,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2520478890989288,
|
||
|
|
"eval_logits/chosen": -0.9490191340446472,
|
||
|
|
"eval_logits/rejected": -1.0179123878479004,
|
||
|
|
"eval_logps/chosen": -61.8110466003418,
|
||
|
|
"eval_logps/rejected": -82.53450775146484,
|
||
|
|
"eval_loss": 0.30638858675956726,
|
||
|
|
"eval_rewards/accuracies": 0.8856287598609924,
|
||
|
|
"eval_rewards/chosen": 0.019522832706570625,
|
||
|
|
"eval_rewards/margins": 1.734211802482605,
|
||
|
|
"eval_rewards/rejected": -1.7146891355514526,
|
||
|
|
"eval_runtime": 72.5423,
|
||
|
|
"eval_samples_per_second": 23.021,
|
||
|
|
"eval_steps_per_second": 23.021,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2570888468809074,
|
||
|
|
"grad_norm": 32.25,
|
||
|
|
"learning_rate": 4.021264994547437e-07,
|
||
|
|
"logits/chosen": -0.9350666999816895,
|
||
|
|
"logits/rejected": -1.0141990184783936,
|
||
|
|
"logps/chosen": -61.0509033203125,
|
||
|
|
"logps/rejected": -82.5855712890625,
|
||
|
|
"loss": 0.2704,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.08123103529214859,
|
||
|
|
"rewards/margins": 1.7908775806427002,
|
||
|
|
"rewards/rejected": -1.7096465826034546,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.26212980466288593,
|
||
|
|
"grad_norm": 25.75,
|
||
|
|
"learning_rate": 3.994002181025082e-07,
|
||
|
|
"logits/chosen": -0.9520455598831177,
|
||
|
|
"logits/rejected": -1.0035117864608765,
|
||
|
|
"logps/chosen": -60.493560791015625,
|
||
|
|
"logps/rejected": -82.90555572509766,
|
||
|
|
"loss": 0.2678,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.1298852264881134,
|
||
|
|
"rewards/margins": 1.8753770589828491,
|
||
|
|
"rewards/rejected": -1.7454917430877686,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.26717076244486454,
|
||
|
|
"grad_norm": 9.375,
|
||
|
|
"learning_rate": 3.966739367502726e-07,
|
||
|
|
"logits/chosen": -0.9617950320243835,
|
||
|
|
"logits/rejected": -1.0328407287597656,
|
||
|
|
"logps/chosen": -61.49128341674805,
|
||
|
|
"logps/rejected": -81.98192596435547,
|
||
|
|
"loss": 0.3276,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.05107005685567856,
|
||
|
|
"rewards/margins": 1.7408430576324463,
|
||
|
|
"rewards/rejected": -1.6897728443145752,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2722117202268431,
|
||
|
|
"grad_norm": 24.375,
|
||
|
|
"learning_rate": 3.9394765539803705e-07,
|
||
|
|
"logits/chosen": -0.9145433306694031,
|
||
|
|
"logits/rejected": -0.9903620481491089,
|
||
|
|
"logps/chosen": -60.99120330810547,
|
||
|
|
"logps/rejected": -83.13854217529297,
|
||
|
|
"loss": 0.2787,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.09986008703708649,
|
||
|
|
"rewards/margins": 1.8553259372711182,
|
||
|
|
"rewards/rejected": -1.7554658651351929,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2772526780088217,
|
||
|
|
"grad_norm": 24.125,
|
||
|
|
"learning_rate": 3.9122137404580155e-07,
|
||
|
|
"logits/chosen": -0.9315235018730164,
|
||
|
|
"logits/rejected": -1.0184853076934814,
|
||
|
|
"logps/chosen": -61.380882263183594,
|
||
|
|
"logps/rejected": -83.67707061767578,
|
||
|
|
"loss": 0.2747,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.0887889489531517,
|
||
|
|
"rewards/margins": 1.894553780555725,
|
||
|
|
"rewards/rejected": -1.8057647943496704,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28229363579080025,
|
||
|
|
"grad_norm": 11.875,
|
||
|
|
"learning_rate": 3.8849509269356593e-07,
|
||
|
|
"logits/chosen": -0.8931863903999329,
|
||
|
|
"logits/rejected": -0.9775617718696594,
|
||
|
|
"logps/chosen": -60.56890869140625,
|
||
|
|
"logps/rejected": -83.23638916015625,
|
||
|
|
"loss": 0.2602,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.1255245953798294,
|
||
|
|
"rewards/margins": 1.8955485820770264,
|
||
|
|
"rewards/rejected": -1.770024061203003,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28733459357277885,
|
||
|
|
"grad_norm": 13.0,
|
||
|
|
"learning_rate": 3.8576881134133037e-07,
|
||
|
|
"logits/chosen": -0.9146768450737,
|
||
|
|
"logits/rejected": -1.018970251083374,
|
||
|
|
"logps/chosen": -60.158790588378906,
|
||
|
|
"logps/rejected": -83.05947875976562,
|
||
|
|
"loss": 0.258,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.1570456475019455,
|
||
|
|
"rewards/margins": 1.9360195398330688,
|
||
|
|
"rewards/rejected": -1.778973937034607,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2923755513547574,
|
||
|
|
"grad_norm": 25.0,
|
||
|
|
"learning_rate": 3.8304252998909486e-07,
|
||
|
|
"logits/chosen": -0.899819552898407,
|
||
|
|
"logits/rejected": -0.990888774394989,
|
||
|
|
"logps/chosen": -61.66679763793945,
|
||
|
|
"logps/rejected": -81.74058532714844,
|
||
|
|
"loss": 0.3258,
|
||
|
|
"rewards/accuracies": 0.84375,
|
||
|
|
"rewards/chosen": 0.05306190997362137,
|
||
|
|
"rewards/margins": 1.7001397609710693,
|
||
|
|
"rewards/rejected": -1.6470777988433838,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.297416509136736,
|
||
|
|
"grad_norm": 15.1875,
|
||
|
|
"learning_rate": 3.803162486368593e-07,
|
||
|
|
"logits/chosen": -0.9033399820327759,
|
||
|
|
"logits/rejected": -0.9803060293197632,
|
||
|
|
"logps/chosen": -60.785057067871094,
|
||
|
|
"logps/rejected": -83.88690185546875,
|
||
|
|
"loss": 0.2538,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.10141563415527344,
|
||
|
|
"rewards/margins": 1.9382518529891968,
|
||
|
|
"rewards/rejected": -1.8368362188339233,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30245746691871456,
|
||
|
|
"grad_norm": 15.25,
|
||
|
|
"learning_rate": 3.775899672846238e-07,
|
||
|
|
"logits/chosen": -0.9281272888183594,
|
||
|
|
"logits/rejected": -1.005056619644165,
|
||
|
|
"logps/chosen": -62.294593811035156,
|
||
|
|
"logps/rejected": -82.4925308227539,
|
||
|
|
"loss": 0.3627,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.008905407041311264,
|
||
|
|
"rewards/margins": 1.7188129425048828,
|
||
|
|
"rewards/rejected": -1.7099075317382812,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30245746691871456,
|
||
|
|
"eval_logits/chosen": -0.9133957028388977,
|
||
|
|
"eval_logits/rejected": -0.9896692037582397,
|
||
|
|
"eval_logps/chosen": -61.07933807373047,
|
||
|
|
"eval_logps/rejected": -82.90621948242188,
|
||
|
|
"eval_loss": 0.30104419589042664,
|
||
|
|
"eval_rewards/accuracies": 0.8844311237335205,
|
||
|
|
"eval_rewards/chosen": 0.09269363433122635,
|
||
|
|
"eval_rewards/margins": 1.8445546627044678,
|
||
|
|
"eval_rewards/rejected": -1.7518609762191772,
|
||
|
|
"eval_runtime": 73.1888,
|
||
|
|
"eval_samples_per_second": 22.818,
|
||
|
|
"eval_steps_per_second": 22.818,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3074984247006931,
|
||
|
|
"grad_norm": 30.5,
|
||
|
|
"learning_rate": 3.7486368593238823e-07,
|
||
|
|
"logits/chosen": -0.9185993075370789,
|
||
|
|
"logits/rejected": -1.0144493579864502,
|
||
|
|
"logps/chosen": -61.656349182128906,
|
||
|
|
"logps/rejected": -81.76588439941406,
|
||
|
|
"loss": 0.3439,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 0.051804833114147186,
|
||
|
|
"rewards/margins": 1.708714485168457,
|
||
|
|
"rewards/rejected": -1.6569095849990845,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3125393824826717,
|
||
|
|
"grad_norm": 18.0,
|
||
|
|
"learning_rate": 3.721374045801526e-07,
|
||
|
|
"logits/chosen": -0.9089874029159546,
|
||
|
|
"logits/rejected": -0.9995435476303101,
|
||
|
|
"logps/chosen": -60.14899444580078,
|
||
|
|
"logps/rejected": -82.75482177734375,
|
||
|
|
"loss": 0.2812,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.16136237978935242,
|
||
|
|
"rewards/margins": 1.9296493530273438,
|
||
|
|
"rewards/rejected": -1.768286943435669,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31758034026465026,
|
||
|
|
"grad_norm": 28.0,
|
||
|
|
"learning_rate": 3.694111232279171e-07,
|
||
|
|
"logits/chosen": -0.9505274891853333,
|
||
|
|
"logits/rejected": -1.028907060623169,
|
||
|
|
"logps/chosen": -60.5921516418457,
|
||
|
|
"logps/rejected": -83.64252471923828,
|
||
|
|
"loss": 0.2844,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.13824741542339325,
|
||
|
|
"rewards/margins": 1.9466102123260498,
|
||
|
|
"rewards/rejected": -1.808362603187561,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32262129804662887,
|
||
|
|
"grad_norm": 22.625,
|
||
|
|
"learning_rate": 3.6668484187568154e-07,
|
||
|
|
"logits/chosen": -0.9480802416801453,
|
||
|
|
"logits/rejected": -1.0301778316497803,
|
||
|
|
"logps/chosen": -60.25774002075195,
|
||
|
|
"logps/rejected": -82.69186401367188,
|
||
|
|
"loss": 0.2888,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.15573999285697937,
|
||
|
|
"rewards/margins": 1.877264380455017,
|
||
|
|
"rewards/rejected": -1.7215244770050049,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3276622558286074,
|
||
|
|
"grad_norm": 28.25,
|
||
|
|
"learning_rate": 3.63958560523446e-07,
|
||
|
|
"logits/chosen": -0.9399679899215698,
|
||
|
|
"logits/rejected": -0.9984865188598633,
|
||
|
|
"logps/chosen": -61.39536666870117,
|
||
|
|
"logps/rejected": -82.51799774169922,
|
||
|
|
"loss": 0.3268,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.06745560467243195,
|
||
|
|
"rewards/margins": 1.796059250831604,
|
||
|
|
"rewards/rejected": -1.7286036014556885,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.332703213610586,
|
||
|
|
"grad_norm": 30.125,
|
||
|
|
"learning_rate": 3.6123227917121047e-07,
|
||
|
|
"logits/chosen": -0.9196429252624512,
|
||
|
|
"logits/rejected": -1.0079903602600098,
|
||
|
|
"logps/chosen": -60.20268630981445,
|
||
|
|
"logps/rejected": -84.8389663696289,
|
||
|
|
"loss": 0.2459,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.16887502372264862,
|
||
|
|
"rewards/margins": 2.105717420578003,
|
||
|
|
"rewards/rejected": -1.9368425607681274,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3377441713925646,
|
||
|
|
"grad_norm": 26.75,
|
||
|
|
"learning_rate": 3.585059978189749e-07,
|
||
|
|
"logits/chosen": -0.9201717376708984,
|
||
|
|
"logits/rejected": -1.001564621925354,
|
||
|
|
"logps/chosen": -60.38823318481445,
|
||
|
|
"logps/rejected": -83.207275390625,
|
||
|
|
"loss": 0.2703,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.17042402923107147,
|
||
|
|
"rewards/margins": 1.9416764974594116,
|
||
|
|
"rewards/rejected": -1.7712528705596924,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3427851291745432,
|
||
|
|
"grad_norm": 10.3125,
|
||
|
|
"learning_rate": 3.557797164667394e-07,
|
||
|
|
"logits/chosen": -0.8961941003799438,
|
||
|
|
"logits/rejected": -0.9779438972473145,
|
||
|
|
"logps/chosen": -59.930824279785156,
|
||
|
|
"logps/rejected": -83.56443786621094,
|
||
|
|
"loss": 0.2806,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.18050600588321686,
|
||
|
|
"rewards/margins": 1.9991801977157593,
|
||
|
|
"rewards/rejected": -1.818674087524414,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34782608695652173,
|
||
|
|
"grad_norm": 22.25,
|
||
|
|
"learning_rate": 3.530534351145038e-07,
|
||
|
|
"logits/chosen": -0.9111859202384949,
|
||
|
|
"logits/rejected": -1.0103601217269897,
|
||
|
|
"logps/chosen": -59.5981559753418,
|
||
|
|
"logps/rejected": -84.48451232910156,
|
||
|
|
"loss": 0.2155,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 0.22089195251464844,
|
||
|
|
"rewards/margins": 2.098818063735962,
|
||
|
|
"rewards/rejected": -1.8779258728027344,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.35286704473850034,
|
||
|
|
"grad_norm": 8.4375,
|
||
|
|
"learning_rate": 3.503271537622682e-07,
|
||
|
|
"logits/chosen": -0.8836026191711426,
|
||
|
|
"logits/rejected": -0.9876821637153625,
|
||
|
|
"logps/chosen": -59.78227615356445,
|
||
|
|
"logps/rejected": -84.5053939819336,
|
||
|
|
"loss": 0.2411,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.20316052436828613,
|
||
|
|
"rewards/margins": 2.080014944076538,
|
||
|
|
"rewards/rejected": -1.8768543004989624,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.35286704473850034,
|
||
|
|
"eval_logits/chosen": -0.9257686138153076,
|
||
|
|
"eval_logits/rejected": -1.0098994970321655,
|
||
|
|
"eval_logps/chosen": -60.61675262451172,
|
||
|
|
"eval_logps/rejected": -83.34400939941406,
|
||
|
|
"eval_loss": 0.2979341447353363,
|
||
|
|
"eval_rewards/accuracies": 0.886227548122406,
|
||
|
|
"eval_rewards/chosen": 0.13895215094089508,
|
||
|
|
"eval_rewards/margins": 1.9345914125442505,
|
||
|
|
"eval_rewards/rejected": -1.7956393957138062,
|
||
|
|
"eval_runtime": 73.2808,
|
||
|
|
"eval_samples_per_second": 22.789,
|
||
|
|
"eval_steps_per_second": 22.789,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3579080025204789,
|
||
|
|
"grad_norm": 11.8125,
|
||
|
|
"learning_rate": 3.476008724100327e-07,
|
||
|
|
"logits/chosen": -0.8866975903511047,
|
||
|
|
"logits/rejected": -0.9669508934020996,
|
||
|
|
"logps/chosen": -60.970123291015625,
|
||
|
|
"logps/rejected": -83.34266662597656,
|
||
|
|
"loss": 0.3138,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.0989324077963829,
|
||
|
|
"rewards/margins": 1.9064470529556274,
|
||
|
|
"rewards/rejected": -1.8075145483016968,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3629489603024575,
|
||
|
|
"grad_norm": 12.125,
|
||
|
|
"learning_rate": 3.4487459105779716e-07,
|
||
|
|
"logits/chosen": -0.8704292178153992,
|
||
|
|
"logits/rejected": -0.9828470349311829,
|
||
|
|
"logps/chosen": -58.7971076965332,
|
||
|
|
"logps/rejected": -83.87742614746094,
|
||
|
|
"loss": 0.2295,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.2823036015033722,
|
||
|
|
"rewards/margins": 2.1361794471740723,
|
||
|
|
"rewards/rejected": -1.8538758754730225,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36798991808443604,
|
||
|
|
"grad_norm": 27.75,
|
||
|
|
"learning_rate": 3.421483097055616e-07,
|
||
|
|
"logits/chosen": -0.8949785232543945,
|
||
|
|
"logits/rejected": -0.9850034713745117,
|
||
|
|
"logps/chosen": -59.54541778564453,
|
||
|
|
"logps/rejected": -85.09830474853516,
|
||
|
|
"loss": 0.1848,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": 0.2564947009086609,
|
||
|
|
"rewards/margins": 2.1756319999694824,
|
||
|
|
"rewards/rejected": -1.9191375970840454,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.37303087586641465,
|
||
|
|
"grad_norm": 30.625,
|
||
|
|
"learning_rate": 3.394220283533261e-07,
|
||
|
|
"logits/chosen": -0.9334227442741394,
|
||
|
|
"logits/rejected": -1.010840654373169,
|
||
|
|
"logps/chosen": -60.2901725769043,
|
||
|
|
"logps/rejected": -84.69157409667969,
|
||
|
|
"loss": 0.2416,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.15744934976100922,
|
||
|
|
"rewards/margins": 2.060612678527832,
|
||
|
|
"rewards/rejected": -1.9031633138656616,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3780718336483932,
|
||
|
|
"grad_norm": 20.0,
|
||
|
|
"learning_rate": 3.3669574700109047e-07,
|
||
|
|
"logits/chosen": -0.9105051159858704,
|
||
|
|
"logits/rejected": -1.0369561910629272,
|
||
|
|
"logps/chosen": -57.91829299926758,
|
||
|
|
"logps/rejected": -85.75810241699219,
|
||
|
|
"loss": 0.1725,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 0.35346147418022156,
|
||
|
|
"rewards/margins": 2.3853201866149902,
|
||
|
|
"rewards/rejected": -2.0318589210510254,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38311279143037175,
|
||
|
|
"grad_norm": 24.375,
|
||
|
|
"learning_rate": 3.339694656488549e-07,
|
||
|
|
"logits/chosen": -0.8942914009094238,
|
||
|
|
"logits/rejected": -1.0089881420135498,
|
||
|
|
"logps/chosen": -60.97709274291992,
|
||
|
|
"logps/rejected": -84.78167724609375,
|
||
|
|
"loss": 0.2952,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.10825882852077484,
|
||
|
|
"rewards/margins": 2.0440988540649414,
|
||
|
|
"rewards/rejected": -1.9358398914337158,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38815374921235035,
|
||
|
|
"grad_norm": 21.875,
|
||
|
|
"learning_rate": 3.312431842966194e-07,
|
||
|
|
"logits/chosen": -0.8745867609977722,
|
||
|
|
"logits/rejected": -0.9834270477294922,
|
||
|
|
"logps/chosen": -60.55878829956055,
|
||
|
|
"logps/rejected": -84.3352279663086,
|
||
|
|
"loss": 0.2894,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.13102281093597412,
|
||
|
|
"rewards/margins": 2.0422165393829346,
|
||
|
|
"rewards/rejected": -1.91119384765625,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3931947069943289,
|
||
|
|
"grad_norm": 25.125,
|
||
|
|
"learning_rate": 3.2851690294438384e-07,
|
||
|
|
"logits/chosen": -0.8937376141548157,
|
||
|
|
"logits/rejected": -0.9868464469909668,
|
||
|
|
"logps/chosen": -59.818359375,
|
||
|
|
"logps/rejected": -85.10429382324219,
|
||
|
|
"loss": 0.2596,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.20219922065734863,
|
||
|
|
"rewards/margins": 2.1438941955566406,
|
||
|
|
"rewards/rejected": -1.9416948556900024,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3982356647763075,
|
||
|
|
"grad_norm": 32.25,
|
||
|
|
"learning_rate": 3.2579062159214833e-07,
|
||
|
|
"logits/chosen": -0.92058265209198,
|
||
|
|
"logits/rejected": -1.0123975276947021,
|
||
|
|
"logps/chosen": -60.107933044433594,
|
||
|
|
"logps/rejected": -84.3521957397461,
|
||
|
|
"loss": 0.294,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.1681869924068451,
|
||
|
|
"rewards/margins": 2.0432465076446533,
|
||
|
|
"rewards/rejected": -1.8750594854354858,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40327662255828606,
|
||
|
|
"grad_norm": 15.0625,
|
||
|
|
"learning_rate": 3.2306434023991277e-07,
|
||
|
|
"logits/chosen": -0.896182656288147,
|
||
|
|
"logits/rejected": -0.9941380620002747,
|
||
|
|
"logps/chosen": -61.2158203125,
|
||
|
|
"logps/rejected": -82.98789978027344,
|
||
|
|
"loss": 0.3677,
|
||
|
|
"rewards/accuracies": 0.831250011920929,
|
||
|
|
"rewards/chosen": 0.1341857612133026,
|
||
|
|
"rewards/margins": 1.88623046875,
|
||
|
|
"rewards/rejected": -1.752044677734375,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40327662255828606,
|
||
|
|
"eval_logits/chosen": -0.911089301109314,
|
||
|
|
"eval_logits/rejected": -0.9992658495903015,
|
||
|
|
"eval_logps/chosen": -60.45991897583008,
|
||
|
|
"eval_logps/rejected": -83.99901580810547,
|
||
|
|
"eval_loss": 0.2951066493988037,
|
||
|
|
"eval_rewards/accuracies": 0.8886227607727051,
|
||
|
|
"eval_rewards/chosen": 0.1546352505683899,
|
||
|
|
"eval_rewards/margins": 2.0157766342163086,
|
||
|
|
"eval_rewards/rejected": -1.861141324043274,
|
||
|
|
"eval_runtime": 71.0897,
|
||
|
|
"eval_samples_per_second": 23.491,
|
||
|
|
"eval_steps_per_second": 23.491,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40831758034026466,
|
||
|
|
"grad_norm": 16.625,
|
||
|
|
"learning_rate": 3.2033805888767715e-07,
|
||
|
|
"logits/chosen": -0.9287660717964172,
|
||
|
|
"logits/rejected": -1.0095508098602295,
|
||
|
|
"logps/chosen": -60.53706741333008,
|
||
|
|
"logps/rejected": -83.99447631835938,
|
||
|
|
"loss": 0.2838,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.1390010416507721,
|
||
|
|
"rewards/margins": 1.9923112392425537,
|
||
|
|
"rewards/rejected": -1.8533103466033936,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4133585381222432,
|
||
|
|
"grad_norm": 33.0,
|
||
|
|
"learning_rate": 3.1761177753544164e-07,
|
||
|
|
"logits/chosen": -0.9320980310440063,
|
||
|
|
"logits/rejected": -1.0176376104354858,
|
||
|
|
"logps/chosen": -59.87189483642578,
|
||
|
|
"logps/rejected": -84.4596176147461,
|
||
|
|
"loss": 0.2483,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.1814599335193634,
|
||
|
|
"rewards/margins": 2.1097805500030518,
|
||
|
|
"rewards/rejected": -1.9283206462860107,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4183994959042218,
|
||
|
|
"grad_norm": 12.9375,
|
||
|
|
"learning_rate": 3.148854961832061e-07,
|
||
|
|
"logits/chosen": -0.9213443994522095,
|
||
|
|
"logits/rejected": -1.0197076797485352,
|
||
|
|
"logps/chosen": -59.96300506591797,
|
||
|
|
"logps/rejected": -84.28089141845703,
|
||
|
|
"loss": 0.2707,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.1989632546901703,
|
||
|
|
"rewards/margins": 2.1028213500976562,
|
||
|
|
"rewards/rejected": -1.9038581848144531,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42344045368620037,
|
||
|
|
"grad_norm": 5.90625,
|
||
|
|
"learning_rate": 3.121592148309705e-07,
|
||
|
|
"logits/chosen": -0.914442241191864,
|
||
|
|
"logits/rejected": -0.9979844093322754,
|
||
|
|
"logps/chosen": -59.945274353027344,
|
||
|
|
"logps/rejected": -83.28790283203125,
|
||
|
|
"loss": 0.2793,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 0.17169013619422913,
|
||
|
|
"rewards/margins": 2.031377077102661,
|
||
|
|
"rewards/rejected": -1.8596868515014648,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.428481411468179,
|
||
|
|
"grad_norm": 13.4375,
|
||
|
|
"learning_rate": 3.09432933478735e-07,
|
||
|
|
"logits/chosen": -0.9071733355522156,
|
||
|
|
"logits/rejected": -1.0141820907592773,
|
||
|
|
"logps/chosen": -60.1943244934082,
|
||
|
|
"logps/rejected": -84.26617431640625,
|
||
|
|
"loss": 0.2751,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.1853206306695938,
|
||
|
|
"rewards/margins": 2.0591628551483154,
|
||
|
|
"rewards/rejected": -1.8738422393798828,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4335223692501575,
|
||
|
|
"grad_norm": 12.875,
|
||
|
|
"learning_rate": 3.0670665212649945e-07,
|
||
|
|
"logits/chosen": -0.9004698991775513,
|
||
|
|
"logits/rejected": -0.9726663827896118,
|
||
|
|
"logps/chosen": -60.701194763183594,
|
||
|
|
"logps/rejected": -83.21146392822266,
|
||
|
|
"loss": 0.3464,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 0.12978795170783997,
|
||
|
|
"rewards/margins": 1.9474796056747437,
|
||
|
|
"rewards/rejected": -1.8176918029785156,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.43856332703213613,
|
||
|
|
"grad_norm": 13.125,
|
||
|
|
"learning_rate": 3.0398037077426394e-07,
|
||
|
|
"logits/chosen": -0.8763822317123413,
|
||
|
|
"logits/rejected": -0.9599603414535522,
|
||
|
|
"logps/chosen": -60.947471618652344,
|
||
|
|
"logps/rejected": -84.69038391113281,
|
||
|
|
"loss": 0.3711,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": 0.12584984302520752,
|
||
|
|
"rewards/margins": 2.0357413291931152,
|
||
|
|
"rewards/rejected": -1.9098914861679077,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4436042848141147,
|
||
|
|
"grad_norm": 12.0,
|
||
|
|
"learning_rate": 3.0125408942202833e-07,
|
||
|
|
"logits/chosen": -0.9101651906967163,
|
||
|
|
"logits/rejected": -1.014981985092163,
|
||
|
|
"logps/chosen": -59.863014221191406,
|
||
|
|
"logps/rejected": -83.40458679199219,
|
||
|
|
"loss": 0.3145,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.2130097895860672,
|
||
|
|
"rewards/margins": 2.023256301879883,
|
||
|
|
"rewards/rejected": -1.810246467590332,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44864524259609323,
|
||
|
|
"grad_norm": 8.5,
|
||
|
|
"learning_rate": 2.9852780806979277e-07,
|
||
|
|
"logits/chosen": -0.8842908143997192,
|
||
|
|
"logits/rejected": -1.0119690895080566,
|
||
|
|
"logps/chosen": -58.0797233581543,
|
||
|
|
"logps/rejected": -84.85250091552734,
|
||
|
|
"loss": 0.1971,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 0.34588780999183655,
|
||
|
|
"rewards/margins": 2.315692186355591,
|
||
|
|
"rewards/rejected": -1.9698044061660767,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45368620037807184,
|
||
|
|
"grad_norm": 21.125,
|
||
|
|
"learning_rate": 2.9580152671755726e-07,
|
||
|
|
"logits/chosen": -0.861519455909729,
|
||
|
|
"logits/rejected": -0.9699891805648804,
|
||
|
|
"logps/chosen": -58.99384689331055,
|
||
|
|
"logps/rejected": -84.5435791015625,
|
||
|
|
"loss": 0.2497,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.2806755006313324,
|
||
|
|
"rewards/margins": 2.2056772708892822,
|
||
|
|
"rewards/rejected": -1.925001859664917,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45368620037807184,
|
||
|
|
"eval_logits/chosen": -0.9106476902961731,
|
||
|
|
"eval_logits/rejected": -1.0023449659347534,
|
||
|
|
"eval_logps/chosen": -60.15101623535156,
|
||
|
|
"eval_logps/rejected": -84.17161560058594,
|
||
|
|
"eval_loss": 0.29339829087257385,
|
||
|
|
"eval_rewards/accuracies": 0.886227548122406,
|
||
|
|
"eval_rewards/chosen": 0.185525581240654,
|
||
|
|
"eval_rewards/margins": 2.0639255046844482,
|
||
|
|
"eval_rewards/rejected": -1.8783999681472778,
|
||
|
|
"eval_runtime": 70.7305,
|
||
|
|
"eval_samples_per_second": 23.611,
|
||
|
|
"eval_steps_per_second": 23.611,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4587271581600504,
|
||
|
|
"grad_norm": 21.5,
|
||
|
|
"learning_rate": 2.930752453653217e-07,
|
||
|
|
"logits/chosen": -0.8937481641769409,
|
||
|
|
"logits/rejected": -0.9957500696182251,
|
||
|
|
"logps/chosen": -59.79345703125,
|
||
|
|
"logps/rejected": -83.85224914550781,
|
||
|
|
"loss": 0.3282,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": 0.231471985578537,
|
||
|
|
"rewards/margins": 2.0565638542175293,
|
||
|
|
"rewards/rejected": -1.8250917196273804,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.463768115942029,
|
||
|
|
"grad_norm": 18.75,
|
||
|
|
"learning_rate": 2.9034896401308613e-07,
|
||
|
|
"logits/chosen": -0.8847681283950806,
|
||
|
|
"logits/rejected": -0.9873048663139343,
|
||
|
|
"logps/chosen": -59.479347229003906,
|
||
|
|
"logps/rejected": -86.35008239746094,
|
||
|
|
"loss": 0.2292,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.2320963442325592,
|
||
|
|
"rewards/margins": 2.310370683670044,
|
||
|
|
"rewards/rejected": -2.0782742500305176,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46880907372400754,
|
||
|
|
"grad_norm": 22.25,
|
||
|
|
"learning_rate": 2.876226826608506e-07,
|
||
|
|
"logits/chosen": -0.915035605430603,
|
||
|
|
"logits/rejected": -1.0067721605300903,
|
||
|
|
"logps/chosen": -59.74500274658203,
|
||
|
|
"logps/rejected": -82.90184020996094,
|
||
|
|
"loss": 0.3143,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.18378940224647522,
|
||
|
|
"rewards/margins": 1.998040795326233,
|
||
|
|
"rewards/rejected": -1.814251184463501,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47385003150598615,
|
||
|
|
"grad_norm": 33.75,
|
||
|
|
"learning_rate": 2.84896401308615e-07,
|
||
|
|
"logits/chosen": -0.9178056716918945,
|
||
|
|
"logits/rejected": -1.0208479166030884,
|
||
|
|
"logps/chosen": -59.2294807434082,
|
||
|
|
"logps/rejected": -84.78729248046875,
|
||
|
|
"loss": 0.2393,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.25158706307411194,
|
||
|
|
"rewards/margins": 2.1938788890838623,
|
||
|
|
"rewards/rejected": -1.9422919750213623,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4788909892879647,
|
||
|
|
"grad_norm": 34.75,
|
||
|
|
"learning_rate": 2.8217011995637945e-07,
|
||
|
|
"logits/chosen": -0.8805239796638489,
|
||
|
|
"logits/rejected": -0.9753271341323853,
|
||
|
|
"logps/chosen": -59.95941162109375,
|
||
|
|
"logps/rejected": -84.86714172363281,
|
||
|
|
"loss": 0.2666,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.19955232739448547,
|
||
|
|
"rewards/margins": 2.1361844539642334,
|
||
|
|
"rewards/rejected": -1.936632513999939,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4839319470699433,
|
||
|
|
"grad_norm": 23.5,
|
||
|
|
"learning_rate": 2.7944383860414394e-07,
|
||
|
|
"logits/chosen": -0.8861673474311829,
|
||
|
|
"logits/rejected": -0.9975956082344055,
|
||
|
|
"logps/chosen": -59.12894821166992,
|
||
|
|
"logps/rejected": -85.26177978515625,
|
||
|
|
"loss": 0.2303,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.2717575132846832,
|
||
|
|
"rewards/margins": 2.239091396331787,
|
||
|
|
"rewards/rejected": -1.9673335552215576,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48897290485192185,
|
||
|
|
"grad_norm": 19.625,
|
||
|
|
"learning_rate": 2.767175572519084e-07,
|
||
|
|
"logits/chosen": -0.9173597097396851,
|
||
|
|
"logits/rejected": -1.0085917711257935,
|
||
|
|
"logps/chosen": -59.86204147338867,
|
||
|
|
"logps/rejected": -84.22465515136719,
|
||
|
|
"loss": 0.2877,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 0.20364825427532196,
|
||
|
|
"rewards/margins": 2.122812509536743,
|
||
|
|
"rewards/rejected": -1.9191640615463257,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49401386263390046,
|
||
|
|
"grad_norm": 20.125,
|
||
|
|
"learning_rate": 2.7399127589967287e-07,
|
||
|
|
"logits/chosen": -0.8748540878295898,
|
||
|
|
"logits/rejected": -0.9848943948745728,
|
||
|
|
"logps/chosen": -58.594139099121094,
|
||
|
|
"logps/rejected": -84.79752349853516,
|
||
|
|
"loss": 0.2749,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.31691378355026245,
|
||
|
|
"rewards/margins": 2.249189615249634,
|
||
|
|
"rewards/rejected": -1.9322757720947266,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.499054820415879,
|
||
|
|
"grad_norm": 11.5625,
|
||
|
|
"learning_rate": 2.712649945474373e-07,
|
||
|
|
"logits/chosen": -0.8857355117797852,
|
||
|
|
"logits/rejected": -0.9768050909042358,
|
||
|
|
"logps/chosen": -59.4735107421875,
|
||
|
|
"logps/rejected": -84.06812286376953,
|
||
|
|
"loss": 0.2732,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.2545910179615021,
|
||
|
|
"rewards/margins": 2.087873935699463,
|
||
|
|
"rewards/rejected": -1.8332828283309937,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5040957781978576,
|
||
|
|
"grad_norm": 35.0,
|
||
|
|
"learning_rate": 2.685387131952017e-07,
|
||
|
|
"logits/chosen": -0.8867548108100891,
|
||
|
|
"logits/rejected": -1.0059692859649658,
|
||
|
|
"logps/chosen": -59.520362854003906,
|
||
|
|
"logps/rejected": -84.2868881225586,
|
||
|
|
"loss": 0.282,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.25644558668136597,
|
||
|
|
"rewards/margins": 2.160256862640381,
|
||
|
|
"rewards/rejected": -1.9038112163543701,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5040957781978576,
|
||
|
|
"eval_logits/chosen": -0.9006710052490234,
|
||
|
|
"eval_logits/rejected": -0.9953308701515198,
|
||
|
|
"eval_logps/chosen": -59.89181900024414,
|
||
|
|
"eval_logps/rejected": -84.33870697021484,
|
||
|
|
"eval_loss": 0.29226598143577576,
|
||
|
|
"eval_rewards/accuracies": 0.8874251246452332,
|
||
|
|
"eval_rewards/chosen": 0.21144606173038483,
|
||
|
|
"eval_rewards/margins": 2.106555700302124,
|
||
|
|
"eval_rewards/rejected": -1.8951095342636108,
|
||
|
|
"eval_runtime": 71.1587,
|
||
|
|
"eval_samples_per_second": 23.469,
|
||
|
|
"eval_steps_per_second": 23.469,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5091367359798362,
|
||
|
|
"grad_norm": 13.375,
|
||
|
|
"learning_rate": 2.658124318429662e-07,
|
||
|
|
"logits/chosen": -0.8491979837417603,
|
||
|
|
"logits/rejected": -0.9555536508560181,
|
||
|
|
"logps/chosen": -59.57502365112305,
|
||
|
|
"logps/rejected": -83.54940795898438,
|
||
|
|
"loss": 0.3238,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 0.27595460414886475,
|
||
|
|
"rewards/margins": 2.066524028778076,
|
||
|
|
"rewards/rejected": -1.7905696630477905,
|
||
|
|
"step": 1010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5141776937618148,
|
||
|
|
"grad_norm": 14.375,
|
||
|
|
"learning_rate": 2.630861504907306e-07,
|
||
|
|
"logits/chosen": -0.871178150177002,
|
||
|
|
"logits/rejected": -0.9892231225967407,
|
||
|
|
"logps/chosen": -59.257232666015625,
|
||
|
|
"logps/rejected": -86.75993347167969,
|
||
|
|
"loss": 0.1845,
|
||
|
|
"rewards/accuracies": 0.9437500238418579,
|
||
|
|
"rewards/chosen": 0.26664021611213684,
|
||
|
|
"rewards/margins": 2.3842711448669434,
|
||
|
|
"rewards/rejected": -2.11763072013855,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5192186515437933,
|
||
|
|
"grad_norm": 8.1875,
|
||
|
|
"learning_rate": 2.6035986913849506e-07,
|
||
|
|
"logits/chosen": -0.9025937914848328,
|
||
|
|
"logits/rejected": -0.9637918472290039,
|
||
|
|
"logps/chosen": -59.708038330078125,
|
||
|
|
"logps/rejected": -83.67890930175781,
|
||
|
|
"loss": 0.3105,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.22001886367797852,
|
||
|
|
"rewards/margins": 2.053025722503662,
|
||
|
|
"rewards/rejected": -1.8330070972442627,
|
||
|
|
"step": 1030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5242596093257719,
|
||
|
|
"grad_norm": 27.625,
|
||
|
|
"learning_rate": 2.5763358778625955e-07,
|
||
|
|
"logits/chosen": -0.8830963373184204,
|
||
|
|
"logits/rejected": -0.9901777505874634,
|
||
|
|
"logps/chosen": -58.61798095703125,
|
||
|
|
"logps/rejected": -84.18465423583984,
|
||
|
|
"loss": 0.2793,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.33424392342567444,
|
||
|
|
"rewards/margins": 2.2203192710876465,
|
||
|
|
"rewards/rejected": -1.8860752582550049,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5293005671077504,
|
||
|
|
"grad_norm": 11.875,
|
||
|
|
"learning_rate": 2.54907306434024e-07,
|
||
|
|
"logits/chosen": -0.8822509050369263,
|
||
|
|
"logits/rejected": -1.008209466934204,
|
||
|
|
"logps/chosen": -60.164794921875,
|
||
|
|
"logps/rejected": -84.7709732055664,
|
||
|
|
"loss": 0.2963,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 0.22792068123817444,
|
||
|
|
"rewards/margins": 2.1442337036132812,
|
||
|
|
"rewards/rejected": -1.91631281375885,
|
||
|
|
"step": 1050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5343415248897291,
|
||
|
|
"grad_norm": 7.46875,
|
||
|
|
"learning_rate": 2.521810250817885e-07,
|
||
|
|
"logits/chosen": -0.8839899897575378,
|
||
|
|
"logits/rejected": -1.0129756927490234,
|
||
|
|
"logps/chosen": -59.81916046142578,
|
||
|
|
"logps/rejected": -85.43331909179688,
|
||
|
|
"loss": 0.2275,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.21425476670265198,
|
||
|
|
"rewards/margins": 2.2582240104675293,
|
||
|
|
"rewards/rejected": -2.04396915435791,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5393824826717076,
|
||
|
|
"grad_norm": 15.5625,
|
||
|
|
"learning_rate": 2.4945474372955287e-07,
|
||
|
|
"logits/chosen": -0.82720947265625,
|
||
|
|
"logits/rejected": -0.907505989074707,
|
||
|
|
"logps/chosen": -59.22252655029297,
|
||
|
|
"logps/rejected": -83.96528625488281,
|
||
|
|
"loss": 0.2807,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.2757384181022644,
|
||
|
|
"rewards/margins": 2.1067721843719482,
|
||
|
|
"rewards/rejected": -1.8310333490371704,
|
||
|
|
"step": 1070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5444234404536862,
|
||
|
|
"grad_norm": 26.75,
|
||
|
|
"learning_rate": 2.467284623773173e-07,
|
||
|
|
"logits/chosen": -0.8436921834945679,
|
||
|
|
"logits/rejected": -0.942459225654602,
|
||
|
|
"logps/chosen": -59.066001892089844,
|
||
|
|
"logps/rejected": -84.55767059326172,
|
||
|
|
"loss": 0.2846,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.2789579927921295,
|
||
|
|
"rewards/margins": 2.1750502586364746,
|
||
|
|
"rewards/rejected": -1.896092176437378,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5494643982356647,
|
||
|
|
"grad_norm": 26.75,
|
||
|
|
"learning_rate": 2.440021810250818e-07,
|
||
|
|
"logits/chosen": -0.9082972407341003,
|
||
|
|
"logits/rejected": -1.0078270435333252,
|
||
|
|
"logps/chosen": -58.6378059387207,
|
||
|
|
"logps/rejected": -86.2591323852539,
|
||
|
|
"loss": 0.224,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.31546300649642944,
|
||
|
|
"rewards/margins": 2.3998687267303467,
|
||
|
|
"rewards/rejected": -2.0844056606292725,
|
||
|
|
"step": 1090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5545053560176434,
|
||
|
|
"grad_norm": 21.75,
|
||
|
|
"learning_rate": 2.4127589967284623e-07,
|
||
|
|
"logits/chosen": -0.8725331425666809,
|
||
|
|
"logits/rejected": -0.9712456464767456,
|
||
|
|
"logps/chosen": -58.98591995239258,
|
||
|
|
"logps/rejected": -84.76383972167969,
|
||
|
|
"loss": 0.2557,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.29521211981773376,
|
||
|
|
"rewards/margins": 2.201046943664551,
|
||
|
|
"rewards/rejected": -1.9058347940444946,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5545053560176434,
|
||
|
|
"eval_logits/chosen": -0.9060633182525635,
|
||
|
|
"eval_logits/rejected": -1.0051296949386597,
|
||
|
|
"eval_logps/chosen": -59.636837005615234,
|
||
|
|
"eval_logps/rejected": -84.44316864013672,
|
||
|
|
"eval_loss": 0.2921236455440521,
|
||
|
|
"eval_rewards/accuracies": 0.8874251246452332,
|
||
|
|
"eval_rewards/chosen": 0.23694448173046112,
|
||
|
|
"eval_rewards/margins": 2.142500638961792,
|
||
|
|
"eval_rewards/rejected": -1.9055562019348145,
|
||
|
|
"eval_runtime": 72.2931,
|
||
|
|
"eval_samples_per_second": 23.1,
|
||
|
|
"eval_steps_per_second": 23.1,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5595463137996219,
|
||
|
|
"grad_norm": 19.25,
|
||
|
|
"learning_rate": 2.3854961832061067e-07,
|
||
|
|
"logits/chosen": -0.8947960734367371,
|
||
|
|
"logits/rejected": -1.0010361671447754,
|
||
|
|
"logps/chosen": -58.5375862121582,
|
||
|
|
"logps/rejected": -84.85458374023438,
|
||
|
|
"loss": 0.2887,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.3221047520637512,
|
||
|
|
"rewards/margins": 2.255513906478882,
|
||
|
|
"rewards/rejected": -1.9334090948104858,
|
||
|
|
"step": 1110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5645872715816005,
|
||
|
|
"grad_norm": 12.5625,
|
||
|
|
"learning_rate": 2.358233369683751e-07,
|
||
|
|
"logits/chosen": -0.8644863367080688,
|
||
|
|
"logits/rejected": -0.971636950969696,
|
||
|
|
"logps/chosen": -58.843421936035156,
|
||
|
|
"logps/rejected": -84.93423461914062,
|
||
|
|
"loss": 0.2627,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.3323742747306824,
|
||
|
|
"rewards/margins": 2.243507146835327,
|
||
|
|
"rewards/rejected": -1.9111328125,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.569628229363579,
|
||
|
|
"grad_norm": 16.25,
|
||
|
|
"learning_rate": 2.3309705561613957e-07,
|
||
|
|
"logits/chosen": -0.8721977472305298,
|
||
|
|
"logits/rejected": -0.9687450528144836,
|
||
|
|
"logps/chosen": -59.44976806640625,
|
||
|
|
"logps/rejected": -82.7881851196289,
|
||
|
|
"loss": 0.3487,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": 0.25436902046203613,
|
||
|
|
"rewards/margins": 2.020705461502075,
|
||
|
|
"rewards/rejected": -1.766336441040039,
|
||
|
|
"step": 1130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5746691871455577,
|
||
|
|
"grad_norm": 22.25,
|
||
|
|
"learning_rate": 2.3037077426390404e-07,
|
||
|
|
"logits/chosen": -0.889614462852478,
|
||
|
|
"logits/rejected": -1.0026637315750122,
|
||
|
|
"logps/chosen": -58.558433532714844,
|
||
|
|
"logps/rejected": -84.0979232788086,
|
||
|
|
"loss": 0.2587,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.33784663677215576,
|
||
|
|
"rewards/margins": 2.1925177574157715,
|
||
|
|
"rewards/rejected": -1.8546711206436157,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5797101449275363,
|
||
|
|
"grad_norm": 27.5,
|
||
|
|
"learning_rate": 2.2764449291166848e-07,
|
||
|
|
"logits/chosen": -0.9427415132522583,
|
||
|
|
"logits/rejected": -1.0442800521850586,
|
||
|
|
"logps/chosen": -58.565757751464844,
|
||
|
|
"logps/rejected": -84.64384460449219,
|
||
|
|
"loss": 0.2617,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.3408879339694977,
|
||
|
|
"rewards/margins": 2.2868194580078125,
|
||
|
|
"rewards/rejected": -1.9459314346313477,
|
||
|
|
"step": 1150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5847511027095148,
|
||
|
|
"grad_norm": 20.625,
|
||
|
|
"learning_rate": 2.2491821155943292e-07,
|
||
|
|
"logits/chosen": -0.8788965344429016,
|
||
|
|
"logits/rejected": -0.9990353584289551,
|
||
|
|
"logps/chosen": -58.735572814941406,
|
||
|
|
"logps/rejected": -84.48644256591797,
|
||
|
|
"loss": 0.2425,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.3037385046482086,
|
||
|
|
"rewards/margins": 2.239537477493286,
|
||
|
|
"rewards/rejected": -1.9357990026474,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5897920604914934,
|
||
|
|
"grad_norm": 20.0,
|
||
|
|
"learning_rate": 2.2219193020719738e-07,
|
||
|
|
"logits/chosen": -0.874742865562439,
|
||
|
|
"logits/rejected": -0.980475902557373,
|
||
|
|
"logps/chosen": -58.799041748046875,
|
||
|
|
"logps/rejected": -85.21891784667969,
|
||
|
|
"loss": 0.2426,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.3289235234260559,
|
||
|
|
"rewards/margins": 2.2946279048919678,
|
||
|
|
"rewards/rejected": -1.965704321861267,
|
||
|
|
"step": 1170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.594833018273472,
|
||
|
|
"grad_norm": 29.5,
|
||
|
|
"learning_rate": 2.1946564885496182e-07,
|
||
|
|
"logits/chosen": -0.875636875629425,
|
||
|
|
"logits/rejected": -0.9794307947158813,
|
||
|
|
"logps/chosen": -59.88359451293945,
|
||
|
|
"logps/rejected": -84.40530395507812,
|
||
|
|
"loss": 0.2653,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.22515201568603516,
|
||
|
|
"rewards/margins": 2.1366682052612305,
|
||
|
|
"rewards/rejected": -1.9115161895751953,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5998739760554506,
|
||
|
|
"grad_norm": 9.875,
|
||
|
|
"learning_rate": 2.1673936750272628e-07,
|
||
|
|
"logits/chosen": -0.8744710683822632,
|
||
|
|
"logits/rejected": -0.9897273182868958,
|
||
|
|
"logps/chosen": -57.4598388671875,
|
||
|
|
"logps/rejected": -86.65351867675781,
|
||
|
|
"loss": 0.1861,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": 0.42013850808143616,
|
||
|
|
"rewards/margins": 2.5188748836517334,
|
||
|
|
"rewards/rejected": -2.098736524581909,
|
||
|
|
"step": 1190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6049149338374291,
|
||
|
|
"grad_norm": 11.1875,
|
||
|
|
"learning_rate": 2.1401308615049072e-07,
|
||
|
|
"logits/chosen": -0.8216146230697632,
|
||
|
|
"logits/rejected": -0.9422334432601929,
|
||
|
|
"logps/chosen": -58.2786979675293,
|
||
|
|
"logps/rejected": -86.90044403076172,
|
||
|
|
"loss": 0.2244,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.3587431311607361,
|
||
|
|
"rewards/margins": 2.450103282928467,
|
||
|
|
"rewards/rejected": -2.091360569000244,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6049149338374291,
|
||
|
|
"eval_logits/chosen": -0.8834900856018066,
|
||
|
|
"eval_logits/rejected": -0.9822787642478943,
|
||
|
|
"eval_logps/chosen": -59.49919891357422,
|
||
|
|
"eval_logps/rejected": -84.61227416992188,
|
||
|
|
"eval_loss": 0.2913074493408203,
|
||
|
|
"eval_rewards/accuracies": 0.8892215490341187,
|
||
|
|
"eval_rewards/chosen": 0.2507072687149048,
|
||
|
|
"eval_rewards/margins": 2.1731746196746826,
|
||
|
|
"eval_rewards/rejected": -1.9224671125411987,
|
||
|
|
"eval_runtime": 72.5811,
|
||
|
|
"eval_samples_per_second": 23.009,
|
||
|
|
"eval_steps_per_second": 23.009,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6099558916194077,
|
||
|
|
"grad_norm": 7.84375,
|
||
|
|
"learning_rate": 2.1128680479825516e-07,
|
||
|
|
"logits/chosen": -0.8642328977584839,
|
||
|
|
"logits/rejected": -0.9527546167373657,
|
||
|
|
"logps/chosen": -59.0882682800293,
|
||
|
|
"logps/rejected": -84.75911712646484,
|
||
|
|
"loss": 0.2518,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.28083863854408264,
|
||
|
|
"rewards/margins": 2.2171006202697754,
|
||
|
|
"rewards/rejected": -1.936261773109436,
|
||
|
|
"step": 1210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6149968494013862,
|
||
|
|
"grad_norm": 7.21875,
|
||
|
|
"learning_rate": 2.0856052344601963e-07,
|
||
|
|
"logits/chosen": -0.8976170420646667,
|
||
|
|
"logits/rejected": -0.9783880114555359,
|
||
|
|
"logps/chosen": -60.150733947753906,
|
||
|
|
"logps/rejected": -84.77664947509766,
|
||
|
|
"loss": 0.3227,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.18833044171333313,
|
||
|
|
"rewards/margins": 2.1451687812805176,
|
||
|
|
"rewards/rejected": -1.9568383693695068,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6200378071833649,
|
||
|
|
"grad_norm": 16.875,
|
||
|
|
"learning_rate": 2.058342420937841e-07,
|
||
|
|
"logits/chosen": -0.8992708325386047,
|
||
|
|
"logits/rejected": -0.9903928637504578,
|
||
|
|
"logps/chosen": -58.91829299926758,
|
||
|
|
"logps/rejected": -84.00004577636719,
|
||
|
|
"loss": 0.3041,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.2912471294403076,
|
||
|
|
"rewards/margins": 2.173732280731201,
|
||
|
|
"rewards/rejected": -1.8824853897094727,
|
||
|
|
"step": 1230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6250787649653434,
|
||
|
|
"grad_norm": 17.125,
|
||
|
|
"learning_rate": 2.031079607415485e-07,
|
||
|
|
"logits/chosen": -0.9080449938774109,
|
||
|
|
"logits/rejected": -1.0125758647918701,
|
||
|
|
"logps/chosen": -59.39638137817383,
|
||
|
|
"logps/rejected": -84.40299224853516,
|
||
|
|
"loss": 0.322,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 0.2707884907722473,
|
||
|
|
"rewards/margins": 2.1304423809051514,
|
||
|
|
"rewards/rejected": -1.8596540689468384,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.630119722747322,
|
||
|
|
"grad_norm": 6.84375,
|
||
|
|
"learning_rate": 2.0038167938931297e-07,
|
||
|
|
"logits/chosen": -0.8706466555595398,
|
||
|
|
"logits/rejected": -0.9584230184555054,
|
||
|
|
"logps/chosen": -58.84540939331055,
|
||
|
|
"logps/rejected": -83.47660064697266,
|
||
|
|
"loss": 0.3036,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.30719193816185,
|
||
|
|
"rewards/margins": 2.1285505294799805,
|
||
|
|
"rewards/rejected": -1.8213586807250977,
|
||
|
|
"step": 1250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6351606805293005,
|
||
|
|
"grad_norm": 25.75,
|
||
|
|
"learning_rate": 1.9765539803707743e-07,
|
||
|
|
"logits/chosen": -0.8904057741165161,
|
||
|
|
"logits/rejected": -0.9874610900878906,
|
||
|
|
"logps/chosen": -59.1779899597168,
|
||
|
|
"logps/rejected": -84.42768859863281,
|
||
|
|
"loss": 0.3073,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.2561061382293701,
|
||
|
|
"rewards/margins": 2.1732170581817627,
|
||
|
|
"rewards/rejected": -1.917110800743103,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6402016383112792,
|
||
|
|
"grad_norm": 16.75,
|
||
|
|
"learning_rate": 1.9492911668484184e-07,
|
||
|
|
"logits/chosen": -0.8631388545036316,
|
||
|
|
"logits/rejected": -0.9788234829902649,
|
||
|
|
"logps/chosen": -58.17034912109375,
|
||
|
|
"logps/rejected": -86.0931167602539,
|
||
|
|
"loss": 0.2073,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 0.3628115653991699,
|
||
|
|
"rewards/margins": 2.3852200508117676,
|
||
|
|
"rewards/rejected": -2.0224084854125977,
|
||
|
|
"step": 1270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6452425960932577,
|
||
|
|
"grad_norm": 7.78125,
|
||
|
|
"learning_rate": 1.922028353326063e-07,
|
||
|
|
"logits/chosen": -0.886549174785614,
|
||
|
|
"logits/rejected": -1.004765510559082,
|
||
|
|
"logps/chosen": -58.592750549316406,
|
||
|
|
"logps/rejected": -84.68618774414062,
|
||
|
|
"loss": 0.2295,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 0.322457492351532,
|
||
|
|
"rewards/margins": 2.2541778087615967,
|
||
|
|
"rewards/rejected": -1.9317200183868408,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6502835538752363,
|
||
|
|
"grad_norm": 13.6875,
|
||
|
|
"learning_rate": 1.8947655398037077e-07,
|
||
|
|
"logits/chosen": -0.8659318685531616,
|
||
|
|
"logits/rejected": -0.9652425646781921,
|
||
|
|
"logps/chosen": -60.613441467285156,
|
||
|
|
"logps/rejected": -84.11669158935547,
|
||
|
|
"loss": 0.3235,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.18255123496055603,
|
||
|
|
"rewards/margins": 2.038862705230713,
|
||
|
|
"rewards/rejected": -1.8563114404678345,
|
||
|
|
"step": 1290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6553245116572148,
|
||
|
|
"grad_norm": 23.375,
|
||
|
|
"learning_rate": 1.8675027262813524e-07,
|
||
|
|
"logits/chosen": -0.8807634115219116,
|
||
|
|
"logits/rejected": -1.0138846635818481,
|
||
|
|
"logps/chosen": -58.674583435058594,
|
||
|
|
"logps/rejected": -85.5340576171875,
|
||
|
|
"loss": 0.2442,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.3219856321811676,
|
||
|
|
"rewards/margins": 2.348407030105591,
|
||
|
|
"rewards/rejected": -2.026421308517456,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6553245116572148,
|
||
|
|
"eval_logits/chosen": -0.9014637470245361,
|
||
|
|
"eval_logits/rejected": -1.0028033256530762,
|
||
|
|
"eval_logps/chosen": -59.48044204711914,
|
||
|
|
"eval_logps/rejected": -84.73458099365234,
|
||
|
|
"eval_loss": 0.29055994749069214,
|
||
|
|
"eval_rewards/accuracies": 0.8856287598609924,
|
||
|
|
"eval_rewards/chosen": 0.25258320569992065,
|
||
|
|
"eval_rewards/margins": 2.1872806549072266,
|
||
|
|
"eval_rewards/rejected": -1.9346975088119507,
|
||
|
|
"eval_runtime": 70.3028,
|
||
|
|
"eval_samples_per_second": 23.754,
|
||
|
|
"eval_steps_per_second": 23.754,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6603654694391935,
|
||
|
|
"grad_norm": 10.0625,
|
||
|
|
"learning_rate": 1.8402399127589965e-07,
|
||
|
|
"logits/chosen": -0.8660598993301392,
|
||
|
|
"logits/rejected": -0.965117335319519,
|
||
|
|
"logps/chosen": -58.19904708862305,
|
||
|
|
"logps/rejected": -85.5027084350586,
|
||
|
|
"loss": 0.2252,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 0.3830675482749939,
|
||
|
|
"rewards/margins": 2.3513522148132324,
|
||
|
|
"rewards/rejected": -1.9682846069335938,
|
||
|
|
"step": 1310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.665406427221172,
|
||
|
|
"grad_norm": 19.75,
|
||
|
|
"learning_rate": 1.8129770992366411e-07,
|
||
|
|
"logits/chosen": -0.8987786173820496,
|
||
|
|
"logits/rejected": -0.9898836016654968,
|
||
|
|
"logps/chosen": -58.44793701171875,
|
||
|
|
"logps/rejected": -85.73442077636719,
|
||
|
|
"loss": 0.2653,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.3300415873527527,
|
||
|
|
"rewards/margins": 2.320068836212158,
|
||
|
|
"rewards/rejected": -1.9900271892547607,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6704473850031506,
|
||
|
|
"grad_norm": 26.625,
|
||
|
|
"learning_rate": 1.7857142857142858e-07,
|
||
|
|
"logits/chosen": -0.8488761186599731,
|
||
|
|
"logits/rejected": -0.9688565135002136,
|
||
|
|
"logps/chosen": -58.56892776489258,
|
||
|
|
"logps/rejected": -84.93611907958984,
|
||
|
|
"loss": 0.2507,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.33988556265830994,
|
||
|
|
"rewards/margins": 2.270322799682617,
|
||
|
|
"rewards/rejected": -1.9304373264312744,
|
||
|
|
"step": 1330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6754883427851291,
|
||
|
|
"grad_norm": 14.25,
|
||
|
|
"learning_rate": 1.7584514721919302e-07,
|
||
|
|
"logits/chosen": -0.8936493992805481,
|
||
|
|
"logits/rejected": -0.9925310015678406,
|
||
|
|
"logps/chosen": -57.687774658203125,
|
||
|
|
"logps/rejected": -86.47679138183594,
|
||
|
|
"loss": 0.1893,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": 0.372401624917984,
|
||
|
|
"rewards/margins": 2.4566822052001953,
|
||
|
|
"rewards/rejected": -2.084280490875244,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6805293005671077,
|
||
|
|
"grad_norm": 12.6875,
|
||
|
|
"learning_rate": 1.7311886586695746e-07,
|
||
|
|
"logits/chosen": -0.8652948141098022,
|
||
|
|
"logits/rejected": -0.9912670254707336,
|
||
|
|
"logps/chosen": -59.039756774902344,
|
||
|
|
"logps/rejected": -84.45045471191406,
|
||
|
|
"loss": 0.2659,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.2832827866077423,
|
||
|
|
"rewards/margins": 2.2053096294403076,
|
||
|
|
"rewards/rejected": -1.9220268726348877,
|
||
|
|
"step": 1350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6855702583490864,
|
||
|
|
"grad_norm": 8.9375,
|
||
|
|
"learning_rate": 1.7039258451472192e-07,
|
||
|
|
"logits/chosen": -0.830104649066925,
|
||
|
|
"logits/rejected": -0.9250701069831848,
|
||
|
|
"logps/chosen": -59.46904754638672,
|
||
|
|
"logps/rejected": -84.41246032714844,
|
||
|
|
"loss": 0.292,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.28448396921157837,
|
||
|
|
"rewards/margins": 2.1490626335144043,
|
||
|
|
"rewards/rejected": -1.8645786046981812,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6906112161310649,
|
||
|
|
"grad_norm": 16.25,
|
||
|
|
"learning_rate": 1.6766630316248636e-07,
|
||
|
|
"logits/chosen": -0.9015452265739441,
|
||
|
|
"logits/rejected": -0.9891805648803711,
|
||
|
|
"logps/chosen": -59.392555236816406,
|
||
|
|
"logps/rejected": -84.06771087646484,
|
||
|
|
"loss": 0.31,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.22625617682933807,
|
||
|
|
"rewards/margins": 2.116712808609009,
|
||
|
|
"rewards/rejected": -1.8904565572738647,
|
||
|
|
"step": 1370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6956521739130435,
|
||
|
|
"grad_norm": 27.25,
|
||
|
|
"learning_rate": 1.6494002181025082e-07,
|
||
|
|
"logits/chosen": -0.8577421307563782,
|
||
|
|
"logits/rejected": -0.9831596612930298,
|
||
|
|
"logps/chosen": -59.06340408325195,
|
||
|
|
"logps/rejected": -84.84831237792969,
|
||
|
|
"loss": 0.2915,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.3013680577278137,
|
||
|
|
"rewards/margins": 2.284956455230713,
|
||
|
|
"rewards/rejected": -1.983588457107544,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.700693131695022,
|
||
|
|
"grad_norm": 28.375,
|
||
|
|
"learning_rate": 1.6221374045801526e-07,
|
||
|
|
"logits/chosen": -0.8520832061767578,
|
||
|
|
"logits/rejected": -0.9697486162185669,
|
||
|
|
"logps/chosen": -59.848876953125,
|
||
|
|
"logps/rejected": -85.40953063964844,
|
||
|
|
"loss": 0.3043,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 0.2575295567512512,
|
||
|
|
"rewards/margins": 2.2461721897125244,
|
||
|
|
"rewards/rejected": -1.9886424541473389,
|
||
|
|
"step": 1390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7057340894770007,
|
||
|
|
"grad_norm": 14.1875,
|
||
|
|
"learning_rate": 1.594874591057797e-07,
|
||
|
|
"logits/chosen": -0.8636363744735718,
|
||
|
|
"logits/rejected": -0.9646850824356079,
|
||
|
|
"logps/chosen": -59.90193557739258,
|
||
|
|
"logps/rejected": -84.7515869140625,
|
||
|
|
"loss": 0.2823,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.22327394783496857,
|
||
|
|
"rewards/margins": 2.1452901363372803,
|
||
|
|
"rewards/rejected": -1.9220161437988281,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7057340894770007,
|
||
|
|
"eval_logits/chosen": -0.8801774382591248,
|
||
|
|
"eval_logits/rejected": -0.9814472794532776,
|
||
|
|
"eval_logps/chosen": -59.3859748840332,
|
||
|
|
"eval_logps/rejected": -84.68141174316406,
|
||
|
|
"eval_loss": 0.29091569781303406,
|
||
|
|
"eval_rewards/accuracies": 0.8892215490341187,
|
||
|
|
"eval_rewards/chosen": 0.2620302140712738,
|
||
|
|
"eval_rewards/margins": 2.191411256790161,
|
||
|
|
"eval_rewards/rejected": -1.9293811321258545,
|
||
|
|
"eval_runtime": 70.8997,
|
||
|
|
"eval_samples_per_second": 23.554,
|
||
|
|
"eval_steps_per_second": 23.554,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7107750472589792,
|
||
|
|
"grad_norm": 15.9375,
|
||
|
|
"learning_rate": 1.5676117775354416e-07,
|
||
|
|
"logits/chosen": -0.9009099006652832,
|
||
|
|
"logits/rejected": -0.9855210185050964,
|
||
|
|
"logps/chosen": -59.35606002807617,
|
||
|
|
"logps/rejected": -83.26178741455078,
|
||
|
|
"loss": 0.291,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.2705304026603699,
|
||
|
|
"rewards/margins": 2.0694515705108643,
|
||
|
|
"rewards/rejected": -1.7989212274551392,
|
||
|
|
"step": 1410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7158160050409578,
|
||
|
|
"grad_norm": 16.375,
|
||
|
|
"learning_rate": 1.540348964013086e-07,
|
||
|
|
"logits/chosen": -0.8992331624031067,
|
||
|
|
"logits/rejected": -0.9897601008415222,
|
||
|
|
"logps/chosen": -58.76881790161133,
|
||
|
|
"logps/rejected": -85.6570816040039,
|
||
|
|
"loss": 0.2559,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.3151175379753113,
|
||
|
|
"rewards/margins": 2.322927951812744,
|
||
|
|
"rewards/rejected": -2.007810354232788,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7208569628229363,
|
||
|
|
"grad_norm": 20.5,
|
||
|
|
"learning_rate": 1.5130861504907304e-07,
|
||
|
|
"logits/chosen": -0.8569045066833496,
|
||
|
|
"logits/rejected": -0.9533321261405945,
|
||
|
|
"logps/chosen": -58.429908752441406,
|
||
|
|
"logps/rejected": -84.6368179321289,
|
||
|
|
"loss": 0.2602,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.35319021344184875,
|
||
|
|
"rewards/margins": 2.2607696056365967,
|
||
|
|
"rewards/rejected": -1.9075794219970703,
|
||
|
|
"step": 1430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.725897920604915,
|
||
|
|
"grad_norm": 18.5,
|
||
|
|
"learning_rate": 1.485823336968375e-07,
|
||
|
|
"logits/chosen": -0.8877646327018738,
|
||
|
|
"logits/rejected": -0.9839455485343933,
|
||
|
|
"logps/chosen": -59.098785400390625,
|
||
|
|
"logps/rejected": -84.78980255126953,
|
||
|
|
"loss": 0.2648,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.26629897952079773,
|
||
|
|
"rewards/margins": 2.2212090492248535,
|
||
|
|
"rewards/rejected": -1.9549100399017334,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7309388783868935,
|
||
|
|
"grad_norm": 21.75,
|
||
|
|
"learning_rate": 1.4585605234460197e-07,
|
||
|
|
"logits/chosen": -0.8866798281669617,
|
||
|
|
"logits/rejected": -0.9940204620361328,
|
||
|
|
"logps/chosen": -58.56513214111328,
|
||
|
|
"logps/rejected": -85.08773040771484,
|
||
|
|
"loss": 0.2643,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.32275161147117615,
|
||
|
|
"rewards/margins": 2.3079631328582764,
|
||
|
|
"rewards/rejected": -1.9852116107940674,
|
||
|
|
"step": 1450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7359798361688721,
|
||
|
|
"grad_norm": 39.25,
|
||
|
|
"learning_rate": 1.4312977099236638e-07,
|
||
|
|
"logits/chosen": -0.9192501306533813,
|
||
|
|
"logits/rejected": -1.0168657302856445,
|
||
|
|
"logps/chosen": -59.260459899902344,
|
||
|
|
"logps/rejected": -83.24293518066406,
|
||
|
|
"loss": 0.3559,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 0.2648276388645172,
|
||
|
|
"rewards/margins": 2.0748324394226074,
|
||
|
|
"rewards/rejected": -1.8100048303604126,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7410207939508506,
|
||
|
|
"grad_norm": 15.6875,
|
||
|
|
"learning_rate": 1.4040348964013085e-07,
|
||
|
|
"logits/chosen": -0.8752067685127258,
|
||
|
|
"logits/rejected": -0.991533100605011,
|
||
|
|
"logps/chosen": -58.654205322265625,
|
||
|
|
"logps/rejected": -85.35607147216797,
|
||
|
|
"loss": 0.3037,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.31618744134902954,
|
||
|
|
"rewards/margins": 2.2922956943511963,
|
||
|
|
"rewards/rejected": -1.976108193397522,
|
||
|
|
"step": 1470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7460617517328293,
|
||
|
|
"grad_norm": 16.75,
|
||
|
|
"learning_rate": 1.376772082878953e-07,
|
||
|
|
"logits/chosen": -0.8595184087753296,
|
||
|
|
"logits/rejected": -0.9737972021102905,
|
||
|
|
"logps/chosen": -58.14202880859375,
|
||
|
|
"logps/rejected": -84.88631439208984,
|
||
|
|
"loss": 0.2482,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.375678151845932,
|
||
|
|
"rewards/margins": 2.3022727966308594,
|
||
|
|
"rewards/rejected": -1.9265944957733154,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7511027095148078,
|
||
|
|
"grad_norm": 22.5,
|
||
|
|
"learning_rate": 1.3495092693565978e-07,
|
||
|
|
"logits/chosen": -0.8643430471420288,
|
||
|
|
"logits/rejected": -0.9914228320121765,
|
||
|
|
"logps/chosen": -57.81464767456055,
|
||
|
|
"logps/rejected": -85.24693298339844,
|
||
|
|
"loss": 0.2338,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.395480215549469,
|
||
|
|
"rewards/margins": 2.3924388885498047,
|
||
|
|
"rewards/rejected": -1.9969587326049805,
|
||
|
|
"step": 1490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7561436672967864,
|
||
|
|
"grad_norm": 17.5,
|
||
|
|
"learning_rate": 1.322246455834242e-07,
|
||
|
|
"logits/chosen": -0.8798907995223999,
|
||
|
|
"logits/rejected": -0.9934282302856445,
|
||
|
|
"logps/chosen": -58.668060302734375,
|
||
|
|
"logps/rejected": -85.68354034423828,
|
||
|
|
"loss": 0.2743,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.3188169598579407,
|
||
|
|
"rewards/margins": 2.338467836380005,
|
||
|
|
"rewards/rejected": -2.019650936126709,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7561436672967864,
|
||
|
|
"eval_logits/chosen": -0.8938608169555664,
|
||
|
|
"eval_logits/rejected": -0.9942108988761902,
|
||
|
|
"eval_logps/chosen": -59.54497528076172,
|
||
|
|
"eval_logps/rejected": -84.85215759277344,
|
||
|
|
"eval_loss": 0.29076066613197327,
|
||
|
|
"eval_rewards/accuracies": 0.8868263363838196,
|
||
|
|
"eval_rewards/chosen": 0.24613051116466522,
|
||
|
|
"eval_rewards/margins": 2.1925852298736572,
|
||
|
|
"eval_rewards/rejected": -1.946454644203186,
|
||
|
|
"eval_runtime": 72.3546,
|
||
|
|
"eval_samples_per_second": 23.081,
|
||
|
|
"eval_steps_per_second": 23.081,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7611846250787649,
|
||
|
|
"grad_norm": 18.875,
|
||
|
|
"learning_rate": 1.2949836423118865e-07,
|
||
|
|
"logits/chosen": -0.8818739652633667,
|
||
|
|
"logits/rejected": -1.0159088373184204,
|
||
|
|
"logps/chosen": -57.401039123535156,
|
||
|
|
"logps/rejected": -87.64556121826172,
|
||
|
|
"loss": 0.1741,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 0.4094136357307434,
|
||
|
|
"rewards/margins": 2.643099784851074,
|
||
|
|
"rewards/rejected": -2.2336859703063965,
|
||
|
|
"step": 1510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7662255828607435,
|
||
|
|
"grad_norm": 20.625,
|
||
|
|
"learning_rate": 1.2677208287895312e-07,
|
||
|
|
"logits/chosen": -0.8670439720153809,
|
||
|
|
"logits/rejected": -0.9854429364204407,
|
||
|
|
"logps/chosen": -58.494041442871094,
|
||
|
|
"logps/rejected": -85.44778442382812,
|
||
|
|
"loss": 0.2207,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.35261982679367065,
|
||
|
|
"rewards/margins": 2.3209452629089355,
|
||
|
|
"rewards/rejected": -1.9683253765106201,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7712665406427222,
|
||
|
|
"grad_norm": 21.25,
|
||
|
|
"learning_rate": 1.2404580152671756e-07,
|
||
|
|
"logits/chosen": -0.8820363879203796,
|
||
|
|
"logits/rejected": -1.001293420791626,
|
||
|
|
"logps/chosen": -59.6056022644043,
|
||
|
|
"logps/rejected": -85.07635498046875,
|
||
|
|
"loss": 0.3197,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.25649771094322205,
|
||
|
|
"rewards/margins": 2.231887102127075,
|
||
|
|
"rewards/rejected": -1.9753892421722412,
|
||
|
|
"step": 1530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7763074984247007,
|
||
|
|
"grad_norm": 25.25,
|
||
|
|
"learning_rate": 1.21319520174482e-07,
|
||
|
|
"logits/chosen": -0.9083970785140991,
|
||
|
|
"logits/rejected": -1.029203176498413,
|
||
|
|
"logps/chosen": -58.7678108215332,
|
||
|
|
"logps/rejected": -84.92195129394531,
|
||
|
|
"loss": 0.2812,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.2764410376548767,
|
||
|
|
"rewards/margins": 2.2596542835235596,
|
||
|
|
"rewards/rejected": -1.983213186264038,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7813484562066793,
|
||
|
|
"grad_norm": 35.5,
|
||
|
|
"learning_rate": 1.1859323882224645e-07,
|
||
|
|
"logits/chosen": -0.848700225353241,
|
||
|
|
"logits/rejected": -0.9551340341567993,
|
||
|
|
"logps/chosen": -59.077735900878906,
|
||
|
|
"logps/rejected": -86.9412612915039,
|
||
|
|
"loss": 0.2465,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.3101661801338196,
|
||
|
|
"rewards/margins": 2.430529832839966,
|
||
|
|
"rewards/rejected": -2.120363712310791,
|
||
|
|
"step": 1550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7863894139886578,
|
||
|
|
"grad_norm": 26.75,
|
||
|
|
"learning_rate": 1.1586695747001091e-07,
|
||
|
|
"logits/chosen": -0.8691787719726562,
|
||
|
|
"logits/rejected": -0.977543830871582,
|
||
|
|
"logps/chosen": -59.203521728515625,
|
||
|
|
"logps/rejected": -84.72015380859375,
|
||
|
|
"loss": 0.2758,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.28436484932899475,
|
||
|
|
"rewards/margins": 2.2119786739349365,
|
||
|
|
"rewards/rejected": -1.9276138544082642,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7914303717706365,
|
||
|
|
"grad_norm": 14.0625,
|
||
|
|
"learning_rate": 1.1314067611777535e-07,
|
||
|
|
"logits/chosen": -0.8494786024093628,
|
||
|
|
"logits/rejected": -0.9657021760940552,
|
||
|
|
"logps/chosen": -58.790069580078125,
|
||
|
|
"logps/rejected": -85.69227600097656,
|
||
|
|
"loss": 0.2438,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.30834752321243286,
|
||
|
|
"rewards/margins": 2.33400297164917,
|
||
|
|
"rewards/rejected": -2.0256552696228027,
|
||
|
|
"step": 1570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.796471329552615,
|
||
|
|
"grad_norm": 9.9375,
|
||
|
|
"learning_rate": 1.1041439476553979e-07,
|
||
|
|
"logits/chosen": -0.898410975933075,
|
||
|
|
"logits/rejected": -0.9827295541763306,
|
||
|
|
"logps/chosen": -59.0511360168457,
|
||
|
|
"logps/rejected": -84.7381591796875,
|
||
|
|
"loss": 0.3078,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.2760066092014313,
|
||
|
|
"rewards/margins": 2.209929943084717,
|
||
|
|
"rewards/rejected": -1.9339231252670288,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8015122873345936,
|
||
|
|
"grad_norm": 11.375,
|
||
|
|
"learning_rate": 1.0768811341330425e-07,
|
||
|
|
"logits/chosen": -0.8804594874382019,
|
||
|
|
"logits/rejected": -1.0089373588562012,
|
||
|
|
"logps/chosen": -57.877601623535156,
|
||
|
|
"logps/rejected": -86.3949966430664,
|
||
|
|
"loss": 0.2237,
|
||
|
|
"rewards/accuracies": 0.9312499761581421,
|
||
|
|
"rewards/chosen": 0.39419177174568176,
|
||
|
|
"rewards/margins": 2.4698028564453125,
|
||
|
|
"rewards/rejected": -2.075611114501953,
|
||
|
|
"step": 1590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8065532451165721,
|
||
|
|
"grad_norm": 13.5625,
|
||
|
|
"learning_rate": 1.0496183206106869e-07,
|
||
|
|
"logits/chosen": -0.885684609413147,
|
||
|
|
"logits/rejected": -1.007638931274414,
|
||
|
|
"logps/chosen": -57.24601364135742,
|
||
|
|
"logps/rejected": -85.00395202636719,
|
||
|
|
"loss": 0.2164,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.43452557921409607,
|
||
|
|
"rewards/margins": 2.38159441947937,
|
||
|
|
"rewards/rejected": -1.9470688104629517,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8065532451165721,
|
||
|
|
"eval_logits/chosen": -0.8853088617324829,
|
||
|
|
"eval_logits/rejected": -0.9853202700614929,
|
||
|
|
"eval_logps/chosen": -59.518455505371094,
|
||
|
|
"eval_logps/rejected": -84.9808349609375,
|
||
|
|
"eval_loss": 0.2906578481197357,
|
||
|
|
"eval_rewards/accuracies": 0.886227548122406,
|
||
|
|
"eval_rewards/chosen": 0.24878181517124176,
|
||
|
|
"eval_rewards/margins": 2.2081050872802734,
|
||
|
|
"eval_rewards/rejected": -1.959323525428772,
|
||
|
|
"eval_runtime": 72.4075,
|
||
|
|
"eval_samples_per_second": 23.064,
|
||
|
|
"eval_steps_per_second": 23.064,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8115942028985508,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 1.0223555070883315e-07,
|
||
|
|
"logits/chosen": -0.8960712552070618,
|
||
|
|
"logits/rejected": -1.01849365234375,
|
||
|
|
"logps/chosen": -59.022682189941406,
|
||
|
|
"logps/rejected": -85.11215209960938,
|
||
|
|
"loss": 0.234,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.30705201625823975,
|
||
|
|
"rewards/margins": 2.2740187644958496,
|
||
|
|
"rewards/rejected": -1.966966986656189,
|
||
|
|
"step": 1610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8166351606805293,
|
||
|
|
"grad_norm": 15.3125,
|
||
|
|
"learning_rate": 9.950926935659759e-08,
|
||
|
|
"logits/chosen": -0.8351577520370483,
|
||
|
|
"logits/rejected": -0.950478196144104,
|
||
|
|
"logps/chosen": -61.142723083496094,
|
||
|
|
"logps/rejected": -83.6924819946289,
|
||
|
|
"loss": 0.3591,
|
||
|
|
"rewards/accuracies": 0.84375,
|
||
|
|
"rewards/chosen": 0.1616726666688919,
|
||
|
|
"rewards/margins": 1.9606307744979858,
|
||
|
|
"rewards/rejected": -1.7989578247070312,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8216761184625079,
|
||
|
|
"grad_norm": 20.125,
|
||
|
|
"learning_rate": 9.678298800436204e-08,
|
||
|
|
"logits/chosen": -0.8941848874092102,
|
||
|
|
"logits/rejected": -0.9981807470321655,
|
||
|
|
"logps/chosen": -59.347694396972656,
|
||
|
|
"logps/rejected": -83.65950012207031,
|
||
|
|
"loss": 0.3604,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": 0.27368858456611633,
|
||
|
|
"rewards/margins": 2.1373450756073,
|
||
|
|
"rewards/rejected": -1.8636566400527954,
|
||
|
|
"step": 1630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8267170762444864,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 9.40567066521265e-08,
|
||
|
|
"logits/chosen": -0.8934575915336609,
|
||
|
|
"logits/rejected": -1.0223013162612915,
|
||
|
|
"logps/chosen": -58.4257698059082,
|
||
|
|
"logps/rejected": -85.53926086425781,
|
||
|
|
"loss": 0.2539,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.34174391627311707,
|
||
|
|
"rewards/margins": 2.34285569190979,
|
||
|
|
"rewards/rejected": -2.0011115074157715,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.831758034026465,
|
||
|
|
"grad_norm": 36.0,
|
||
|
|
"learning_rate": 9.133042529989095e-08,
|
||
|
|
"logits/chosen": -0.834098219871521,
|
||
|
|
"logits/rejected": -0.95869380235672,
|
||
|
|
"logps/chosen": -60.06525802612305,
|
||
|
|
"logps/rejected": -84.23382568359375,
|
||
|
|
"loss": 0.3107,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.22542810440063477,
|
||
|
|
"rewards/margins": 2.11167573928833,
|
||
|
|
"rewards/rejected": -1.8862476348876953,
|
||
|
|
"step": 1650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8367989918084436,
|
||
|
|
"grad_norm": 13.375,
|
||
|
|
"learning_rate": 8.860414394765539e-08,
|
||
|
|
"logits/chosen": -0.8735988736152649,
|
||
|
|
"logits/rejected": -0.9876992106437683,
|
||
|
|
"logps/chosen": -58.84014892578125,
|
||
|
|
"logps/rejected": -85.5834732055664,
|
||
|
|
"loss": 0.2594,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.31726759672164917,
|
||
|
|
"rewards/margins": 2.3258700370788574,
|
||
|
|
"rewards/rejected": -2.0086026191711426,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8418399495904222,
|
||
|
|
"grad_norm": 32.25,
|
||
|
|
"learning_rate": 8.587786259541985e-08,
|
||
|
|
"logits/chosen": -0.845401406288147,
|
||
|
|
"logits/rejected": -0.9550067782402039,
|
||
|
|
"logps/chosen": -58.90953826904297,
|
||
|
|
"logps/rejected": -85.50616455078125,
|
||
|
|
"loss": 0.293,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 0.3285283148288727,
|
||
|
|
"rewards/margins": 2.3436825275421143,
|
||
|
|
"rewards/rejected": -2.0151543617248535,
|
||
|
|
"step": 1670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8468809073724007,
|
||
|
|
"grad_norm": 29.625,
|
||
|
|
"learning_rate": 8.315158124318429e-08,
|
||
|
|
"logits/chosen": -0.846836268901825,
|
||
|
|
"logits/rejected": -0.9689435958862305,
|
||
|
|
"logps/chosen": -59.2591552734375,
|
||
|
|
"logps/rejected": -84.11135864257812,
|
||
|
|
"loss": 0.3235,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.30486565828323364,
|
||
|
|
"rewards/margins": 2.1752853393554688,
|
||
|
|
"rewards/rejected": -1.8704197406768799,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8519218651543793,
|
||
|
|
"grad_norm": 18.25,
|
||
|
|
"learning_rate": 8.042529989094875e-08,
|
||
|
|
"logits/chosen": -0.8394562005996704,
|
||
|
|
"logits/rejected": -0.9826558828353882,
|
||
|
|
"logps/chosen": -58.2289924621582,
|
||
|
|
"logps/rejected": -85.12105560302734,
|
||
|
|
"loss": 0.2287,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.386637419462204,
|
||
|
|
"rewards/margins": 2.350778341293335,
|
||
|
|
"rewards/rejected": -1.9641408920288086,
|
||
|
|
"step": 1690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.856962822936358,
|
||
|
|
"grad_norm": 14.1875,
|
||
|
|
"learning_rate": 7.769901853871319e-08,
|
||
|
|
"logits/chosen": -0.8795615434646606,
|
||
|
|
"logits/rejected": -0.9935046434402466,
|
||
|
|
"logps/chosen": -56.979454040527344,
|
||
|
|
"logps/rejected": -86.98414611816406,
|
||
|
|
"loss": 0.1638,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 0.47562724351882935,
|
||
|
|
"rewards/margins": 2.5870258808135986,
|
||
|
|
"rewards/rejected": -2.111398220062256,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.856962822936358,
|
||
|
|
"eval_logits/chosen": -0.8943283557891846,
|
||
|
|
"eval_logits/rejected": -0.9974154829978943,
|
||
|
|
"eval_logps/chosen": -59.26322937011719,
|
||
|
|
"eval_logps/rejected": -84.75007629394531,
|
||
|
|
"eval_loss": 0.29021164774894714,
|
||
|
|
"eval_rewards/accuracies": 0.8892215490341187,
|
||
|
|
"eval_rewards/chosen": 0.2743041515350342,
|
||
|
|
"eval_rewards/margins": 2.2105515003204346,
|
||
|
|
"eval_rewards/rejected": -1.9362471103668213,
|
||
|
|
"eval_runtime": 70.6147,
|
||
|
|
"eval_samples_per_second": 23.649,
|
||
|
|
"eval_steps_per_second": 23.649,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8620037807183365,
|
||
|
|
"grad_norm": 31.125,
|
||
|
|
"learning_rate": 7.497273718647764e-08,
|
||
|
|
"logits/chosen": -0.9100006818771362,
|
||
|
|
"logits/rejected": -1.028938889503479,
|
||
|
|
"logps/chosen": -58.05731964111328,
|
||
|
|
"logps/rejected": -84.74702453613281,
|
||
|
|
"loss": 0.2487,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.3694465756416321,
|
||
|
|
"rewards/margins": 2.2923641204833984,
|
||
|
|
"rewards/rejected": -1.9229179620742798,
|
||
|
|
"step": 1710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.867044738500315,
|
||
|
|
"grad_norm": 16.625,
|
||
|
|
"learning_rate": 7.22464558342421e-08,
|
||
|
|
"logits/chosen": -0.8805274963378906,
|
||
|
|
"logits/rejected": -0.9727686047554016,
|
||
|
|
"logps/chosen": -58.530845642089844,
|
||
|
|
"logps/rejected": -84.3791275024414,
|
||
|
|
"loss": 0.2809,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.3172784447669983,
|
||
|
|
"rewards/margins": 2.2292685508728027,
|
||
|
|
"rewards/rejected": -1.9119901657104492,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8720856962822936,
|
||
|
|
"grad_norm": 24.125,
|
||
|
|
"learning_rate": 6.952017448200655e-08,
|
||
|
|
"logits/chosen": -0.903597354888916,
|
||
|
|
"logits/rejected": -1.0219088792800903,
|
||
|
|
"logps/chosen": -58.6904411315918,
|
||
|
|
"logps/rejected": -84.50187683105469,
|
||
|
|
"loss": 0.252,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.3121050298213959,
|
||
|
|
"rewards/margins": 2.2391486167907715,
|
||
|
|
"rewards/rejected": -1.9270436763763428,
|
||
|
|
"step": 1730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8771266540642723,
|
||
|
|
"grad_norm": 8.8125,
|
||
|
|
"learning_rate": 6.679389312977098e-08,
|
||
|
|
"logits/chosen": -0.8927151560783386,
|
||
|
|
"logits/rejected": -0.9953739047050476,
|
||
|
|
"logps/chosen": -59.15825271606445,
|
||
|
|
"logps/rejected": -85.2041244506836,
|
||
|
|
"loss": 0.2347,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 0.29449015855789185,
|
||
|
|
"rewards/margins": 2.263671398162842,
|
||
|
|
"rewards/rejected": -1.9691814184188843,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8821676118462508,
|
||
|
|
"grad_norm": 8.25,
|
||
|
|
"learning_rate": 6.406761177753544e-08,
|
||
|
|
"logits/chosen": -0.857822597026825,
|
||
|
|
"logits/rejected": -0.9733842015266418,
|
||
|
|
"logps/chosen": -58.579246520996094,
|
||
|
|
"logps/rejected": -85.5455551147461,
|
||
|
|
"loss": 0.2629,
|
||
|
|
"rewards/accuracies": 0.893750011920929,
|
||
|
|
"rewards/chosen": 0.3299351930618286,
|
||
|
|
"rewards/margins": 2.326323986053467,
|
||
|
|
"rewards/rejected": -1.9963890314102173,
|
||
|
|
"step": 1750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8872085696282294,
|
||
|
|
"grad_norm": 12.5,
|
||
|
|
"learning_rate": 6.134133042529989e-08,
|
||
|
|
"logits/chosen": -0.9069849848747253,
|
||
|
|
"logits/rejected": -1.0028917789459229,
|
||
|
|
"logps/chosen": -58.58331298828125,
|
||
|
|
"logps/rejected": -86.31389617919922,
|
||
|
|
"loss": 0.2365,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.3112293779850006,
|
||
|
|
"rewards/margins": 2.403074026107788,
|
||
|
|
"rewards/rejected": -2.0918445587158203,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8922495274102079,
|
||
|
|
"grad_norm": 26.375,
|
||
|
|
"learning_rate": 5.861504907306434e-08,
|
||
|
|
"logits/chosen": -0.8625677227973938,
|
||
|
|
"logits/rejected": -0.9746575355529785,
|
||
|
|
"logps/chosen": -57.2651252746582,
|
||
|
|
"logps/rejected": -85.93891906738281,
|
||
|
|
"loss": 0.2094,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.417421817779541,
|
||
|
|
"rewards/margins": 2.4767680168151855,
|
||
|
|
"rewards/rejected": -2.0593464374542236,
|
||
|
|
"step": 1770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8972904851921865,
|
||
|
|
"grad_norm": 29.875,
|
||
|
|
"learning_rate": 5.588876772082879e-08,
|
||
|
|
"logits/chosen": -0.8995459675788879,
|
||
|
|
"logits/rejected": -1.0156245231628418,
|
||
|
|
"logps/chosen": -58.09168243408203,
|
||
|
|
"logps/rejected": -85.68585968017578,
|
||
|
|
"loss": 0.2295,
|
||
|
|
"rewards/accuracies": 0.918749988079071,
|
||
|
|
"rewards/chosen": 0.3603942096233368,
|
||
|
|
"rewards/margins": 2.3807685375213623,
|
||
|
|
"rewards/rejected": -2.020374298095703,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9023314429741651,
|
||
|
|
"grad_norm": 11.75,
|
||
|
|
"learning_rate": 5.316248636859324e-08,
|
||
|
|
"logits/chosen": -0.8375666737556458,
|
||
|
|
"logits/rejected": -0.9661946296691895,
|
||
|
|
"logps/chosen": -59.528541564941406,
|
||
|
|
"logps/rejected": -84.20811462402344,
|
||
|
|
"loss": 0.3174,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.2562711834907532,
|
||
|
|
"rewards/margins": 2.144699811935425,
|
||
|
|
"rewards/rejected": -1.8884284496307373,
|
||
|
|
"step": 1790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9073724007561437,
|
||
|
|
"grad_norm": 26.25,
|
||
|
|
"learning_rate": 5.043620501635769e-08,
|
||
|
|
"logits/chosen": -0.8782273530960083,
|
||
|
|
"logits/rejected": -0.9897629618644714,
|
||
|
|
"logps/chosen": -58.66547393798828,
|
||
|
|
"logps/rejected": -84.6033706665039,
|
||
|
|
"loss": 0.2588,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 0.3361870348453522,
|
||
|
|
"rewards/margins": 2.2399086952209473,
|
||
|
|
"rewards/rejected": -1.903721570968628,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9073724007561437,
|
||
|
|
"eval_logits/chosen": -0.891097903251648,
|
||
|
|
"eval_logits/rejected": -0.9927994608879089,
|
||
|
|
"eval_logps/chosen": -59.322288513183594,
|
||
|
|
"eval_logps/rejected": -84.85601806640625,
|
||
|
|
"eval_loss": 0.2902388572692871,
|
||
|
|
"eval_rewards/accuracies": 0.8886227607727051,
|
||
|
|
"eval_rewards/chosen": 0.2683981657028198,
|
||
|
|
"eval_rewards/margins": 2.215238094329834,
|
||
|
|
"eval_rewards/rejected": -1.9468399286270142,
|
||
|
|
"eval_runtime": 73.6165,
|
||
|
|
"eval_samples_per_second": 22.685,
|
||
|
|
"eval_steps_per_second": 22.685,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9124133585381222,
|
||
|
|
"grad_norm": 23.125,
|
||
|
|
"learning_rate": 4.770992366412214e-08,
|
||
|
|
"logits/chosen": -0.9097870588302612,
|
||
|
|
"logits/rejected": -1.0100575685501099,
|
||
|
|
"logps/chosen": -58.4896354675293,
|
||
|
|
"logps/rejected": -85.04032897949219,
|
||
|
|
"loss": 0.2725,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.3351896405220032,
|
||
|
|
"rewards/margins": 2.258465528488159,
|
||
|
|
"rewards/rejected": -1.9232757091522217,
|
||
|
|
"step": 1810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9174543163201008,
|
||
|
|
"grad_norm": 10.0625,
|
||
|
|
"learning_rate": 4.498364231188658e-08,
|
||
|
|
"logits/chosen": -0.9119001626968384,
|
||
|
|
"logits/rejected": -0.9829280972480774,
|
||
|
|
"logps/chosen": -59.015846252441406,
|
||
|
|
"logps/rejected": -85.16139221191406,
|
||
|
|
"loss": 0.2527,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.29453665018081665,
|
||
|
|
"rewards/margins": 2.282649517059326,
|
||
|
|
"rewards/rejected": -1.9881130456924438,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9224952741020794,
|
||
|
|
"grad_norm": 37.0,
|
||
|
|
"learning_rate": 4.225736095965103e-08,
|
||
|
|
"logits/chosen": -0.8899482488632202,
|
||
|
|
"logits/rejected": -1.003125786781311,
|
||
|
|
"logps/chosen": -58.26552200317383,
|
||
|
|
"logps/rejected": -84.43729400634766,
|
||
|
|
"loss": 0.2777,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.3805728554725647,
|
||
|
|
"rewards/margins": 2.2908310890197754,
|
||
|
|
"rewards/rejected": -1.9102582931518555,
|
||
|
|
"step": 1830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.927536231884058,
|
||
|
|
"grad_norm": 28.5,
|
||
|
|
"learning_rate": 3.953107960741548e-08,
|
||
|
|
"logits/chosen": -0.8962399363517761,
|
||
|
|
"logits/rejected": -1.0039699077606201,
|
||
|
|
"logps/chosen": -59.460975646972656,
|
||
|
|
"logps/rejected": -84.71418762207031,
|
||
|
|
"loss": 0.3002,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": 0.2653732895851135,
|
||
|
|
"rewards/margins": 2.18709135055542,
|
||
|
|
"rewards/rejected": -1.9217180013656616,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9325771896660365,
|
||
|
|
"grad_norm": 25.375,
|
||
|
|
"learning_rate": 3.680479825517993e-08,
|
||
|
|
"logits/chosen": -0.8921709060668945,
|
||
|
|
"logits/rejected": -0.9834814071655273,
|
||
|
|
"logps/chosen": -59.8410530090332,
|
||
|
|
"logps/rejected": -84.02848052978516,
|
||
|
|
"loss": 0.2857,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.2075158655643463,
|
||
|
|
"rewards/margins": 2.0984721183776855,
|
||
|
|
"rewards/rejected": -1.890955924987793,
|
||
|
|
"step": 1850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9376181474480151,
|
||
|
|
"grad_norm": 17.0,
|
||
|
|
"learning_rate": 3.407851690294438e-08,
|
||
|
|
"logits/chosen": -0.8759490847587585,
|
||
|
|
"logits/rejected": -0.9916941523551941,
|
||
|
|
"logps/chosen": -58.60956954956055,
|
||
|
|
"logps/rejected": -84.21783447265625,
|
||
|
|
"loss": 0.2933,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.34195294976234436,
|
||
|
|
"rewards/margins": 2.2317111492156982,
|
||
|
|
"rewards/rejected": -1.8897583484649658,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9426591052299937,
|
||
|
|
"grad_norm": 16.125,
|
||
|
|
"learning_rate": 3.135223555070883e-08,
|
||
|
|
"logits/chosen": -0.8751128315925598,
|
||
|
|
"logits/rejected": -0.9996623992919922,
|
||
|
|
"logps/chosen": -58.18467330932617,
|
||
|
|
"logps/rejected": -85.35218811035156,
|
||
|
|
"loss": 0.2311,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.3810577392578125,
|
||
|
|
"rewards/margins": 2.361508369445801,
|
||
|
|
"rewards/rejected": -1.9804503917694092,
|
||
|
|
"step": 1870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9477000630119723,
|
||
|
|
"grad_norm": 20.625,
|
||
|
|
"learning_rate": 2.862595419847328e-08,
|
||
|
|
"logits/chosen": -0.8567900657653809,
|
||
|
|
"logits/rejected": -0.96808922290802,
|
||
|
|
"logps/chosen": -58.792396545410156,
|
||
|
|
"logps/rejected": -84.65996551513672,
|
||
|
|
"loss": 0.2498,
|
||
|
|
"rewards/accuracies": 0.90625,
|
||
|
|
"rewards/chosen": 0.33555370569229126,
|
||
|
|
"rewards/margins": 2.2720112800598145,
|
||
|
|
"rewards/rejected": -1.9364579916000366,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9527410207939508,
|
||
|
|
"grad_norm": 16.25,
|
||
|
|
"learning_rate": 2.589967284623773e-08,
|
||
|
|
"logits/chosen": -0.8800870776176453,
|
||
|
|
"logits/rejected": -1.0060070753097534,
|
||
|
|
"logps/chosen": -57.40453338623047,
|
||
|
|
"logps/rejected": -85.77619171142578,
|
||
|
|
"loss": 0.2096,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 0.4209045469760895,
|
||
|
|
"rewards/margins": 2.4640016555786133,
|
||
|
|
"rewards/rejected": -2.0430970191955566,
|
||
|
|
"step": 1890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9577819785759294,
|
||
|
|
"grad_norm": 32.75,
|
||
|
|
"learning_rate": 2.317339149400218e-08,
|
||
|
|
"logits/chosen": -0.8548176884651184,
|
||
|
|
"logits/rejected": -0.9700508117675781,
|
||
|
|
"logps/chosen": -59.573455810546875,
|
||
|
|
"logps/rejected": -84.24436950683594,
|
||
|
|
"loss": 0.2916,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": 0.2536848485469818,
|
||
|
|
"rewards/margins": 2.134523391723633,
|
||
|
|
"rewards/rejected": -1.8808386325836182,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9577819785759294,
|
||
|
|
"eval_logits/chosen": -0.8832784295082092,
|
||
|
|
"eval_logits/rejected": -0.9866493344306946,
|
||
|
|
"eval_logps/chosen": -59.40935516357422,
|
||
|
|
"eval_logps/rejected": -84.93498229980469,
|
||
|
|
"eval_loss": 0.2900025248527527,
|
||
|
|
"eval_rewards/accuracies": 0.8886227607727051,
|
||
|
|
"eval_rewards/chosen": 0.25969186425209045,
|
||
|
|
"eval_rewards/margins": 2.2144293785095215,
|
||
|
|
"eval_rewards/rejected": -1.9547375440597534,
|
||
|
|
"eval_runtime": 70.3437,
|
||
|
|
"eval_samples_per_second": 23.741,
|
||
|
|
"eval_steps_per_second": 23.741,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.962822936357908,
|
||
|
|
"grad_norm": 22.125,
|
||
|
|
"learning_rate": 2.044711014176663e-08,
|
||
|
|
"logits/chosen": -0.9094161987304688,
|
||
|
|
"logits/rejected": -1.0066895484924316,
|
||
|
|
"logps/chosen": -58.97810745239258,
|
||
|
|
"logps/rejected": -83.76020812988281,
|
||
|
|
"loss": 0.2996,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 0.2789512574672699,
|
||
|
|
"rewards/margins": 2.176806688308716,
|
||
|
|
"rewards/rejected": -1.897855520248413,
|
||
|
|
"step": 1910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9678638941398866,
|
||
|
|
"grad_norm": 16.75,
|
||
|
|
"learning_rate": 1.772082878953108e-08,
|
||
|
|
"logits/chosen": -0.8881312608718872,
|
||
|
|
"logits/rejected": -0.9979362487792969,
|
||
|
|
"logps/chosen": -58.24678421020508,
|
||
|
|
"logps/rejected": -84.76899719238281,
|
||
|
|
"loss": 0.2573,
|
||
|
|
"rewards/accuracies": 0.8812500238418579,
|
||
|
|
"rewards/chosen": 0.3678635060787201,
|
||
|
|
"rewards/margins": 2.299140453338623,
|
||
|
|
"rewards/rejected": -1.9312770366668701,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9729048519218652,
|
||
|
|
"grad_norm": 10.75,
|
||
|
|
"learning_rate": 1.4994547437295527e-08,
|
||
|
|
"logits/chosen": -0.8843402862548828,
|
||
|
|
"logits/rejected": -0.9742966890335083,
|
||
|
|
"logps/chosen": -59.153114318847656,
|
||
|
|
"logps/rejected": -85.64107513427734,
|
||
|
|
"loss": 0.2745,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.2713843286037445,
|
||
|
|
"rewards/margins": 2.270181179046631,
|
||
|
|
"rewards/rejected": -1.998796820640564,
|
||
|
|
"step": 1930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9779458097038437,
|
||
|
|
"grad_norm": 25.375,
|
||
|
|
"learning_rate": 1.2268266085059978e-08,
|
||
|
|
"logits/chosen": -0.872350811958313,
|
||
|
|
"logits/rejected": -0.9762080907821655,
|
||
|
|
"logps/chosen": -58.68449783325195,
|
||
|
|
"logps/rejected": -84.45391845703125,
|
||
|
|
"loss": 0.2923,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.31912150979042053,
|
||
|
|
"rewards/margins": 2.221708059310913,
|
||
|
|
"rewards/rejected": -1.9025866985321045,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9829867674858223,
|
||
|
|
"grad_norm": 17.875,
|
||
|
|
"learning_rate": 9.541984732824428e-09,
|
||
|
|
"logits/chosen": -0.8793309926986694,
|
||
|
|
"logits/rejected": -0.9880158305168152,
|
||
|
|
"logps/chosen": -58.13665771484375,
|
||
|
|
"logps/rejected": -85.4889144897461,
|
||
|
|
"loss": 0.2236,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 0.3792751729488373,
|
||
|
|
"rewards/margins": 2.364917039871216,
|
||
|
|
"rewards/rejected": -1.9856418371200562,
|
||
|
|
"step": 1950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9880277252678009,
|
||
|
|
"grad_norm": 14.8125,
|
||
|
|
"learning_rate": 6.815703380588876e-09,
|
||
|
|
"logits/chosen": -0.8793843984603882,
|
||
|
|
"logits/rejected": -0.9873378872871399,
|
||
|
|
"logps/chosen": -57.81184005737305,
|
||
|
|
"logps/rejected": -85.71204376220703,
|
||
|
|
"loss": 0.235,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.3983749747276306,
|
||
|
|
"rewards/margins": 2.4253978729248047,
|
||
|
|
"rewards/rejected": -2.0270228385925293,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9930686830497795,
|
||
|
|
"grad_norm": 22.75,
|
||
|
|
"learning_rate": 4.089422028353326e-09,
|
||
|
|
"logits/chosen": -0.8643286824226379,
|
||
|
|
"logits/rejected": -0.9733026623725891,
|
||
|
|
"logps/chosen": -58.93427658081055,
|
||
|
|
"logps/rejected": -84.8313980102539,
|
||
|
|
"loss": 0.2905,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 0.29614943265914917,
|
||
|
|
"rewards/margins": 2.2761871814727783,
|
||
|
|
"rewards/rejected": -1.9800374507904053,
|
||
|
|
"step": 1970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.998109640831758,
|
||
|
|
"grad_norm": 31.75,
|
||
|
|
"learning_rate": 1.3631406761177753e-09,
|
||
|
|
"logits/chosen": -0.8575819134712219,
|
||
|
|
"logits/rejected": -0.9438503980636597,
|
||
|
|
"logps/chosen": -60.40618896484375,
|
||
|
|
"logps/rejected": -82.56617736816406,
|
||
|
|
"loss": 0.3551,
|
||
|
|
"rewards/accuracies": 0.84375,
|
||
|
|
"rewards/chosen": 0.2122369259595871,
|
||
|
|
"rewards/margins": 1.9236053228378296,
|
||
|
|
"rewards/rejected": -1.7113683223724365,
|
||
|
|
"step": 1980
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 1984,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 100,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|