2062 lines
72 KiB
JSON
2062 lines
72 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 2.96,
|
||
|
|
"eval_steps": 100,
|
||
|
|
"global_step": 1000,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0,
|
||
|
|
"eval_entropy": 1.7289148477407603,
|
||
|
|
"eval_logits/chosen": 0.8636333912138532,
|
||
|
|
"eval_logits/rejected": -0.6579137279251289,
|
||
|
|
"eval_logps/chosen": -111.15125904216633,
|
||
|
|
"eval_logps/rejected": -175.04988199514108,
|
||
|
|
"eval_loss": 0.6931471228599548,
|
||
|
|
"eval_mean_token_accuracy": 0.4934375546612106,
|
||
|
|
"eval_num_tokens": 0.0,
|
||
|
|
"eval_rewards/accuracies": 0.0,
|
||
|
|
"eval_rewards/chosen": 0.0,
|
||
|
|
"eval_rewards/margins": 0.0,
|
||
|
|
"eval_rewards/rejected": 0.0,
|
||
|
|
"eval_runtime": 39.0805,
|
||
|
|
"eval_samples_per_second": 7.293,
|
||
|
|
"eval_steps_per_second": 3.659,
|
||
|
|
"step": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.7504533976316452,
|
||
|
|
"epoch": 0.002962962962962963,
|
||
|
|
"grad_norm": 83.5,
|
||
|
|
"learning_rate": 0.0,
|
||
|
|
"logits/chosen": 1.08207520218349,
|
||
|
|
"logits/rejected": -0.5940652974601717,
|
||
|
|
"logps/chosen": -110.63330745697021,
|
||
|
|
"logps/rejected": -182.49396514892578,
|
||
|
|
"loss": 0.6931471824645996,
|
||
|
|
"mean_token_accuracy": 0.5048413500189781,
|
||
|
|
"num_tokens": 10169.0,
|
||
|
|
"rewards/accuracies": 0.0,
|
||
|
|
"rewards/chosen": 0.0,
|
||
|
|
"rewards/margins": 0.0,
|
||
|
|
"rewards/rejected": 0.0,
|
||
|
|
"step": 1
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.7299097461832895,
|
||
|
|
"epoch": 0.02962962962962963,
|
||
|
|
"grad_norm": 29.875,
|
||
|
|
"learning_rate": 4.455445544554456e-06,
|
||
|
|
"logits/chosen": 0.8212555533910425,
|
||
|
|
"logits/rejected": -0.6693587809238846,
|
||
|
|
"logps/chosen": -100.81309244367812,
|
||
|
|
"logps/rejected": -171.8307899898953,
|
||
|
|
"loss": 0.5316412713792589,
|
||
|
|
"mean_token_accuracy": 0.5100594382319186,
|
||
|
|
"num_tokens": 89122.0,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 0.306247040655257,
|
||
|
|
"rewards/margins": 0.41913801974280634,
|
||
|
|
"rewards/rejected": -0.11289098787690616,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.5854810774326324,
|
||
|
|
"epoch": 0.05925925925925926,
|
||
|
|
"grad_norm": 0.07470703125,
|
||
|
|
"learning_rate": 9.405940594059407e-06,
|
||
|
|
"logits/chosen": 1.2175614897456257,
|
||
|
|
"logits/rejected": -0.8402934758508309,
|
||
|
|
"logps/chosen": -77.16593537330627,
|
||
|
|
"logps/rejected": -196.31047840118407,
|
||
|
|
"loss": 0.036623942852020266,
|
||
|
|
"mean_token_accuracy": 0.5996901635080576,
|
||
|
|
"num_tokens": 182773.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.910937874764204,
|
||
|
|
"rewards/margins": 5.368858252465725,
|
||
|
|
"rewards/rejected": -2.4579203579574824,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.4120451480150222,
|
||
|
|
"epoch": 0.08888888888888889,
|
||
|
|
"grad_norm": 0.019287109375,
|
||
|
|
"learning_rate": 1.4356435643564355e-05,
|
||
|
|
"logits/chosen": 2.264547569453187,
|
||
|
|
"logits/rejected": -1.0346384454567032,
|
||
|
|
"logps/chosen": -64.55358953475952,
|
||
|
|
"logps/rejected": -259.64974784851074,
|
||
|
|
"loss": 6.736992509104311e-05,
|
||
|
|
"mean_token_accuracy": 0.6582182548940182,
|
||
|
|
"num_tokens": 274816.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.462539532780648,
|
||
|
|
"rewards/margins": 12.943859684467316,
|
||
|
|
"rewards/rejected": -8.481320106983185,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.355741088092327,
|
||
|
|
"epoch": 0.11851851851851852,
|
||
|
|
"grad_norm": 0.0004863739013671875,
|
||
|
|
"learning_rate": 1.930693069306931e-05,
|
||
|
|
"logits/chosen": 2.3368093516258224,
|
||
|
|
"logits/rejected": -1.142538018090554,
|
||
|
|
"logps/chosen": -61.829122233390805,
|
||
|
|
"logps/rejected": -295.9105268478394,
|
||
|
|
"loss": 3.1130424758885055e-06,
|
||
|
|
"mean_token_accuracy": 0.6669795714318753,
|
||
|
|
"num_tokens": 367268.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.366855585575104,
|
||
|
|
"rewards/margins": 16.085866248607637,
|
||
|
|
"rewards/rejected": -11.719010663032531,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.320019119977951,
|
||
|
|
"epoch": 0.14814814814814814,
|
||
|
|
"grad_norm": 0.0001201629638671875,
|
||
|
|
"learning_rate": 2.4257425742574257e-05,
|
||
|
|
"logits/chosen": 2.315207261423255,
|
||
|
|
"logits/rejected": -1.0571934877438707,
|
||
|
|
"logps/chosen": -63.68361803293228,
|
||
|
|
"logps/rejected": -302.812979888916,
|
||
|
|
"loss": 7.578286749776453e-06,
|
||
|
|
"mean_token_accuracy": 0.6602454461157322,
|
||
|
|
"num_tokens": 461524.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.11230229139328,
|
||
|
|
"rewards/margins": 16.546387827396394,
|
||
|
|
"rewards/rejected": -12.43408569097519,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.326953475177288,
|
||
|
|
"epoch": 0.17777777777777778,
|
||
|
|
"grad_norm": 0.0003871917724609375,
|
||
|
|
"learning_rate": 2.9207920792079208e-05,
|
||
|
|
"logits/chosen": 2.2619415245350254,
|
||
|
|
"logits/rejected": -1.0863394894289136,
|
||
|
|
"logps/chosen": -63.424767637252806,
|
||
|
|
"logps/rejected": -304.6159143447876,
|
||
|
|
"loss": 2.0567724277498202e-06,
|
||
|
|
"mean_token_accuracy": 0.6664863616228104,
|
||
|
|
"num_tokens": 555169.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.521066862344742,
|
||
|
|
"rewards/margins": 17.257024788856505,
|
||
|
|
"rewards/rejected": -12.735957789421082,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3109250903129577,
|
||
|
|
"epoch": 0.2074074074074074,
|
||
|
|
"grad_norm": 1.0609626770019531e-05,
|
||
|
|
"learning_rate": 3.415841584158416e-05,
|
||
|
|
"logits/chosen": 2.3168210798300413,
|
||
|
|
"logits/rejected": -1.1463735020520454,
|
||
|
|
"logps/chosen": -62.93964213132858,
|
||
|
|
"logps/rejected": -301.49362087249756,
|
||
|
|
"loss": 2.6633450761437417e-05,
|
||
|
|
"mean_token_accuracy": 0.6684455543756485,
|
||
|
|
"num_tokens": 646219.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.369684019684792,
|
||
|
|
"rewards/margins": 17.029080402851104,
|
||
|
|
"rewards/rejected": -12.659396553039551,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3183406174182892,
|
||
|
|
"epoch": 0.23703703703703705,
|
||
|
|
"grad_norm": 8.463859558105469e-06,
|
||
|
|
"learning_rate": 3.910891089108911e-05,
|
||
|
|
"logits/chosen": 2.2528845473791295,
|
||
|
|
"logits/rejected": -1.1451393864081993,
|
||
|
|
"logps/chosen": -64.70127091407775,
|
||
|
|
"logps/rejected": -313.31810626983645,
|
||
|
|
"loss": 2.219452881035977e-07,
|
||
|
|
"mean_token_accuracy": 0.6682900048792362,
|
||
|
|
"num_tokens": 741395.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.620437514781952,
|
||
|
|
"rewards/margins": 18.064030134677886,
|
||
|
|
"rewards/rejected": -13.443592536449433,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.306735871732235,
|
||
|
|
"epoch": 0.26666666666666666,
|
||
|
|
"grad_norm": 0.00014209747314453125,
|
||
|
|
"learning_rate": 4.405940594059406e-05,
|
||
|
|
"logits/chosen": 2.2645358825214656,
|
||
|
|
"logits/rejected": -1.1447062023763102,
|
||
|
|
"logps/chosen": -62.47025227546692,
|
||
|
|
"logps/rejected": -309.65871505737306,
|
||
|
|
"loss": 6.8541798100341115e-06,
|
||
|
|
"mean_token_accuracy": 0.6588799960911274,
|
||
|
|
"num_tokens": 833512.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.536789815127849,
|
||
|
|
"rewards/margins": 17.808473789691924,
|
||
|
|
"rewards/rejected": -13.271684014797211,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3041012182831764,
|
||
|
|
"epoch": 0.2962962962962963,
|
||
|
|
"grad_norm": 0.0001277923583984375,
|
||
|
|
"learning_rate": 4.9009900990099014e-05,
|
||
|
|
"logits/chosen": 2.3280876455235577,
|
||
|
|
"logits/rejected": -1.0593703708195146,
|
||
|
|
"logps/chosen": -57.693548345565794,
|
||
|
|
"logps/rejected": -307.1459705352783,
|
||
|
|
"loss": 2.0000723452540113e-06,
|
||
|
|
"mean_token_accuracy": 0.6723956800997257,
|
||
|
|
"num_tokens": 925332.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.472308248281479,
|
||
|
|
"rewards/margins": 17.747703313827515,
|
||
|
|
"rewards/rejected": -13.275395059585572,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2962962962962963,
|
||
|
|
"eval_entropy": 1.3091890520149179,
|
||
|
|
"eval_logits/chosen": 2.2560516885971,
|
||
|
|
"eval_logits/rejected": -1.1380674279781138,
|
||
|
|
"eval_logps/chosen": -66.05370368157233,
|
||
|
|
"eval_logps/rejected": -310.2874222335282,
|
||
|
|
"eval_loss": 3.520939208101481e-05,
|
||
|
|
"eval_mean_token_accuracy": 0.662857598358101,
|
||
|
|
"eval_num_tokens": 925332.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.5097555984983915,
|
||
|
|
"eval_rewards/margins": 18.033509881346376,
|
||
|
|
"eval_rewards/rejected": -13.523754186563558,
|
||
|
|
"eval_runtime": 38.3237,
|
||
|
|
"eval_samples_per_second": 7.437,
|
||
|
|
"eval_steps_per_second": 3.731,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.284974904358387,
|
||
|
|
"epoch": 0.32592592592592595,
|
||
|
|
"grad_norm": 0.000812530517578125,
|
||
|
|
"learning_rate": 4.999050767562379e-05,
|
||
|
|
"logits/chosen": 2.3396127391184693,
|
||
|
|
"logits/rejected": -1.1138358422485608,
|
||
|
|
"logps/chosen": -61.88268647193909,
|
||
|
|
"logps/rejected": -304.68117294311526,
|
||
|
|
"loss": 3.422158624744043e-06,
|
||
|
|
"mean_token_accuracy": 0.6786789156496524,
|
||
|
|
"num_tokens": 1016416.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.462823995947838,
|
||
|
|
"rewards/margins": 17.767807400226594,
|
||
|
|
"rewards/rejected": -13.304983520507813,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2844461277127266,
|
||
|
|
"epoch": 0.35555555555555557,
|
||
|
|
"grad_norm": 0.00010204315185546875,
|
||
|
|
"learning_rate": 4.9951957461646706e-05,
|
||
|
|
"logits/chosen": 2.3448500846553584,
|
||
|
|
"logits/rejected": -1.0939366953296292,
|
||
|
|
"logps/chosen": -61.62471632957458,
|
||
|
|
"logps/rejected": -307.099747467041,
|
||
|
|
"loss": 1.316130146733485e-06,
|
||
|
|
"mean_token_accuracy": 0.6710287682712078,
|
||
|
|
"num_tokens": 1105772.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.635418501496315,
|
||
|
|
"rewards/margins": 18.310668289661407,
|
||
|
|
"rewards/rejected": -13.675249755382538,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2960298083722592,
|
||
|
|
"epoch": 0.3851851851851852,
|
||
|
|
"grad_norm": 0.0009765625,
|
||
|
|
"learning_rate": 4.988380179235843e-05,
|
||
|
|
"logits/chosen": 2.329744072704608,
|
||
|
|
"logits/rejected": -1.0829270884851314,
|
||
|
|
"logps/chosen": -60.90654925107956,
|
||
|
|
"logps/rejected": -320.0984771728516,
|
||
|
|
"loss": 2.281175875396002e-06,
|
||
|
|
"mean_token_accuracy": 0.6639963097870349,
|
||
|
|
"num_tokens": 1198464.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.473316392302513,
|
||
|
|
"rewards/margins": 18.291567063331605,
|
||
|
|
"rewards/rejected": -13.818250668048858,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2743454918265342,
|
||
|
|
"epoch": 0.4148148148148148,
|
||
|
|
"grad_norm": 0.00115966796875,
|
||
|
|
"learning_rate": 4.9786121534345265e-05,
|
||
|
|
"logits/chosen": 2.310538935492521,
|
||
|
|
"logits/rejected": -1.1208351833476473,
|
||
|
|
"logps/chosen": -58.558825969696045,
|
||
|
|
"logps/rejected": -314.1108278274536,
|
||
|
|
"loss": 1.2155161130067428e-06,
|
||
|
|
"mean_token_accuracy": 0.674963940680027,
|
||
|
|
"num_tokens": 1289352.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.255219912528991,
|
||
|
|
"rewards/margins": 18.211954617500304,
|
||
|
|
"rewards/rejected": -13.956734585762025,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2854739636182786,
|
||
|
|
"epoch": 0.4444444444444444,
|
||
|
|
"grad_norm": 0.0001659393310546875,
|
||
|
|
"learning_rate": 4.965903258506806e-05,
|
||
|
|
"logits/chosen": 2.33276765322688,
|
||
|
|
"logits/rejected": -1.1813850704996935,
|
||
|
|
"logps/chosen": -60.029894495010375,
|
||
|
|
"logps/rejected": -315.8014057159424,
|
||
|
|
"loss": 4.05059563490795e-07,
|
||
|
|
"mean_token_accuracy": 0.6699102215468884,
|
||
|
|
"num_tokens": 1380079.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.356227645277977,
|
||
|
|
"rewards/margins": 18.526466822624208,
|
||
|
|
"rewards/rejected": -14.170239269733429,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3013202093541623,
|
||
|
|
"epoch": 0.4740740740740741,
|
||
|
|
"grad_norm": 1.0132789611816406e-05,
|
||
|
|
"learning_rate": 4.950268573535012e-05,
|
||
|
|
"logits/chosen": 2.3245691177602428,
|
||
|
|
"logits/rejected": -1.0899626289102018,
|
||
|
|
"logps/chosen": -62.87379167079926,
|
||
|
|
"logps/rejected": -325.68464183807373,
|
||
|
|
"loss": 2.103474116665893e-07,
|
||
|
|
"mean_token_accuracy": 0.6679917454719544,
|
||
|
|
"num_tokens": 1474262.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.370475701987743,
|
||
|
|
"rewards/margins": 18.631532168388368,
|
||
|
|
"rewards/rejected": -14.261056447029114,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2932921536266804,
|
||
|
|
"epoch": 0.5037037037037037,
|
||
|
|
"grad_norm": 0.00017070770263671875,
|
||
|
|
"learning_rate": 4.93172664904641e-05,
|
||
|
|
"logits/chosen": 2.199208436926322,
|
||
|
|
"logits/rejected": -1.1313941206494675,
|
||
|
|
"logps/chosen": -66.81836426258087,
|
||
|
|
"logps/rejected": -313.0147771835327,
|
||
|
|
"loss": 5.84564577366109e-07,
|
||
|
|
"mean_token_accuracy": 0.6591029770672321,
|
||
|
|
"num_tokens": 1567351.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.2477530300617214,
|
||
|
|
"rewards/margins": 18.021551287174226,
|
||
|
|
"rewards/rejected": -13.773798143863678,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2906752578914165,
|
||
|
|
"epoch": 0.5333333333333333,
|
||
|
|
"grad_norm": 1.1622905731201172e-05,
|
||
|
|
"learning_rate": 4.9102994850030344e-05,
|
||
|
|
"logits/chosen": 2.257452431422654,
|
||
|
|
"logits/rejected": -1.1859710789627558,
|
||
|
|
"logps/chosen": -60.773719620704654,
|
||
|
|
"logps/rejected": -319.68780155181884,
|
||
|
|
"loss": 4.461732260097051e-07,
|
||
|
|
"mean_token_accuracy": 0.6724127769470215,
|
||
|
|
"num_tokens": 1659070.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.278084766864777,
|
||
|
|
"rewards/margins": 18.284436917304994,
|
||
|
|
"rewards/rejected": -14.006352269649506,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3018329843878746,
|
||
|
|
"epoch": 0.562962962962963,
|
||
|
|
"grad_norm": 6.4849853515625e-05,
|
||
|
|
"learning_rate": 4.886012504698769e-05,
|
||
|
|
"logits/chosen": 2.301720358967305,
|
||
|
|
"logits/rejected": -1.0489440936301595,
|
||
|
|
"logps/chosen": -61.81112604141235,
|
||
|
|
"logps/rejected": -315.3358419418335,
|
||
|
|
"loss": 5.486400823428994e-07,
|
||
|
|
"mean_token_accuracy": 0.669282128661871,
|
||
|
|
"num_tokens": 1750181.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.455310028791428,
|
||
|
|
"rewards/margins": 18.440570056438446,
|
||
|
|
"rewards/rejected": -13.985260021686553,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3009084612131119,
|
||
|
|
"epoch": 0.5925925925925926,
|
||
|
|
"grad_norm": 0.0087890625,
|
||
|
|
"learning_rate": 4.858894524594653e-05,
|
||
|
|
"logits/chosen": 2.2154390003090785,
|
||
|
|
"logits/rejected": -1.1232223850457532,
|
||
|
|
"logps/chosen": -66.61707370281219,
|
||
|
|
"logps/rejected": -316.3582260131836,
|
||
|
|
"loss": 5.881736069568433e-06,
|
||
|
|
"mean_token_accuracy": 0.6558836951851845,
|
||
|
|
"num_tokens": 1841453.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.423358170688152,
|
||
|
|
"rewards/margins": 18.573253047466277,
|
||
|
|
"rewards/rejected": -14.149895036220551,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5925925925925926,
|
||
|
|
"eval_entropy": 1.294411522108358,
|
||
|
|
"eval_logits/chosen": 2.25081917507982,
|
||
|
|
"eval_logits/rejected": -1.1530030385301466,
|
||
|
|
"eval_logps/chosen": -66.83867399842589,
|
||
|
|
"eval_logps/rejected": -318.13998082301,
|
||
|
|
"eval_loss": 1.2471568879846018e-05,
|
||
|
|
"eval_mean_token_accuracy": 0.6633061449844521,
|
||
|
|
"eval_num_tokens": 1841453.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.431258590904982,
|
||
|
|
"eval_rewards/margins": 18.740268787304004,
|
||
|
|
"eval_rewards/rejected": -14.309010145547507,
|
||
|
|
"eval_runtime": 38.3878,
|
||
|
|
"eval_samples_per_second": 7.424,
|
||
|
|
"eval_steps_per_second": 3.725,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2944884866476059,
|
||
|
|
"epoch": 0.6222222222222222,
|
||
|
|
"grad_norm": 5.125999450683594e-05,
|
||
|
|
"learning_rate": 4.8289777201281974e-05,
|
||
|
|
"logits/chosen": 2.3454347980352996,
|
||
|
|
"logits/rejected": -1.1855823049416294,
|
||
|
|
"logps/chosen": -64.12132394313812,
|
||
|
|
"logps/rejected": -325.72925930023194,
|
||
|
|
"loss": 2.409943363090861e-07,
|
||
|
|
"mean_token_accuracy": 0.6670842688530684,
|
||
|
|
"num_tokens": 1935659.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.459967152774334,
|
||
|
|
"rewards/margins": 19.128938269615173,
|
||
|
|
"rewards/rejected": -14.668970954418182,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2553015358746051,
|
||
|
|
"epoch": 0.6518518518518519,
|
||
|
|
"grad_norm": 4.76837158203125e-05,
|
||
|
|
"learning_rate": 4.796297587537286e-05,
|
||
|
|
"logits/chosen": 2.308509979398422,
|
||
|
|
"logits/rejected": -1.0750038709449357,
|
||
|
|
"logps/chosen": -58.983665919303895,
|
||
|
|
"logps/rejected": -315.6023241043091,
|
||
|
|
"loss": 3.415452965782606e-07,
|
||
|
|
"mean_token_accuracy": 0.6794103197753429,
|
||
|
|
"num_tokens": 2029268.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.352584506571293,
|
||
|
|
"rewards/margins": 18.749567639827728,
|
||
|
|
"rewards/rejected": -14.396983158588409,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2749639593064785,
|
||
|
|
"epoch": 0.6814814814814815,
|
||
|
|
"grad_norm": 2.7894973754882812e-05,
|
||
|
|
"learning_rate": 4.760892901743944e-05,
|
||
|
|
"logits/chosen": 2.263447201357434,
|
||
|
|
"logits/rejected": -1.0746111454871972,
|
||
|
|
"logps/chosen": -64.98015263080597,
|
||
|
|
"logps/rejected": -320.59440212249757,
|
||
|
|
"loss": 1.3830200259690174e-06,
|
||
|
|
"mean_token_accuracy": 0.6660085044801235,
|
||
|
|
"num_tokens": 2123260.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.317273432016373,
|
||
|
|
"rewards/margins": 18.91294218301773,
|
||
|
|
"rewards/rejected": -14.59566867351532,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2890466079115868,
|
||
|
|
"epoch": 0.7111111111111111,
|
||
|
|
"grad_norm": 2.7894973754882812e-05,
|
||
|
|
"learning_rate": 4.722805670347963e-05,
|
||
|
|
"logits/chosen": 2.3375335604609835,
|
||
|
|
"logits/rejected": -1.1522054103145247,
|
||
|
|
"logps/chosen": -62.949660015106204,
|
||
|
|
"logps/rejected": -325.21918926239016,
|
||
|
|
"loss": 1.8574786508906982e-07,
|
||
|
|
"mean_token_accuracy": 0.6677135609090328,
|
||
|
|
"num_tokens": 2211888.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.302155223488808,
|
||
|
|
"rewards/margins": 19.20953975915909,
|
||
|
|
"rewards/rejected": -14.90738466978073,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.297672052681446,
|
||
|
|
"epoch": 0.7407407407407407,
|
||
|
|
"grad_norm": 1.3649463653564453e-05,
|
||
|
|
"learning_rate": 4.6820810837849536e-05,
|
||
|
|
"logits/chosen": 2.3567186399215143,
|
||
|
|
"logits/rejected": -1.2153430976155177,
|
||
|
|
"logps/chosen": -61.082769107818606,
|
||
|
|
"logps/rejected": -327.5022741317749,
|
||
|
|
"loss": 4.804733634955483e-07,
|
||
|
|
"mean_token_accuracy": 0.6871423445641994,
|
||
|
|
"num_tokens": 2307633.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.415084969997406,
|
||
|
|
"rewards/margins": 19.260318982601166,
|
||
|
|
"rewards/rejected": -14.845234000682831,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.289575246721506,
|
||
|
|
"epoch": 0.7703703703703704,
|
||
|
|
"grad_norm": 0.00013637542724609375,
|
||
|
|
"learning_rate": 4.63876746170797e-05,
|
||
|
|
"logits/chosen": 2.2677862381437905,
|
||
|
|
"logits/rejected": -1.1125468158062497,
|
||
|
|
"logps/chosen": -67.2604984998703,
|
||
|
|
"logps/rejected": -321.5008083343506,
|
||
|
|
"loss": 1.5667228581150994e-07,
|
||
|
|
"mean_token_accuracy": 0.6551826104521752,
|
||
|
|
"num_tokens": 2404339.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.320639500021935,
|
||
|
|
"rewards/margins": 18.89977525472641,
|
||
|
|
"rewards/rejected": -14.57913579940796,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2907186925411225,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 2.5987625122070312e-05,
|
||
|
|
"learning_rate": 4.592916195656322e-05,
|
||
|
|
"logits/chosen": 2.337716368388003,
|
||
|
|
"logits/rejected": -1.1807755553775976,
|
||
|
|
"logps/chosen": -61.246209836006166,
|
||
|
|
"logps/rejected": -321.838525390625,
|
||
|
|
"loss": 3.219545988031314e-07,
|
||
|
|
"mean_token_accuracy": 0.67101164534688,
|
||
|
|
"num_tokens": 2493582.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.55239104628563,
|
||
|
|
"rewards/margins": 19.164004135131837,
|
||
|
|
"rewards/rejected": -14.611613166332244,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2973388336598872,
|
||
|
|
"epoch": 0.8296296296296296,
|
||
|
|
"grad_norm": 1.0728836059570312e-05,
|
||
|
|
"learning_rate": 4.544581688079602e-05,
|
||
|
|
"logits/chosen": 2.2622078397191254,
|
||
|
|
"logits/rejected": -1.1987081179562755,
|
||
|
|
"logps/chosen": -62.545769691467285,
|
||
|
|
"logps/rejected": -328.43379402160645,
|
||
|
|
"loss": 1.6840259604578022e-07,
|
||
|
|
"mean_token_accuracy": 0.6727346412837505,
|
||
|
|
"num_tokens": 2583501.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.605609413981438,
|
||
|
|
"rewards/margins": 19.350289702415466,
|
||
|
|
"rewards/rejected": -14.744680321216583,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.270925548672676,
|
||
|
|
"epoch": 0.8592592592592593,
|
||
|
|
"grad_norm": 1.4722347259521484e-05,
|
||
|
|
"learning_rate": 4.4938212877892725e-05,
|
||
|
|
"logits/chosen": 2.304147609404456,
|
||
|
|
"logits/rejected": -1.1603299402866345,
|
||
|
|
"logps/chosen": -64.19616587162018,
|
||
|
|
"logps/rejected": -321.61998023986814,
|
||
|
|
"loss": 1.092423531190434e-07,
|
||
|
|
"mean_token_accuracy": 0.6694903835654259,
|
||
|
|
"num_tokens": 2673092.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.521212270855903,
|
||
|
|
"rewards/margins": 19.481856608390807,
|
||
|
|
"rewards/rejected": -14.960644149780274,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2587753705680371,
|
||
|
|
"epoch": 0.8888888888888888,
|
||
|
|
"grad_norm": 8.046627044677734e-06,
|
||
|
|
"learning_rate": 4.4406952219143936e-05,
|
||
|
|
"logits/chosen": 2.2447284227823454,
|
||
|
|
"logits/rejected": -1.2047798430182726,
|
||
|
|
"logps/chosen": -63.220883178710935,
|
||
|
|
"logps/rejected": -316.7784074783325,
|
||
|
|
"loss": 9.590096851752606e-07,
|
||
|
|
"mean_token_accuracy": 0.6734147310256958,
|
||
|
|
"num_tokens": 2765264.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.453023025393486,
|
||
|
|
"rewards/margins": 19.15658880472183,
|
||
|
|
"rewards/rejected": -14.703565871715545,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8888888888888888,
|
||
|
|
"eval_entropy": 1.2864730945833913,
|
||
|
|
"eval_logits/chosen": 2.245142647914562,
|
||
|
|
"eval_logits/rejected": -1.1577701959581186,
|
||
|
|
"eval_logps/chosen": -67.51500383123651,
|
||
|
|
"eval_logps/rejected": -323.1507350681545,
|
||
|
|
"eval_loss": 9.1151678134338e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6627988360978506,
|
||
|
|
"eval_num_tokens": 2765264.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.3636256072904684,
|
||
|
|
"eval_rewards/margins": 19.173711116497334,
|
||
|
|
"eval_rewards/rejected": -14.810085516709547,
|
||
|
|
"eval_runtime": 38.408,
|
||
|
|
"eval_samples_per_second": 7.42,
|
||
|
|
"eval_steps_per_second": 3.723,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2768845409154892,
|
||
|
|
"epoch": 0.9185185185185185,
|
||
|
|
"grad_norm": 4.315376281738281e-05,
|
||
|
|
"learning_rate": 4.385266524442241e-05,
|
||
|
|
"logits/chosen": 2.2955878654382667,
|
||
|
|
"logits/rejected": -1.200506409819956,
|
||
|
|
"logps/chosen": -62.47047922611237,
|
||
|
|
"logps/rejected": -329.1678291320801,
|
||
|
|
"loss": 2.0756017420353602e-07,
|
||
|
|
"mean_token_accuracy": 0.6736313477158546,
|
||
|
|
"num_tokens": 2860290.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.335641312599182,
|
||
|
|
"rewards/margins": 19.35282862186432,
|
||
|
|
"rewards/rejected": -15.01718727350235,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.267455793917179,
|
||
|
|
"epoch": 0.9481481481481482,
|
||
|
|
"grad_norm": 2.2649765014648438e-05,
|
||
|
|
"learning_rate": 4.3276009614285825e-05,
|
||
|
|
"logits/chosen": 2.301217628396178,
|
||
|
|
"logits/rejected": -1.1612455940414776,
|
||
|
|
"logps/chosen": -60.544256925582886,
|
||
|
|
"logps/rejected": -325.25582218170166,
|
||
|
|
"loss": 8.637363748675853e-08,
|
||
|
|
"mean_token_accuracy": 0.6695008479058743,
|
||
|
|
"num_tokens": 2951326.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.26083664894104,
|
||
|
|
"rewards/margins": 19.293893361091612,
|
||
|
|
"rewards/rejected": -15.033056676387787,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2802438855171203,
|
||
|
|
"epoch": 0.9777777777777777,
|
||
|
|
"grad_norm": 3.1441450119018555e-06,
|
||
|
|
"learning_rate": 4.267766952966369e-05,
|
||
|
|
"logits/chosen": 2.3043263280692856,
|
||
|
|
"logits/rejected": -1.1247859920954724,
|
||
|
|
"logps/chosen": -65.50963315963745,
|
||
|
|
"logps/rejected": -327.67705821990967,
|
||
|
|
"loss": 1.6503474398632533e-06,
|
||
|
|
"mean_token_accuracy": 0.6599100351333618,
|
||
|
|
"num_tokens": 3045520.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.30358607918024,
|
||
|
|
"rewards/margins": 19.168729376792907,
|
||
|
|
"rewards/rejected": -14.865143322944641,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2861412669482983,
|
||
|
|
"epoch": 1.005925925925926,
|
||
|
|
"grad_norm": 5.793571472167969e-05,
|
||
|
|
"learning_rate": 4.205835492005405e-05,
|
||
|
|
"logits/chosen": 2.319402775016898,
|
||
|
|
"logits/rejected": -1.2147055357233236,
|
||
|
|
"logps/chosen": -67.06965022338063,
|
||
|
|
"logps/rejected": -330.37220262226305,
|
||
|
|
"loss": 3.5803138871415284e-07,
|
||
|
|
"mean_token_accuracy": 0.6624507841311003,
|
||
|
|
"num_tokens": 3130709.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.281106482603048,
|
||
|
|
"rewards/margins": 19.5897334500363,
|
||
|
|
"rewards/rejected": -15.308626877634149,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2823633886873722,
|
||
|
|
"epoch": 1.0355555555555556,
|
||
|
|
"grad_norm": 6.008148193359375e-05,
|
||
|
|
"learning_rate": 4.141880060119336e-05,
|
||
|
|
"logits/chosen": 2.339569692570856,
|
||
|
|
"logits/rejected": -1.147382590046395,
|
||
|
|
"logps/chosen": -60.31965174674988,
|
||
|
|
"logps/rejected": -335.1733335494995,
|
||
|
|
"loss": 1.532446731289383e-07,
|
||
|
|
"mean_token_accuracy": 0.6735855638980865,
|
||
|
|
"num_tokens": 3225369.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.492888362705708,
|
||
|
|
"rewards/margins": 19.827855718135833,
|
||
|
|
"rewards/rejected": -15.334967303276063,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2730201914906503,
|
||
|
|
"epoch": 1.0651851851851852,
|
||
|
|
"grad_norm": 0.0001926422119140625,
|
||
|
|
"learning_rate": 4.075976540319888e-05,
|
||
|
|
"logits/chosen": 2.3003171439882366,
|
||
|
|
"logits/rejected": -1.2015335101578284,
|
||
|
|
"logps/chosen": -60.594508814811704,
|
||
|
|
"logps/rejected": -329.16415615081786,
|
||
|
|
"loss": 4.122233804082498e-07,
|
||
|
|
"mean_token_accuracy": 0.6743721053004265,
|
||
|
|
"num_tokens": 3318818.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.401582342386246,
|
||
|
|
"rewards/margins": 19.628432750701904,
|
||
|
|
"rewards/rejected": -15.22685045003891,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2773501306772232,
|
||
|
|
"epoch": 1.094814814814815,
|
||
|
|
"grad_norm": 0.000591278076171875,
|
||
|
|
"learning_rate": 4.0082031270217976e-05,
|
||
|
|
"logits/chosen": 2.2876778332841776,
|
||
|
|
"logits/rejected": -1.154664545777339,
|
||
|
|
"logps/chosen": -63.76930930614471,
|
||
|
|
"logps/rejected": -319.00229358673096,
|
||
|
|
"loss": 1.1508095667522866e-06,
|
||
|
|
"mean_token_accuracy": 0.6718519743531942,
|
||
|
|
"num_tokens": 3410676.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.197161988914013,
|
||
|
|
"rewards/margins": 19.0069660782814,
|
||
|
|
"rewards/rejected": -14.809803891181947,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2946350619196891,
|
||
|
|
"epoch": 1.1244444444444444,
|
||
|
|
"grad_norm": 0.00022411346435546875,
|
||
|
|
"learning_rate": 3.9386402332652754e-05,
|
||
|
|
"logits/chosen": 2.293869972488772,
|
||
|
|
"logits/rejected": -1.1689247316110847,
|
||
|
|
"logps/chosen": -64.16428699493409,
|
||
|
|
"logps/rejected": -339.3799625396729,
|
||
|
|
"loss": 1.2371945103950566e-07,
|
||
|
|
"mean_token_accuracy": 0.664111914485693,
|
||
|
|
"num_tokens": 3504356.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.358540344238281,
|
||
|
|
"rewards/margins": 19.867380928993224,
|
||
|
|
"rewards/rejected": -15.508840537071228,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2926758706569672,
|
||
|
|
"epoch": 1.154074074074074,
|
||
|
|
"grad_norm": 3.6954879760742188e-06,
|
||
|
|
"learning_rate": 3.867370395306068e-05,
|
||
|
|
"logits/chosen": 2.1766569425767703,
|
||
|
|
"logits/rejected": -1.087643152794101,
|
||
|
|
"logps/chosen": -69.47466747760772,
|
||
|
|
"logps/rejected": -327.71664810180664,
|
||
|
|
"loss": 1.194763967760082e-07,
|
||
|
|
"mean_token_accuracy": 0.6569934625178575,
|
||
|
|
"num_tokens": 3594396.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.461885306239128,
|
||
|
|
"rewards/margins": 19.697250258922576,
|
||
|
|
"rewards/rejected": -15.235364770889282,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2546571284532546,
|
||
|
|
"epoch": 1.1837037037037037,
|
||
|
|
"grad_norm": 8.940696716308594e-06,
|
||
|
|
"learning_rate": 3.794478174686328e-05,
|
||
|
|
"logits/chosen": 2.28344883152218,
|
||
|
|
"logits/rejected": -1.1421346511748105,
|
||
|
|
"logps/chosen": -64.66728138923645,
|
||
|
|
"logps/rejected": -324.4414520263672,
|
||
|
|
"loss": 7.843665343898465e-08,
|
||
|
|
"mean_token_accuracy": 0.662892397120595,
|
||
|
|
"num_tokens": 3682850.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.419300290942192,
|
||
|
|
"rewards/margins": 19.782128620147706,
|
||
|
|
"rewards/rejected": -15.362828314304352,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1837037037037037,
|
||
|
|
"eval_entropy": 1.2814960267160322,
|
||
|
|
"eval_logits/chosen": 2.2466129676027395,
|
||
|
|
"eval_logits/rejected": -1.158793695801587,
|
||
|
|
"eval_logps/chosen": -67.91167767398007,
|
||
|
|
"eval_logps/rejected": -326.0148432804988,
|
||
|
|
"eval_loss": 7.35781259209034e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6629326431067674,
|
||
|
|
"eval_num_tokens": 3682850.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.323958210178189,
|
||
|
|
"eval_rewards/margins": 19.420454478764032,
|
||
|
|
"eval_rewards/rejected": -15.096496268585845,
|
||
|
|
"eval_runtime": 38.5811,
|
||
|
|
"eval_samples_per_second": 7.387,
|
||
|
|
"eval_steps_per_second": 3.706,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2794287726283073,
|
||
|
|
"epoch": 1.2133333333333334,
|
||
|
|
"grad_norm": 6.0498714447021484e-06,
|
||
|
|
"learning_rate": 3.720050057902496e-05,
|
||
|
|
"logits/chosen": 2.2348389102512294,
|
||
|
|
"logits/rejected": -1.20443931083074,
|
||
|
|
"logps/chosen": -61.86296122074127,
|
||
|
|
"logps/rejected": -331.40450420379636,
|
||
|
|
"loss": 1.5817265648365718e-07,
|
||
|
|
"mean_token_accuracy": 0.6723401784896851,
|
||
|
|
"num_tokens": 3776120.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.320593364536762,
|
||
|
|
"rewards/margins": 19.594025874137877,
|
||
|
|
"rewards/rejected": -15.273432505130767,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2816326960921287,
|
||
|
|
"epoch": 1.242962962962963,
|
||
|
|
"grad_norm": 2.7865171432495117e-06,
|
||
|
|
"learning_rate": 3.644174353789204e-05,
|
||
|
|
"logits/chosen": 2.2707540738720113,
|
||
|
|
"logits/rejected": -1.191927789160487,
|
||
|
|
"logps/chosen": -64.48058602809905,
|
||
|
|
"logps/rejected": -332.1292999267578,
|
||
|
|
"loss": 9.944698149411124e-08,
|
||
|
|
"mean_token_accuracy": 0.6633661832660437,
|
||
|
|
"num_tokens": 3868574.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.300156235694885,
|
||
|
|
"rewards/margins": 19.600781667232514,
|
||
|
|
"rewards/rejected": -15.30062551498413,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2759614169597626,
|
||
|
|
"epoch": 1.2725925925925927,
|
||
|
|
"grad_norm": 6.29425048828125e-05,
|
||
|
|
"learning_rate": 3.5669410887410095e-05,
|
||
|
|
"logits/chosen": 2.3054515861422304,
|
||
|
|
"logits/rejected": -1.1359497002203924,
|
||
|
|
"logps/chosen": -64.89167664051055,
|
||
|
|
"logps/rejected": -334.02518730163575,
|
||
|
|
"loss": 2.404902943453635e-07,
|
||
|
|
"mean_token_accuracy": 0.6750707887113094,
|
||
|
|
"num_tokens": 3961842.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.182991920411586,
|
||
|
|
"rewards/margins": 19.472245895862578,
|
||
|
|
"rewards/rejected": -15.289253902435302,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2799105122685432,
|
||
|
|
"epoch": 1.3022222222222222,
|
||
|
|
"grad_norm": 0.00012683868408203125,
|
||
|
|
"learning_rate": 3.488441899896217e-05,
|
||
|
|
"logits/chosen": 2.345937521884795,
|
||
|
|
"logits/rejected": -1.2320921962004756,
|
||
|
|
"logps/chosen": -58.358681523799895,
|
||
|
|
"logps/rejected": -320.94243869781496,
|
||
|
|
"loss": 2.9142654511815635e-07,
|
||
|
|
"mean_token_accuracy": 0.679117338731885,
|
||
|
|
"num_tokens": 4051000.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.320470458269119,
|
||
|
|
"rewards/margins": 19.39275724887848,
|
||
|
|
"rewards/rejected": -15.072286760807037,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2606690004467964,
|
||
|
|
"epoch": 1.3318518518518518,
|
||
|
|
"grad_norm": 3.933906555175781e-05,
|
||
|
|
"learning_rate": 3.4087699264095745e-05,
|
||
|
|
"logits/chosen": 2.3590029943979434,
|
||
|
|
"logits/rejected": -1.180752685047298,
|
||
|
|
"logps/chosen": -57.81958304643631,
|
||
|
|
"logps/rejected": -327.3783897399902,
|
||
|
|
"loss": 1.31027945826645e-07,
|
||
|
|
"mean_token_accuracy": 0.6829047620296478,
|
||
|
|
"num_tokens": 4144901.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.355239224433899,
|
||
|
|
"rewards/margins": 19.49090631008148,
|
||
|
|
"rewards/rejected": -15.135667026042938,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2720958605408668,
|
||
|
|
"epoch": 1.3614814814814815,
|
||
|
|
"grad_norm": 7.152557373046875e-06,
|
||
|
|
"learning_rate": 3.328019698942826e-05,
|
||
|
|
"logits/chosen": 2.28385155999331,
|
||
|
|
"logits/rejected": -1.0655708622856899,
|
||
|
|
"logps/chosen": -65.57811061143875,
|
||
|
|
"logps/rejected": -324.391535949707,
|
||
|
|
"loss": 1.4767925904379808e-07,
|
||
|
|
"mean_token_accuracy": 0.6687033347785473,
|
||
|
|
"num_tokens": 4235782.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.3411215707659725,
|
||
|
|
"rewards/margins": 19.290599715709686,
|
||
|
|
"rewards/rejected": -14.949478232860566,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2874604403972625,
|
||
|
|
"epoch": 1.3911111111111112,
|
||
|
|
"grad_norm": 0.00011920928955078125,
|
||
|
|
"learning_rate": 3.246287027504237e-05,
|
||
|
|
"logits/chosen": 2.2650682684340646,
|
||
|
|
"logits/rejected": -1.1018559007875284,
|
||
|
|
"logps/chosen": -63.989255738258365,
|
||
|
|
"logps/rejected": -333.0100872039795,
|
||
|
|
"loss": 2.971911953864037e-07,
|
||
|
|
"mean_token_accuracy": 0.6702111043035984,
|
||
|
|
"num_tokens": 4327602.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.187621220946312,
|
||
|
|
"rewards/margins": 19.38519823551178,
|
||
|
|
"rewards/rejected": -15.197577047348023,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.293594116717577,
|
||
|
|
"epoch": 1.4207407407407406,
|
||
|
|
"grad_norm": 9.870529174804688e-05,
|
||
|
|
"learning_rate": 3.163668887770181e-05,
|
||
|
|
"logits/chosen": 2.306861956625707,
|
||
|
|
"logits/rejected": -1.1661940995045186,
|
||
|
|
"logps/chosen": -61.81727409362793,
|
||
|
|
"logps/rejected": -332.9074993133545,
|
||
|
|
"loss": 2.5863685095828257e-07,
|
||
|
|
"mean_token_accuracy": 0.6736553743481636,
|
||
|
|
"num_tokens": 4419888.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.17706933170557,
|
||
|
|
"rewards/margins": 19.331134915351868,
|
||
|
|
"rewards/rejected": -15.154065454006195,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2759695634245873,
|
||
|
|
"epoch": 1.4503703703703703,
|
||
|
|
"grad_norm": 6.771087646484375e-05,
|
||
|
|
"learning_rate": 3.08026330602367e-05,
|
||
|
|
"logits/chosen": 2.338572971372103,
|
||
|
|
"logits/rejected": -1.1794468686720447,
|
||
|
|
"logps/chosen": -58.53629996776581,
|
||
|
|
"logps/rejected": -332.7186403274536,
|
||
|
|
"loss": 8.574451726417465e-08,
|
||
|
|
"mean_token_accuracy": 0.6770360998809337,
|
||
|
|
"num_tokens": 4511414.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.380688160657883,
|
||
|
|
"rewards/margins": 19.57960090637207,
|
||
|
|
"rewards/rejected": -15.198912739753723,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2591664418578148,
|
||
|
|
"epoch": 1.48,
|
||
|
|
"grad_norm": 8.046627044677734e-06,
|
||
|
|
"learning_rate": 2.9961692428463285e-05,
|
||
|
|
"logits/chosen": 2.2509338882701817,
|
||
|
|
"logits/rejected": -1.136340958920119,
|
||
|
|
"logps/chosen": -66.00177557468415,
|
||
|
|
"logps/rejected": -332.10723934173586,
|
||
|
|
"loss": 3.2082580219139346e-07,
|
||
|
|
"mean_token_accuracy": 0.6653452910482883,
|
||
|
|
"num_tokens": 4604818.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.144159619510174,
|
||
|
|
"rewards/margins": 19.529365801811217,
|
||
|
|
"rewards/rejected": -15.385206091403962,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.48,
|
||
|
|
"eval_entropy": 1.2810596311009013,
|
||
|
|
"eval_logits/chosen": 2.2449733761384874,
|
||
|
|
"eval_logits/rejected": -1.1597314363523379,
|
||
|
|
"eval_logps/chosen": -67.92375413854639,
|
||
|
|
"eval_logps/rejected": -326.1967938829969,
|
||
|
|
"eval_loss": 7.327445018745493e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6633227342492217,
|
||
|
|
"eval_num_tokens": 4604818.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.322750580894364,
|
||
|
|
"eval_rewards/margins": 19.43744218599546,
|
||
|
|
"eval_rewards/rejected": -15.114691607601992,
|
||
|
|
"eval_runtime": 38.6636,
|
||
|
|
"eval_samples_per_second": 7.371,
|
||
|
|
"eval_steps_per_second": 3.699,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2616984777152538,
|
||
|
|
"epoch": 1.5096296296296297,
|
||
|
|
"grad_norm": 0.000125885009765625,
|
||
|
|
"learning_rate": 2.9114864757018352e-05,
|
||
|
|
"logits/chosen": 2.294192165543051,
|
||
|
|
"logits/rejected": -1.0944581394998258,
|
||
|
|
"logps/chosen": -64.26435973644257,
|
||
|
|
"logps/rejected": -325.8657173156738,
|
||
|
|
"loss": 1.0070654070659657e-07,
|
||
|
|
"mean_token_accuracy": 0.6660138607025147,
|
||
|
|
"num_tokens": 4695387.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.35933136343956,
|
||
|
|
"rewards/margins": 19.695837664604188,
|
||
|
|
"rewards/rejected": -15.336506283283233,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2706036016345024,
|
||
|
|
"epoch": 1.5392592592592593,
|
||
|
|
"grad_norm": 8.821487426757812e-05,
|
||
|
|
"learning_rate": 2.8263154805501297e-05,
|
||
|
|
"logits/chosen": 2.228209829000261,
|
||
|
|
"logits/rejected": -1.113362437458028,
|
||
|
|
"logps/chosen": -69.59165489673615,
|
||
|
|
"logps/rejected": -321.3457368850708,
|
||
|
|
"loss": 1.4221999435903854e-07,
|
||
|
|
"mean_token_accuracy": 0.6667736105620861,
|
||
|
|
"num_tokens": 4788711.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.314350298047065,
|
||
|
|
"rewards/margins": 19.27485271692276,
|
||
|
|
"rewards/rejected": -14.960502445697784,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2767572030425072,
|
||
|
|
"epoch": 1.568888888888889,
|
||
|
|
"grad_norm": 0.0002460479736328125,
|
||
|
|
"learning_rate": 2.7407573126328545e-05,
|
||
|
|
"logits/chosen": 2.3174187011624654,
|
||
|
|
"logits/rejected": -1.2211792118899774,
|
||
|
|
"logps/chosen": -58.480189538002016,
|
||
|
|
"logps/rejected": -321.1500442504883,
|
||
|
|
"loss": 1.383585185976699e-07,
|
||
|
|
"mean_token_accuracy": 0.675158528238535,
|
||
|
|
"num_tokens": 4877592.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.162171944975853,
|
||
|
|
"rewards/margins": 19.111147022247316,
|
||
|
|
"rewards/rejected": -14.948975014686585,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2621485769748688,
|
||
|
|
"epoch": 1.5985185185185187,
|
||
|
|
"grad_norm": 3.844499588012695e-06,
|
||
|
|
"learning_rate": 2.6549134865714868e-05,
|
||
|
|
"logits/chosen": 2.26315295877127,
|
||
|
|
"logits/rejected": -1.12053805020359,
|
||
|
|
"logps/chosen": -64.27606203556061,
|
||
|
|
"logps/rejected": -328.73465633392334,
|
||
|
|
"loss": 1.7509564713691362e-07,
|
||
|
|
"mean_token_accuracy": 0.670067022740841,
|
||
|
|
"num_tokens": 4969700.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.281642350554466,
|
||
|
|
"rewards/margins": 19.606130039691926,
|
||
|
|
"rewards/rejected": -15.32448751926422,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.266157256066799,
|
||
|
|
"epoch": 1.6281481481481481,
|
||
|
|
"grad_norm": 1.4901161193847656e-05,
|
||
|
|
"learning_rate": 2.5688858559204053e-05,
|
||
|
|
"logits/chosen": 2.3036601533908074,
|
||
|
|
"logits/rejected": -1.1204950201201778,
|
||
|
|
"logps/chosen": -63.52790377140045,
|
||
|
|
"logps/rejected": -332.37768840789795,
|
||
|
|
"loss": 1.9515678104653488e-07,
|
||
|
|
"mean_token_accuracy": 0.6726867496967316,
|
||
|
|
"num_tokens": 5064766.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.242260693013668,
|
||
|
|
"rewards/margins": 19.574455058574678,
|
||
|
|
"rewards/rejected": -15.33219426870346,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.272486585378647,
|
||
|
|
"epoch": 1.6577777777777778,
|
||
|
|
"grad_norm": 3.933906555175781e-06,
|
||
|
|
"learning_rate": 2.4827764923178246e-05,
|
||
|
|
"logits/chosen": 2.3149282625691305,
|
||
|
|
"logits/rejected": -1.2307484890552138,
|
||
|
|
"logps/chosen": -60.608607578277585,
|
||
|
|
"logps/rejected": -325.5159580230713,
|
||
|
|
"loss": 1.471338123337773e-07,
|
||
|
|
"mean_token_accuracy": 0.6735777668654919,
|
||
|
|
"num_tokens": 5155008.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.4290194630622866,
|
||
|
|
"rewards/margins": 19.601059877872466,
|
||
|
|
"rewards/rejected": -15.172040581703186,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2813369497656821,
|
||
|
|
"epoch": 1.6874074074074072,
|
||
|
|
"grad_norm": 0.000339508056640625,
|
||
|
|
"learning_rate": 2.3966875643779667e-05,
|
||
|
|
"logits/chosen": 2.2549751619675136,
|
||
|
|
"logits/rejected": -1.1594325055530381,
|
||
|
|
"logps/chosen": -64.34185855388641,
|
||
|
|
"logps/rejected": -327.56716537475586,
|
||
|
|
"loss": 1.8051871393254259e-07,
|
||
|
|
"mean_token_accuracy": 0.6600819997489452,
|
||
|
|
"num_tokens": 5248360.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.4127004653215405,
|
||
|
|
"rewards/margins": 19.473870050907134,
|
||
|
|
"rewards/rejected": -15.061169505119324,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2690633878111839,
|
||
|
|
"epoch": 1.717037037037037,
|
||
|
|
"grad_norm": 0.000118255615234375,
|
||
|
|
"learning_rate": 2.3107212164681776e-05,
|
||
|
|
"logits/chosen": 2.277724768747875,
|
||
|
|
"logits/rejected": -1.1787549020085861,
|
||
|
|
"logps/chosen": -66.66345942020416,
|
||
|
|
"logps/rejected": -323.8808866500855,
|
||
|
|
"loss": 2.298829940627911e-07,
|
||
|
|
"mean_token_accuracy": 0.6669816076755524,
|
||
|
|
"num_tokens": 5340554.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.2985071912407875,
|
||
|
|
"rewards/margins": 19.356661868095397,
|
||
|
|
"rewards/rejected": -15.058154690265656,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.303315782546997,
|
||
|
|
"epoch": 1.7466666666666666,
|
||
|
|
"grad_norm": 9.894371032714844e-06,
|
||
|
|
"learning_rate": 2.224979447514802e-05,
|
||
|
|
"logits/chosen": 2.2417929976862743,
|
||
|
|
"logits/rejected": -1.1242295347690896,
|
||
|
|
"logps/chosen": -66.85722150802613,
|
||
|
|
"logps/rejected": -326.75849361419677,
|
||
|
|
"loss": 5.121967660670635e-07,
|
||
|
|
"mean_token_accuracy": 0.6579514071345329,
|
||
|
|
"num_tokens": 5433019.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.279269021749497,
|
||
|
|
"rewards/margins": 19.181958985328674,
|
||
|
|
"rewards/rejected": -14.902689921855927,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2511712044477463,
|
||
|
|
"epoch": 1.7762962962962963,
|
||
|
|
"grad_norm": 1.811981201171875e-05,
|
||
|
|
"learning_rate": 2.1395639899816333e-05,
|
||
|
|
"logits/chosen": 2.4288169740201644,
|
||
|
|
"logits/rejected": -1.0983362881453427,
|
||
|
|
"logps/chosen": -58.963488447666165,
|
||
|
|
"logps/rejected": -328.273935508728,
|
||
|
|
"loss": 5.187651368032676e-07,
|
||
|
|
"mean_token_accuracy": 0.6844911381602288,
|
||
|
|
"num_tokens": 5527564.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.607324048876762,
|
||
|
|
"rewards/margins": 19.77645412683487,
|
||
|
|
"rewards/rejected": -15.169130110740662,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7762962962962963,
|
||
|
|
"eval_entropy": 1.2805373877078503,
|
||
|
|
"eval_logits/chosen": 2.243851235815972,
|
||
|
|
"eval_logits/rejected": -1.1594670058021077,
|
||
|
|
"eval_logps/chosen": -67.99770445923706,
|
||
|
|
"eval_logps/rejected": -326.55811144421983,
|
||
|
|
"eval_loss": 6.923921773704933e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6634519100189209,
|
||
|
|
"eval_num_tokens": 5527564.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.315355520148377,
|
||
|
|
"eval_rewards/margins": 19.466178794007202,
|
||
|
|
"eval_rewards/rejected": -15.15082325968709,
|
||
|
|
"eval_runtime": 39.0736,
|
||
|
|
"eval_samples_per_second": 7.294,
|
||
|
|
"eval_steps_per_second": 3.66,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2721578769385815,
|
||
|
|
"epoch": 1.805925925925926,
|
||
|
|
"grad_norm": 1.3053417205810547e-05,
|
||
|
|
"learning_rate": 2.054576189164518e-05,
|
||
|
|
"logits/chosen": 2.3202715445664515,
|
||
|
|
"logits/rejected": -1.132963813611814,
|
||
|
|
"logps/chosen": -60.805744123458865,
|
||
|
|
"logps/rejected": -332.585825920105,
|
||
|
|
"loss": 3.902006255884771e-08,
|
||
|
|
"mean_token_accuracy": 0.6726082235574722,
|
||
|
|
"num_tokens": 5620386.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.585019749403,
|
||
|
|
"rewards/margins": 20.017272901535033,
|
||
|
|
"rewards/rejected": -15.43225302696228,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2652241297066211,
|
||
|
|
"epoch": 1.8355555555555556,
|
||
|
|
"grad_norm": 1.633167266845703e-05,
|
||
|
|
"learning_rate": 1.9701168829453306e-05,
|
||
|
|
"logits/chosen": 2.259219360616968,
|
||
|
|
"logits/rejected": -1.0503303018419232,
|
||
|
|
"logps/chosen": -66.24489846229554,
|
||
|
|
"logps/rejected": -322.5876424789429,
|
||
|
|
"loss": 1.7130523701780476e-07,
|
||
|
|
"mean_token_accuracy": 0.6619290798902512,
|
||
|
|
"num_tokens": 5712637.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.140004143118858,
|
||
|
|
"rewards/margins": 19.201012241840363,
|
||
|
|
"rewards/rejected": -15.06100798845291,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2682488977909088,
|
||
|
|
"epoch": 1.8651851851851853,
|
||
|
|
"grad_norm": 2.956390380859375e-05,
|
||
|
|
"learning_rate": 1.8862862821480025e-05,
|
||
|
|
"logits/chosen": 2.3071657025101677,
|
||
|
|
"logits/rejected": -1.1965660990092328,
|
||
|
|
"logps/chosen": -65.7512573480606,
|
||
|
|
"logps/rejected": -326.63800010681155,
|
||
|
|
"loss": 9.80972572506289e-08,
|
||
|
|
"mean_token_accuracy": 0.6676984764635563,
|
||
|
|
"num_tokens": 5804212.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.233162444829941,
|
||
|
|
"rewards/margins": 19.35871180295944,
|
||
|
|
"rewards/rejected": -15.125549244880677,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2738682702183723,
|
||
|
|
"epoch": 1.894814814814815,
|
||
|
|
"grad_norm": 1.3709068298339844e-06,
|
||
|
|
"learning_rate": 1.8031838516385423e-05,
|
||
|
|
"logits/chosen": 2.277379836690332,
|
||
|
|
"logits/rejected": -1.1057443809996192,
|
||
|
|
"logps/chosen": -64.37387623786927,
|
||
|
|
"logps/rejected": -329.1884712219238,
|
||
|
|
"loss": 5.206376272326452e-08,
|
||
|
|
"mean_token_accuracy": 0.6642427556216717,
|
||
|
|
"num_tokens": 5897138.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.30352343916893,
|
||
|
|
"rewards/margins": 19.877286291122438,
|
||
|
|
"rewards/rejected": -15.573762822151185,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2760708816349506,
|
||
|
|
"epoch": 1.9244444444444444,
|
||
|
|
"grad_norm": 5.1021575927734375e-05,
|
||
|
|
"learning_rate": 1.7209081923101472e-05,
|
||
|
|
"logits/chosen": 2.327136815796652,
|
||
|
|
"logits/rejected": -1.1425043440165608,
|
||
|
|
"logps/chosen": -66.02759101390839,
|
||
|
|
"logps/rejected": -332.0861042022705,
|
||
|
|
"loss": 1.4513241239910712e-07,
|
||
|
|
"mean_token_accuracy": 0.66774348244071,
|
||
|
|
"num_tokens": 5990180.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.311979895830154,
|
||
|
|
"rewards/margins": 19.720207822322845,
|
||
|
|
"rewards/rejected": -15.408227932453155,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2696824759244918,
|
||
|
|
"epoch": 1.954074074074074,
|
||
|
|
"grad_norm": 2.4557113647460938e-05,
|
||
|
|
"learning_rate": 1.639556924093404e-05,
|
||
|
|
"logits/chosen": 2.2885721283972242,
|
||
|
|
"logits/rejected": -1.12765834632216,
|
||
|
|
"logps/chosen": -64.96237716674804,
|
||
|
|
"logps/rejected": -326.4824584960937,
|
||
|
|
"loss": 1.2898549357487353e-07,
|
||
|
|
"mean_token_accuracy": 0.6572523653507233,
|
||
|
|
"num_tokens": 6084186.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.2255510374903675,
|
||
|
|
"rewards/margins": 19.46041429042816,
|
||
|
|
"rewards/rejected": -15.234863257408142,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2641157731413841,
|
||
|
|
"epoch": 1.9837037037037037,
|
||
|
|
"grad_norm": 1.9073486328125e-06,
|
||
|
|
"learning_rate": 1.5592265701304114e-05,
|
||
|
|
"logits/chosen": 2.3036376802600183,
|
||
|
|
"logits/rejected": -1.1734117772095596,
|
||
|
|
"logps/chosen": -65.30954070091248,
|
||
|
|
"logps/rejected": -323.9220626831055,
|
||
|
|
"loss": 2.2752515178581233e-07,
|
||
|
|
"mean_token_accuracy": 0.6635996960103512,
|
||
|
|
"num_tokens": 6177151.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.288326196372509,
|
||
|
|
"rewards/margins": 19.406356739997864,
|
||
|
|
"rewards/rejected": -15.118030607700348,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2779945597836846,
|
||
|
|
"epoch": 2.011851851851852,
|
||
|
|
"grad_norm": 0.000255584716796875,
|
||
|
|
"learning_rate": 1.4800124422502334e-05,
|
||
|
|
"logits/chosen": 2.3972204176263543,
|
||
|
|
"logits/rejected": -1.1209583839266675,
|
||
|
|
"logps/chosen": -61.41229806448284,
|
||
|
|
"logps/rejected": -324.14299693860505,
|
||
|
|
"loss": 3.6133037610852626e-07,
|
||
|
|
"mean_token_accuracy": 0.6630792186448449,
|
||
|
|
"num_tokens": 6262744.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.139316172976243,
|
||
|
|
"rewards/margins": 19.215763267717865,
|
||
|
|
"rewards/rejected": -15.076447160620438,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.266596408188343,
|
||
|
|
"epoch": 2.0414814814814815,
|
||
|
|
"grad_norm": 4.4889748096466064e-07,
|
||
|
|
"learning_rate": 1.4020085278815745e-05,
|
||
|
|
"logits/chosen": 2.3154754788175156,
|
||
|
|
"logits/rejected": -1.152066471886049,
|
||
|
|
"logps/chosen": -62.28092246055603,
|
||
|
|
"logps/rejected": -331.86610050201415,
|
||
|
|
"loss": 7.074050927258213e-08,
|
||
|
|
"mean_token_accuracy": 0.6724916785955429,
|
||
|
|
"num_tokens": 6357968.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.411373123526573,
|
||
|
|
"rewards/margins": 19.874622988700867,
|
||
|
|
"rewards/rejected": -15.4632497549057,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2671823538839817,
|
||
|
|
"epoch": 2.071111111111111,
|
||
|
|
"grad_norm": 0.0002994537353515625,
|
||
|
|
"learning_rate": 1.3253073785368547e-05,
|
||
|
|
"logits/chosen": 2.29187693793979,
|
||
|
|
"logits/rejected": -1.1263780908242413,
|
||
|
|
"logps/chosen": -65.86139830350876,
|
||
|
|
"logps/rejected": -331.38874130249025,
|
||
|
|
"loss": 2.163802719223895e-07,
|
||
|
|
"mean_token_accuracy": 0.6635798320174218,
|
||
|
|
"num_tokens": 6449816.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.283302050828934,
|
||
|
|
"rewards/margins": 19.748378670215608,
|
||
|
|
"rewards/rejected": -15.465076541900634,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.071111111111111,
|
||
|
|
"eval_entropy": 1.2803548124286679,
|
||
|
|
"eval_logits/chosen": 2.243403260803776,
|
||
|
|
"eval_logits/rejected": -1.1587966303436696,
|
||
|
|
"eval_logps/chosen": -68.02522693980823,
|
||
|
|
"eval_logps/rejected": -326.6569073016827,
|
||
|
|
"eval_loss": 7.350444775511278e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6639135730016482,
|
||
|
|
"eval_num_tokens": 6449816.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.312603289847607,
|
||
|
|
"eval_rewards/margins": 19.473306129028746,
|
||
|
|
"eval_rewards/rejected": -15.160702772073812,
|
||
|
|
"eval_runtime": 39.1309,
|
||
|
|
"eval_samples_per_second": 7.283,
|
||
|
|
"eval_steps_per_second": 3.654,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2728232815861702,
|
||
|
|
"epoch": 2.100740740740741,
|
||
|
|
"grad_norm": 8.821487426757812e-05,
|
||
|
|
"learning_rate": 1.2500000000000006e-05,
|
||
|
|
"logits/chosen": 2.255679331290835,
|
||
|
|
"logits/rejected": -1.1216202176026864,
|
||
|
|
"logps/chosen": -61.825205373764035,
|
||
|
|
"logps/rejected": -323.78655910491943,
|
||
|
|
"loss": 2.141799996024929e-07,
|
||
|
|
"mean_token_accuracy": 0.667589108645916,
|
||
|
|
"num_tokens": 6540123.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.437801697850228,
|
||
|
|
"rewards/margins": 19.34732162952423,
|
||
|
|
"rewards/rejected": -14.90951988697052,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.271144437789917,
|
||
|
|
"epoch": 2.1303703703703705,
|
||
|
|
"grad_norm": 6.771087646484375e-05,
|
||
|
|
"learning_rate": 1.1761757443482286e-05,
|
||
|
|
"logits/chosen": 2.2690192203566064,
|
||
|
|
"logits/rejected": -1.176060724310163,
|
||
|
|
"logps/chosen": -65.25097322463989,
|
||
|
|
"logps/rejected": -325.64191341400146,
|
||
|
|
"loss": 7.844523679523263e-08,
|
||
|
|
"mean_token_accuracy": 0.6650686420500278,
|
||
|
|
"num_tokens": 6629998.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.375448957085609,
|
||
|
|
"rewards/margins": 19.631182181835175,
|
||
|
|
"rewards/rejected": -15.255733144283294,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.263370680809021,
|
||
|
|
"epoch": 2.16,
|
||
|
|
"grad_norm": 0.00067901611328125,
|
||
|
|
"learning_rate": 1.1039222039359645e-05,
|
||
|
|
"logits/chosen": 2.2999157883359946,
|
||
|
|
"logits/rejected": -1.1241040776625622,
|
||
|
|
"logps/chosen": -65.3091173171997,
|
||
|
|
"logps/rejected": -327.6589548110962,
|
||
|
|
"loss": 6.326518359855982e-07,
|
||
|
|
"mean_token_accuracy": 0.6666021130979061,
|
||
|
|
"num_tokens": 6721579.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.218965843319893,
|
||
|
|
"rewards/margins": 19.43423910140991,
|
||
|
|
"rewards/rejected": -15.21527328491211,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2835220411419868,
|
||
|
|
"epoch": 2.18962962962963,
|
||
|
|
"grad_norm": 9.059906005859375e-06,
|
||
|
|
"learning_rate": 1.0333251074666609e-05,
|
||
|
|
"logits/chosen": 2.2917566691940756,
|
||
|
|
"logits/rejected": -1.2816091428578473,
|
||
|
|
"logps/chosen": -66.77954509258271,
|
||
|
|
"logps/rejected": -333.47667198181153,
|
||
|
|
"loss": 1.7912802832142916e-07,
|
||
|
|
"mean_token_accuracy": 0.6699437476694584,
|
||
|
|
"num_tokens": 6813377.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.310945153236389,
|
||
|
|
"rewards/margins": 19.620563447475433,
|
||
|
|
"rewards/rejected": -15.309618008136749,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2834552891552449,
|
||
|
|
"epoch": 2.2192592592592595,
|
||
|
|
"grad_norm": 2.8252601623535156e-05,
|
||
|
|
"learning_rate": 9.644682182758306e-06,
|
||
|
|
"logits/chosen": 2.296253967390375,
|
||
|
|
"logits/rejected": -1.1382098734020025,
|
||
|
|
"logps/chosen": -65.18716852664947,
|
||
|
|
"logps/rejected": -333.20018062591555,
|
||
|
|
"loss": 3.619171593527426e-07,
|
||
|
|
"mean_token_accuracy": 0.670500548928976,
|
||
|
|
"num_tokens": 6906265.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.34758489727974,
|
||
|
|
"rewards/margins": 19.480838096141817,
|
||
|
|
"rewards/rejected": -15.133253240585328,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2889503717422486,
|
||
|
|
"epoch": 2.2488888888888887,
|
||
|
|
"grad_norm": 3.6716461181640625e-05,
|
||
|
|
"learning_rate": 8.974332349459994e-06,
|
||
|
|
"logits/chosen": 2.274084705882701,
|
||
|
|
"logits/rejected": -1.1973438289074938,
|
||
|
|
"logps/chosen": -62.72564001083374,
|
||
|
|
"logps/rejected": -335.81445999145507,
|
||
|
|
"loss": 2.772670995909721e-07,
|
||
|
|
"mean_token_accuracy": 0.663724122941494,
|
||
|
|
"num_tokens": 7004404.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.22343917787075,
|
||
|
|
"rewards/margins": 19.607576775550843,
|
||
|
|
"rewards/rejected": -15.384137523174285,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2709729745984077,
|
||
|
|
"epoch": 2.2785185185185184,
|
||
|
|
"grad_norm": 6.818771362304688e-05,
|
||
|
|
"learning_rate": 8.322996943714672e-06,
|
||
|
|
"logits/chosen": 2.315020811648067,
|
||
|
|
"logits/rejected": -1.1844662490433007,
|
||
|
|
"logps/chosen": -60.06333501338959,
|
||
|
|
"logps/rejected": -324.07994575500487,
|
||
|
|
"loss": 6.899633149259899e-08,
|
||
|
|
"mean_token_accuracy": 0.674921452999115,
|
||
|
|
"num_tokens": 7091630.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.219157862663269,
|
||
|
|
"rewards/margins": 19.409486842155456,
|
||
|
|
"rewards/rejected": -15.190328931808471,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2802273795008658,
|
||
|
|
"epoch": 2.308148148148148,
|
||
|
|
"grad_norm": 7.677078247070312e-05,
|
||
|
|
"learning_rate": 7.691448773879257e-06,
|
||
|
|
"logits/chosen": 2.25499691013824,
|
||
|
|
"logits/rejected": -1.1512603749381776,
|
||
|
|
"logps/chosen": -63.52460696697235,
|
||
|
|
"logps/rejected": -325.2589776992798,
|
||
|
|
"loss": 1.4859768953101594e-07,
|
||
|
|
"mean_token_accuracy": 0.6620502531528473,
|
||
|
|
"num_tokens": 7182707.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.2716171652078625,
|
||
|
|
"rewards/margins": 19.37758446931839,
|
||
|
|
"rewards/rejected": -15.10596705675125,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2827171966433526,
|
||
|
|
"epoch": 2.3377777777777777,
|
||
|
|
"grad_norm": 6.645917892456055e-06,
|
||
|
|
"learning_rate": 7.080437170788723e-06,
|
||
|
|
"logits/chosen": 2.311279925102533,
|
||
|
|
"logits/rejected": -1.1746961520146812,
|
||
|
|
"logps/chosen": -59.46050326824188,
|
||
|
|
"logps/rejected": -336.5900556564331,
|
||
|
|
"loss": 3.455784508332727e-07,
|
||
|
|
"mean_token_accuracy": 0.6743544064462185,
|
||
|
|
"num_tokens": 7275656.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.199921746551991,
|
||
|
|
"rewards/margins": 19.863027906417848,
|
||
|
|
"rewards/rejected": -15.663106298446655,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2675680316984654,
|
||
|
|
"epoch": 2.3674074074074074,
|
||
|
|
"grad_norm": 0.00015926361083984375,
|
||
|
|
"learning_rate": 6.490687098676332e-06,
|
||
|
|
"logits/chosen": 2.3366430778544096,
|
||
|
|
"logits/rejected": -1.1229084532499813,
|
||
|
|
"logps/chosen": -58.34785094261169,
|
||
|
|
"logps/rejected": -331.7050970077515,
|
||
|
|
"loss": 6.806028522987617e-07,
|
||
|
|
"mean_token_accuracy": 0.6765504986047745,
|
||
|
|
"num_tokens": 7367211.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.387070021033287,
|
||
|
|
"rewards/margins": 19.79595556259155,
|
||
|
|
"rewards/rejected": -15.40888546705246,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3674074074074074,
|
||
|
|
"eval_entropy": 1.2803315270197142,
|
||
|
|
"eval_logits/chosen": 2.2436578263578553,
|
||
|
|
"eval_logits/rejected": -1.158513689404391,
|
||
|
|
"eval_logps/chosen": -68.02018330980847,
|
||
|
|
"eval_logps/rejected": -326.71273963768164,
|
||
|
|
"eval_loss": 7.024308615655173e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6631912376497175,
|
||
|
|
"eval_num_tokens": 7367211.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.313107631006441,
|
||
|
|
"eval_rewards/margins": 19.479393625592852,
|
||
|
|
"eval_rewards/rejected": -15.166285921643665,
|
||
|
|
"eval_runtime": 39.4139,
|
||
|
|
"eval_samples_per_second": 7.231,
|
||
|
|
"eval_steps_per_second": 3.628,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2536483012139796,
|
||
|
|
"epoch": 2.397037037037037,
|
||
|
|
"grad_norm": 0.00010061264038085938,
|
||
|
|
"learning_rate": 5.9228982950048416e-06,
|
||
|
|
"logits/chosen": 2.306488303700283,
|
||
|
|
"logits/rejected": -1.102614074804673,
|
||
|
|
"logps/chosen": -64.7256872177124,
|
||
|
|
"logps/rejected": -316.81419982910154,
|
||
|
|
"loss": 2.7541559575183785e-07,
|
||
|
|
"mean_token_accuracy": 0.6695533588528633,
|
||
|
|
"num_tokens": 7456110.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.3090651541948315,
|
||
|
|
"rewards/margins": 19.122487926483153,
|
||
|
|
"rewards/rejected": -14.813422644138337,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2691614858806133,
|
||
|
|
"epoch": 2.4266666666666667,
|
||
|
|
"grad_norm": 3.844499588012695e-06,
|
||
|
|
"learning_rate": 5.377744440229135e-06,
|
||
|
|
"logits/chosen": 2.276143363869749,
|
||
|
|
"logits/rejected": -1.2205610514710212,
|
||
|
|
"logps/chosen": -64.77891628742218,
|
||
|
|
"logps/rejected": -330.2338497161865,
|
||
|
|
"loss": 9.86900658972445e-08,
|
||
|
|
"mean_token_accuracy": 0.6663463093340397,
|
||
|
|
"num_tokens": 7548949.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.394037923216819,
|
||
|
|
"rewards/margins": 19.882170844078065,
|
||
|
|
"rewards/rejected": -15.488132905960082,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2759164057672023,
|
||
|
|
"epoch": 2.4562962962962964,
|
||
|
|
"grad_norm": 1.5735626220703125e-05,
|
||
|
|
"learning_rate": 4.855872358475547e-06,
|
||
|
|
"logits/chosen": 2.2611198054529975,
|
||
|
|
"logits/rejected": -1.1668456534925964,
|
||
|
|
"logps/chosen": -63.698548150062564,
|
||
|
|
"logps/rejected": -323.2269037246704,
|
||
|
|
"loss": 1.5968792013154598e-07,
|
||
|
|
"mean_token_accuracy": 0.6731547065079212,
|
||
|
|
"num_tokens": 7637248.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.312624426186085,
|
||
|
|
"rewards/margins": 19.461497557163238,
|
||
|
|
"rewards/rejected": -15.148873174190522,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2611838012933732,
|
||
|
|
"epoch": 2.485925925925926,
|
||
|
|
"grad_norm": 0.0002841949462890625,
|
||
|
|
"learning_rate": 4.357901250086108e-06,
|
||
|
|
"logits/chosen": 2.2900714239429774,
|
||
|
|
"logits/rejected": -1.2470973023927523,
|
||
|
|
"logps/chosen": -62.78718073368073,
|
||
|
|
"logps/rejected": -324.9568237304687,
|
||
|
|
"loss": 1.4255396081352955e-07,
|
||
|
|
"mean_token_accuracy": 0.6746141783893108,
|
||
|
|
"num_tokens": 7726306.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.291703987121582,
|
||
|
|
"rewards/margins": 19.421786904335022,
|
||
|
|
"rewards/rejected": -15.130082988739014,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2697825357317924,
|
||
|
|
"epoch": 2.5155555555555553,
|
||
|
|
"grad_norm": 0.0002613067626953125,
|
||
|
|
"learning_rate": 3.884421956938378e-06,
|
||
|
|
"logits/chosen": 2.3862500773086817,
|
||
|
|
"logits/rejected": -1.1250058733609287,
|
||
|
|
"logps/chosen": -61.92496125698089,
|
||
|
|
"logps/rejected": -327.44327507019045,
|
||
|
|
"loss": 1.493016384301882e-07,
|
||
|
|
"mean_token_accuracy": 0.6789508298039436,
|
||
|
|
"num_tokens": 7822125.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.364183455705643,
|
||
|
|
"rewards/margins": 19.506937170028685,
|
||
|
|
"rewards/rejected": -15.142753863334656,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2504698053002357,
|
||
|
|
"epoch": 2.5451851851851854,
|
||
|
|
"grad_norm": 4.506111145019531e-05,
|
||
|
|
"learning_rate": 3.435996261412591e-06,
|
||
|
|
"logits/chosen": 2.2986260733744173,
|
||
|
|
"logits/rejected": -1.1058551351474077,
|
||
|
|
"logps/chosen": -65.89442076683045,
|
||
|
|
"logps/rejected": -330.00583362579346,
|
||
|
|
"loss": 1.4223222706277737e-07,
|
||
|
|
"mean_token_accuracy": 0.6768516503274441,
|
||
|
|
"num_tokens": 7915620.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.385465979576111,
|
||
|
|
"rewards/margins": 20.04776966571808,
|
||
|
|
"rewards/rejected": -15.662303495407105,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2795700900256635,
|
||
|
|
"epoch": 2.5748148148148147,
|
||
|
|
"grad_norm": 2.5153160095214844e-05,
|
||
|
|
"learning_rate": 3.013156219837776e-06,
|
||
|
|
"logits/chosen": 2.237134730489094,
|
||
|
|
"logits/rejected": -1.1218436377455188,
|
||
|
|
"logps/chosen": -65.73119521141052,
|
||
|
|
"logps/rejected": -322.3418972015381,
|
||
|
|
"loss": 2.536130750740995e-07,
|
||
|
|
"mean_token_accuracy": 0.665914348512888,
|
||
|
|
"num_tokens": 8006742.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.355473381280899,
|
||
|
|
"rewards/margins": 19.215320122241973,
|
||
|
|
"rewards/rejected": -14.85984662771225,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2761830180883407,
|
||
|
|
"epoch": 2.6044444444444443,
|
||
|
|
"grad_norm": 1.1742115020751953e-05,
|
||
|
|
"learning_rate": 2.6164035312078448e-06,
|
||
|
|
"logits/chosen": 2.2800606568463015,
|
||
|
|
"logits/rejected": -1.1315629358769943,
|
||
|
|
"logps/chosen": -61.99486291408539,
|
||
|
|
"logps/rejected": -326.9346130371094,
|
||
|
|
"loss": 2.3778172817401355e-07,
|
||
|
|
"mean_token_accuracy": 0.6662385709583759,
|
||
|
|
"num_tokens": 8100018.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.3306570738554,
|
||
|
|
"rewards/margins": 19.479354202747345,
|
||
|
|
"rewards/rejected": -15.148697006702424,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.279340496659279,
|
||
|
|
"epoch": 2.634074074074074,
|
||
|
|
"grad_norm": 9.632110595703125e-05,
|
||
|
|
"learning_rate": 2.2462089419165777e-06,
|
||
|
|
"logits/chosen": 2.3029893365635097,
|
||
|
|
"logits/rejected": -1.171205228496424,
|
||
|
|
"logps/chosen": -65.00987663269044,
|
||
|
|
"logps/rejected": -330.74864501953124,
|
||
|
|
"loss": 6.256735787246726e-08,
|
||
|
|
"mean_token_accuracy": 0.6665534779429436,
|
||
|
|
"num_tokens": 8194303.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.474057069420814,
|
||
|
|
"rewards/margins": 19.85085200071335,
|
||
|
|
"rewards/rejected": -15.376794970035553,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2603400394320488,
|
||
|
|
"epoch": 2.6637037037037037,
|
||
|
|
"grad_norm": 2.4557113647460938e-05,
|
||
|
|
"learning_rate": 1.9030116872178316e-06,
|
||
|
|
"logits/chosen": 2.249223475648134,
|
||
|
|
"logits/rejected": -1.0936998011992798,
|
||
|
|
"logps/chosen": -64.42266466617585,
|
||
|
|
"logps/rejected": -319.57002086639403,
|
||
|
|
"loss": 1.4027624501977697e-07,
|
||
|
|
"mean_token_accuracy": 0.6666488841176033,
|
||
|
|
"num_tokens": 8285684.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.301889230310917,
|
||
|
|
"rewards/margins": 19.297891116142274,
|
||
|
|
"rewards/rejected": -14.99600180387497,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6637037037037037,
|
||
|
|
"eval_entropy": 1.2802694969243937,
|
||
|
|
"eval_logits/chosen": 2.2445470870400355,
|
||
|
|
"eval_logits/rejected": -1.159552092885801,
|
||
|
|
"eval_logps/chosen": -67.97919230694538,
|
||
|
|
"eval_logps/rejected": -326.7502327232094,
|
||
|
|
"eval_loss": 6.49735875413171e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6639414579718264,
|
||
|
|
"eval_num_tokens": 8285684.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.317206760803303,
|
||
|
|
"eval_rewards/margins": 19.48724203843337,
|
||
|
|
"eval_rewards/rejected": -15.17003523553168,
|
||
|
|
"eval_runtime": 39.451,
|
||
|
|
"eval_samples_per_second": 7.224,
|
||
|
|
"eval_steps_per_second": 3.625,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2844877071678638,
|
||
|
|
"epoch": 2.6933333333333334,
|
||
|
|
"grad_norm": 2.205371856689453e-05,
|
||
|
|
"learning_rate": 1.5872189700736339e-06,
|
||
|
|
"logits/chosen": 2.341007675682582,
|
||
|
|
"logits/rejected": -1.1647508343869704,
|
||
|
|
"logps/chosen": -65.77681732177734,
|
||
|
|
"logps/rejected": -333.8214714050293,
|
||
|
|
"loss": 9.888794920698274e-08,
|
||
|
|
"mean_token_accuracy": 0.6609948001801967,
|
||
|
|
"num_tokens": 8379162.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.243448624014855,
|
||
|
|
"rewards/margins": 19.678018867969513,
|
||
|
|
"rewards/rejected": -15.434570217132569,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2820228539407252,
|
||
|
|
"epoch": 2.722962962962963,
|
||
|
|
"grad_norm": 2.9325485229492188e-05,
|
||
|
|
"learning_rate": 1.2992054780085694e-06,
|
||
|
|
"logits/chosen": 2.308397402508807,
|
||
|
|
"logits/rejected": -1.1578754052596163,
|
||
|
|
"logps/chosen": -60.28462275266647,
|
||
|
|
"logps/rejected": -329.8702238082886,
|
||
|
|
"loss": 5.674120302501251e-08,
|
||
|
|
"mean_token_accuracy": 0.6794499330222606,
|
||
|
|
"num_tokens": 8471171.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.262164607644081,
|
||
|
|
"rewards/margins": 19.57164797782898,
|
||
|
|
"rewards/rejected": -15.309483373165131,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2623218022286893,
|
||
|
|
"epoch": 2.7525925925925927,
|
||
|
|
"grad_norm": 0.00011157989501953125,
|
||
|
|
"learning_rate": 1.0393129385436824e-06,
|
||
|
|
"logits/chosen": 2.3240682024976924,
|
||
|
|
"logits/rejected": -1.0668921112986016,
|
||
|
|
"logps/chosen": -59.67337925434113,
|
||
|
|
"logps/rejected": -327.08252410888673,
|
||
|
|
"loss": 1.114461042561743e-07,
|
||
|
|
"mean_token_accuracy": 0.6755911834537983,
|
||
|
|
"num_tokens": 8562918.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.329451468586922,
|
||
|
|
"rewards/margins": 19.44830514192581,
|
||
|
|
"rewards/rejected": -15.118853759765624,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2947206407785417,
|
||
|
|
"epoch": 2.7822222222222224,
|
||
|
|
"grad_norm": 0.00011396408081054688,
|
||
|
|
"learning_rate": 8.078497137373242e-07,
|
||
|
|
"logits/chosen": 2.221305657513471,
|
||
|
|
"logits/rejected": -1.151073296194637,
|
||
|
|
"logps/chosen": -66.11703178882598,
|
||
|
|
"logps/rejected": -329.9900022506714,
|
||
|
|
"loss": 1.417975795448001e-07,
|
||
|
|
"mean_token_accuracy": 0.6663264043629169,
|
||
|
|
"num_tokens": 8655421.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.399758037924767,
|
||
|
|
"rewards/margins": 19.651811373233794,
|
||
|
|
"rewards/rejected": -15.252053284645081,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2984899178147316,
|
||
|
|
"epoch": 2.811851851851852,
|
||
|
|
"grad_norm": 0.00030517578125,
|
||
|
|
"learning_rate": 6.050904343141095e-07,
|
||
|
|
"logits/chosen": 2.22722939090554,
|
||
|
|
"logits/rejected": -1.1332568821088258,
|
||
|
|
"logps/chosen": -66.14050345420837,
|
||
|
|
"logps/rejected": -324.22646408081056,
|
||
|
|
"loss": 2.464663111823029e-07,
|
||
|
|
"mean_token_accuracy": 0.6699574515223503,
|
||
|
|
"num_tokens": 8745995.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.108328685164452,
|
||
|
|
"rewards/margins": 19.001902163028717,
|
||
|
|
"rewards/rejected": -14.893573427200318,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2638258799910544,
|
||
|
|
"epoch": 2.8414814814814813,
|
||
|
|
"grad_norm": 2.4437904357910156e-06,
|
||
|
|
"learning_rate": 4.312756738160145e-07,
|
||
|
|
"logits/chosen": 2.2900851054697773,
|
||
|
|
"logits/rejected": -1.1548115943695305,
|
||
|
|
"logps/chosen": -62.25515162944794,
|
||
|
|
"logps/rejected": -332.41545429229734,
|
||
|
|
"loss": 1.340860762866214e-07,
|
||
|
|
"mean_token_accuracy": 0.6710602834820747,
|
||
|
|
"num_tokens": 8840853.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.389033368229866,
|
||
|
|
"rewards/margins": 19.693468022346497,
|
||
|
|
"rewards/rejected": -15.304434645175935,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2863718561828137,
|
||
|
|
"epoch": 2.871111111111111,
|
||
|
|
"grad_norm": 1.8358230590820312e-05,
|
||
|
|
"learning_rate": 2.8661166316229225e-07,
|
||
|
|
"logits/chosen": 2.3169417969141524,
|
||
|
|
"logits/rejected": -1.186848696642711,
|
||
|
|
"logps/chosen": -62.72764225006104,
|
||
|
|
"logps/rejected": -326.7688617706299,
|
||
|
|
"loss": 1.440903929506021e-07,
|
||
|
|
"mean_token_accuracy": 0.6690138332545758,
|
||
|
|
"num_tokens": 8932082.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.227344951033592,
|
||
|
|
"rewards/margins": 19.289413845539094,
|
||
|
|
"rewards/rejected": -15.062068808078767,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2781345948576928,
|
||
|
|
"epoch": 2.9007407407407406,
|
||
|
|
"grad_norm": 4.887580871582031e-06,
|
||
|
|
"learning_rate": 1.7127004595681729e-07,
|
||
|
|
"logits/chosen": 2.351716086708728,
|
||
|
|
"logits/rejected": -1.1749225494133453,
|
||
|
|
"logps/chosen": -69.60287075042724,
|
||
|
|
"logps/rejected": -329.7513824462891,
|
||
|
|
"loss": 1.1806017710114248e-07,
|
||
|
|
"mean_token_accuracy": 0.6486908756196499,
|
||
|
|
"num_tokens": 9029000.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.309024539589882,
|
||
|
|
"rewards/margins": 19.48654924631119,
|
||
|
|
"rewards/rejected": -15.177524662017822,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2663434036076069,
|
||
|
|
"epoch": 2.9303703703703703,
|
||
|
|
"grad_norm": 7.063150405883789e-06,
|
||
|
|
"learning_rate": 8.538767483325383e-08,
|
||
|
|
"logits/chosen": 2.3055120496617296,
|
||
|
|
"logits/rejected": -1.0696752761558757,
|
||
|
|
"logps/chosen": -65.15313308238983,
|
||
|
|
"logps/rejected": -333.3471397399902,
|
||
|
|
"loss": 1.6038972034948528e-07,
|
||
|
|
"mean_token_accuracy": 0.6714354790747166,
|
||
|
|
"num_tokens": 9120849.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.179977935552597,
|
||
|
|
"rewards/margins": 19.668525290489196,
|
||
|
|
"rewards/rejected": -15.488547360897064,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.279862828552723,
|
||
|
|
"epoch": 2.96,
|
||
|
|
"grad_norm": 3.2633543014526367e-06,
|
||
|
|
"learning_rate": 2.9066449079634406e-08,
|
||
|
|
"logits/chosen": 2.3262460395698,
|
||
|
|
"logits/rejected": -1.0832974043675088,
|
||
|
|
"logps/chosen": -65.35808026790619,
|
||
|
|
"logps/rejected": -333.4561340332031,
|
||
|
|
"loss": 1.2052357760694577e-07,
|
||
|
|
"mean_token_accuracy": 0.654293454438448,
|
||
|
|
"num_tokens": 9214314.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.2864476755261425,
|
||
|
|
"rewards/margins": 19.81372677087784,
|
||
|
|
"rewards/rejected": -15.527279019355774,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.96,
|
||
|
|
"eval_entropy": 1.280155298593161,
|
||
|
|
"eval_logits/chosen": 2.2437926653321396,
|
||
|
|
"eval_logits/rejected": -1.1593534352886283,
|
||
|
|
"eval_logps/chosen": -68.00064637777689,
|
||
|
|
"eval_logps/rejected": -326.6981409246271,
|
||
|
|
"eval_loss": 6.490902251243824e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6637652328797987,
|
||
|
|
"eval_num_tokens": 9214314.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.315061373310489,
|
||
|
|
"eval_rewards/margins": 19.479887595543495,
|
||
|
|
"eval_rewards/rejected": -15.164826126365394,
|
||
|
|
"eval_runtime": 39.7099,
|
||
|
|
"eval_samples_per_second": 7.177,
|
||
|
|
"eval_steps_per_second": 3.601,
|
||
|
|
"step": 1000
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 1013,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 100,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 4.87406059521024e+17,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|