289 lines
9.6 KiB
JSON
289 lines
9.6 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 63,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"entropy": 2.555654156208038,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 35.3809700012207,
|
||
|
|
"learning_rate": 2.8571428571428573e-06,
|
||
|
|
"logits/chosen": 0.4110352631849171,
|
||
|
|
"logits/rejected": 0.4300117739250047,
|
||
|
|
"logps/chosen": -157.81439809799195,
|
||
|
|
"logps/rejected": -150.4315113067627,
|
||
|
|
"loss": 0.6903757572174072,
|
||
|
|
"mean_token_accuracy": 0.45491820499300956,
|
||
|
|
"num_tokens": 63280.0,
|
||
|
|
"rewards/accuracies": 0.54375,
|
||
|
|
"rewards/chosen": 0.018655257727368736,
|
||
|
|
"rewards/margins": 0.006082923361100256,
|
||
|
|
"rewards/rejected": 0.012572333973366767,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.5844572484493256,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 47.283817291259766,
|
||
|
|
"learning_rate": 4.984280524733107e-06,
|
||
|
|
"logits/chosen": 0.2089756903127898,
|
||
|
|
"logits/rejected": 0.3923359114864852,
|
||
|
|
"logps/chosen": -160.2436939239502,
|
||
|
|
"logps/rejected": -154.7825647354126,
|
||
|
|
"loss": 0.7087126731872558,
|
||
|
|
"mean_token_accuracy": 0.4676915630698204,
|
||
|
|
"num_tokens": 128751.0,
|
||
|
|
"rewards/accuracies": 0.46875,
|
||
|
|
"rewards/chosen": 0.026459120609797538,
|
||
|
|
"rewards/margins": -0.018515753094106913,
|
||
|
|
"rewards/rejected": 0.04497487433254719,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.4940366342663767,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 48.20133972167969,
|
||
|
|
"learning_rate": 4.809698831278217e-06,
|
||
|
|
"logits/chosen": 0.12422276096059184,
|
||
|
|
"logits/rejected": 0.18387425940175633,
|
||
|
|
"logps/chosen": -151.31794481277467,
|
||
|
|
"logps/rejected": -163.5261468887329,
|
||
|
|
"loss": 0.6959157466888428,
|
||
|
|
"mean_token_accuracy": 0.47235521450638773,
|
||
|
|
"num_tokens": 194142.0,
|
||
|
|
"rewards/accuracies": 0.46875,
|
||
|
|
"rewards/chosen": -0.059232240472920236,
|
||
|
|
"rewards/margins": 0.021671066922135652,
|
||
|
|
"rewards/rejected": -0.0809033086989075,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.5471011281013487,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 42.294281005859375,
|
||
|
|
"learning_rate": 4.454578706170075e-06,
|
||
|
|
"logits/chosen": -0.030133471365837695,
|
||
|
|
"logits/rejected": 0.08451241878365641,
|
||
|
|
"logps/chosen": -135.41445970535278,
|
||
|
|
"logps/rejected": -135.83921966552734,
|
||
|
|
"loss": 0.6954083442687988,
|
||
|
|
"mean_token_accuracy": 0.4549576163291931,
|
||
|
|
"num_tokens": 254871.0,
|
||
|
|
"rewards/accuracies": 0.5375,
|
||
|
|
"rewards/chosen": -0.18437330080196263,
|
||
|
|
"rewards/margins": 0.07001338726840914,
|
||
|
|
"rewards/rejected": -0.2543866881169379,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.4626115679740908,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 53.21565628051758,
|
||
|
|
"learning_rate": 3.946678240449515e-06,
|
||
|
|
"logits/chosen": 0.035210923156354317,
|
||
|
|
"logits/rejected": 0.03131556176291432,
|
||
|
|
"logps/chosen": -147.02334337234498,
|
||
|
|
"logps/rejected": -139.0251603126526,
|
||
|
|
"loss": 0.7144542694091797,
|
||
|
|
"mean_token_accuracy": 0.4505583204329014,
|
||
|
|
"num_tokens": 317752.0,
|
||
|
|
"rewards/accuracies": 0.55,
|
||
|
|
"rewards/chosen": -0.3056703276000917,
|
||
|
|
"rewards/margins": 0.030420188792049885,
|
||
|
|
"rewards/rejected": -0.33609051471576096,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.418185669183731,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 50.63364791870117,
|
||
|
|
"learning_rate": 3.3256976548879183e-06,
|
||
|
|
"logits/chosen": -0.0063577626599184795,
|
||
|
|
"logits/rejected": 0.0660789555792278,
|
||
|
|
"logps/chosen": -151.3002492904663,
|
||
|
|
"logps/rejected": -150.41238498687744,
|
||
|
|
"loss": 0.6920054435729981,
|
||
|
|
"mean_token_accuracy": 0.445500810444355,
|
||
|
|
"num_tokens": 387452.0,
|
||
|
|
"rewards/accuracies": 0.55625,
|
||
|
|
"rewards/chosen": -0.2962045251857489,
|
||
|
|
"rewards/margins": 0.08783619347959756,
|
||
|
|
"rewards/rejected": -0.38404072066769,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.42596016228199,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 47.31837844848633,
|
||
|
|
"learning_rate": 2.6401761180929798e-06,
|
||
|
|
"logits/chosen": -0.04082900087497934,
|
||
|
|
"logits/rejected": 0.015806176234233794,
|
||
|
|
"logps/chosen": -182.25081090927125,
|
||
|
|
"logps/rejected": -175.18973140716554,
|
||
|
|
"loss": 0.6894076824188232,
|
||
|
|
"mean_token_accuracy": 0.46841963976621626,
|
||
|
|
"num_tokens": 454873.0,
|
||
|
|
"rewards/accuracies": 0.55625,
|
||
|
|
"rewards/chosen": -0.30482858400791885,
|
||
|
|
"rewards/margins": 0.07829219745472074,
|
||
|
|
"rewards/rejected": -0.38312078421004114,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.421294018626213,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 47.72428894042969,
|
||
|
|
"learning_rate": 1.9436976651092143e-06,
|
||
|
|
"logits/chosen": 0.04491766159482387,
|
||
|
|
"logits/rejected": 0.15141158319763592,
|
||
|
|
"logps/chosen": -160.55729579925537,
|
||
|
|
"logps/rejected": -161.91398558616638,
|
||
|
|
"loss": 0.7262070178985596,
|
||
|
|
"mean_token_accuracy": 0.4616738587617874,
|
||
|
|
"num_tokens": 518527.0,
|
||
|
|
"rewards/accuracies": 0.51875,
|
||
|
|
"rewards/chosen": -0.2627937063574791,
|
||
|
|
"rewards/margins": 0.037597383465617897,
|
||
|
|
"rewards/rejected": -0.30039109364151956,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.3493838876485826,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 41.70436096191406,
|
||
|
|
"learning_rate": 1.2907027822369006e-06,
|
||
|
|
"logits/chosen": 0.06111884454416239,
|
||
|
|
"logits/rejected": 0.1077471313778626,
|
||
|
|
"logps/chosen": -155.73900985717773,
|
||
|
|
"logps/rejected": -162.23499221801757,
|
||
|
|
"loss": 0.6567366123199463,
|
||
|
|
"mean_token_accuracy": 0.47065152823925016,
|
||
|
|
"num_tokens": 586496.0,
|
||
|
|
"rewards/accuracies": 0.5875,
|
||
|
|
"rewards/chosen": -0.15574295134283603,
|
||
|
|
"rewards/margins": 0.16263325293548406,
|
||
|
|
"rewards/rejected": -0.31837619938887657,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.3660752564668655,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 46.26735305786133,
|
||
|
|
"learning_rate": 7.322330470336314e-07,
|
||
|
|
"logits/chosen": 0.04797605992282007,
|
||
|
|
"logits/rejected": 0.137719508348569,
|
||
|
|
"logps/chosen": -165.56337423324584,
|
||
|
|
"logps/rejected": -165.80605907440184,
|
||
|
|
"loss": 0.6880872249603271,
|
||
|
|
"mean_token_accuracy": 0.4618512228131294,
|
||
|
|
"num_tokens": 659345.0,
|
||
|
|
"rewards/accuracies": 0.51875,
|
||
|
|
"rewards/chosen": -0.13928775684908032,
|
||
|
|
"rewards/margins": 0.09893286619335413,
|
||
|
|
"rewards/rejected": -0.238220629747957,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_entropy": 2.4305524230003357,
|
||
|
|
"eval_logits/chosen": 0.060791561005096514,
|
||
|
|
"eval_logits/rejected": 0.1157722840661163,
|
||
|
|
"eval_logps/chosen": -163.22620516967774,
|
||
|
|
"eval_logps/rejected": -155.69468914794922,
|
||
|
|
"eval_loss": 0.699152946472168,
|
||
|
|
"eval_mean_token_accuracy": 0.4591814261674881,
|
||
|
|
"eval_num_tokens": 659345.0,
|
||
|
|
"eval_rewards/accuracies": 0.523,
|
||
|
|
"eval_rewards/chosen": -0.17783968531899155,
|
||
|
|
"eval_rewards/margins": 0.07608089716732502,
|
||
|
|
"eval_rewards/rejected": -0.2539205798432231,
|
||
|
|
"eval_runtime": 204.6975,
|
||
|
|
"eval_samples_per_second": 4.885,
|
||
|
|
"eval_steps_per_second": 1.221,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.3940737187862395,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 48.0695915222168,
|
||
|
|
"learning_rate": 3.119414452281158e-07,
|
||
|
|
"logits/chosen": 0.07112344903443206,
|
||
|
|
"logits/rejected": 0.19535127089682655,
|
||
|
|
"logps/chosen": -150.24068050384523,
|
||
|
|
"logps/rejected": -145.48797540664674,
|
||
|
|
"loss": 0.697087287902832,
|
||
|
|
"mean_token_accuracy": 0.4701634831726551,
|
||
|
|
"num_tokens": 731475.0,
|
||
|
|
"rewards/accuracies": 0.56875,
|
||
|
|
"rewards/chosen": -0.14963094238191843,
|
||
|
|
"rewards/margins": 0.08578414842486382,
|
||
|
|
"rewards/rejected": -0.2354150922037661,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 2.357463392615318,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 48.10399627685547,
|
||
|
|
"learning_rate": 6.268021954544095e-08,
|
||
|
|
"logits/chosen": 0.01995007363434443,
|
||
|
|
"logits/rejected": 0.16687794622622143,
|
||
|
|
"logps/chosen": -158.93942432403566,
|
||
|
|
"logps/rejected": -166.66946430206298,
|
||
|
|
"loss": 0.6701779842376709,
|
||
|
|
"mean_token_accuracy": 0.4689980365335941,
|
||
|
|
"num_tokens": 797757.0,
|
||
|
|
"rewards/accuracies": 0.5375,
|
||
|
|
"rewards/chosen": -0.1631853505037725,
|
||
|
|
"rewards/margins": 0.13551276298239828,
|
||
|
|
"rewards/rejected": -0.2986981191672385,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_entropy": 2.431350471973419,
|
||
|
|
"eval_logits/chosen": 0.058027013522972216,
|
||
|
|
"eval_logits/rejected": 0.11281541237759458,
|
||
|
|
"eval_logps/chosen": -163.3362089691162,
|
||
|
|
"eval_logps/rejected": -155.8263526611328,
|
||
|
|
"eval_loss": 0.6985306143760681,
|
||
|
|
"eval_mean_token_accuracy": 0.45935997557640074,
|
||
|
|
"eval_num_tokens": 829784.0,
|
||
|
|
"eval_rewards/accuracies": 0.529,
|
||
|
|
"eval_rewards/chosen": -0.188840083822608,
|
||
|
|
"eval_rewards/margins": 0.07824680726230145,
|
||
|
|
"eval_rewards/rejected": -0.26708688925206664,
|
||
|
|
"eval_runtime": 204.726,
|
||
|
|
"eval_samples_per_second": 4.885,
|
||
|
|
"eval_steps_per_second": 1.221,
|
||
|
|
"step": 63
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 63,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 100,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 3267470732328960.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|