445 lines
13 KiB
JSON
445 lines
13 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 411,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 2.396573783457279,
|
|
"epoch": 0.024384668139907032,
|
|
"grad_norm": 9.35292911529541,
|
|
"learning_rate": 3.4615384615384617e-06,
|
|
"loss": 2.2944561004638673,
|
|
"mean_token_accuracy": 0.5621824176982045,
|
|
"num_tokens": 1138906.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 1.8874999050050973,
|
|
"epoch": 0.048769336279814064,
|
|
"grad_norm": 0.24744118750095367,
|
|
"learning_rate": 4.92462311557789e-06,
|
|
"loss": 1.7899208068847656,
|
|
"mean_token_accuracy": 0.628978100977838,
|
|
"num_tokens": 2293055.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 1.7037040002644062,
|
|
"epoch": 0.0731540044197211,
|
|
"grad_norm": 0.16765649616718292,
|
|
"learning_rate": 4.7989949748743725e-06,
|
|
"loss": 1.619252586364746,
|
|
"mean_token_accuracy": 0.6523888424038887,
|
|
"num_tokens": 3409941.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 1.587071031332016,
|
|
"epoch": 0.09753867255962813,
|
|
"grad_norm": 0.36697572469711304,
|
|
"learning_rate": 4.673366834170855e-06,
|
|
"loss": 1.5249744415283204,
|
|
"mean_token_accuracy": 0.6715210393071175,
|
|
"num_tokens": 4532623.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 1.5395442467182874,
|
|
"epoch": 0.12192334069953517,
|
|
"grad_norm": 0.15763863921165466,
|
|
"learning_rate": 4.547738693467337e-06,
|
|
"loss": 1.4616802215576172,
|
|
"mean_token_accuracy": 0.6869728203862906,
|
|
"num_tokens": 5700954.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 1.4842158311977982,
|
|
"epoch": 0.1463080088394422,
|
|
"grad_norm": 0.05921294912695885,
|
|
"learning_rate": 4.42211055276382e-06,
|
|
"loss": 1.3743048667907716,
|
|
"mean_token_accuracy": 0.7031451839953661,
|
|
"num_tokens": 6797204.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 1.4583968281745912,
|
|
"epoch": 0.17069267697934923,
|
|
"grad_norm": 0.15839053690433502,
|
|
"learning_rate": 4.2964824120603016e-06,
|
|
"loss": 1.364408302307129,
|
|
"mean_token_accuracy": 0.7028716931119561,
|
|
"num_tokens": 7935684.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 1.3737002125009894,
|
|
"epoch": 0.19507734511925626,
|
|
"grad_norm": 0.2055569589138031,
|
|
"learning_rate": 4.170854271356784e-06,
|
|
"loss": 1.2981891632080078,
|
|
"mean_token_accuracy": 0.7154177609831095,
|
|
"num_tokens": 9100164.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 1.3365630799904467,
|
|
"epoch": 0.2194620132591633,
|
|
"grad_norm": 0.08240273594856262,
|
|
"learning_rate": 4.045226130653266e-06,
|
|
"loss": 1.2741600036621095,
|
|
"mean_token_accuracy": 0.7182379385456443,
|
|
"num_tokens": 10251679.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 1.287783969938755,
|
|
"epoch": 0.24384668139907034,
|
|
"grad_norm": 0.06663907319307327,
|
|
"learning_rate": 3.919597989949749e-06,
|
|
"loss": 1.2690627098083496,
|
|
"mean_token_accuracy": 0.7233840074390173,
|
|
"num_tokens": 11356051.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 1.2535302273929119,
|
|
"epoch": 0.26823134953897737,
|
|
"grad_norm": 0.11975441128015518,
|
|
"learning_rate": 3.7939698492462314e-06,
|
|
"loss": 1.2374841690063476,
|
|
"mean_token_accuracy": 0.728168161958456,
|
|
"num_tokens": 12484208.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 1.2418669171631336,
|
|
"epoch": 0.2926160176788844,
|
|
"grad_norm": 0.09109269082546234,
|
|
"learning_rate": 3.6683417085427137e-06,
|
|
"loss": 1.2369081497192382,
|
|
"mean_token_accuracy": 0.7338437670841813,
|
|
"num_tokens": 13561759.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 1.213769260980189,
|
|
"epoch": 0.31700068581879143,
|
|
"grad_norm": 0.09609343856573105,
|
|
"learning_rate": 3.542713567839196e-06,
|
|
"loss": 1.2193414688110351,
|
|
"mean_token_accuracy": 0.7347961757332087,
|
|
"num_tokens": 14727153.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 1.2099591370671987,
|
|
"epoch": 0.34138535395869846,
|
|
"grad_norm": 0.045514918863773346,
|
|
"learning_rate": 3.4170854271356786e-06,
|
|
"loss": 1.2130016326904296,
|
|
"mean_token_accuracy": 0.7383950665593147,
|
|
"num_tokens": 15840961.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 1.187617147155106,
|
|
"epoch": 0.3657700220986055,
|
|
"grad_norm": 0.13311414420604706,
|
|
"learning_rate": 3.291457286432161e-06,
|
|
"loss": 1.2047300338745117,
|
|
"mean_token_accuracy": 0.7410888081416488,
|
|
"num_tokens": 16984769.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 1.172801567055285,
|
|
"epoch": 0.3901546902385125,
|
|
"grad_norm": 0.11037907749414444,
|
|
"learning_rate": 3.165829145728643e-06,
|
|
"loss": 1.191337490081787,
|
|
"mean_token_accuracy": 0.7454195737838745,
|
|
"num_tokens": 18135300.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 1.1579898716881871,
|
|
"epoch": 0.41453935837841954,
|
|
"grad_norm": 0.11947419494390488,
|
|
"learning_rate": 3.040201005025126e-06,
|
|
"loss": 1.1634675025939942,
|
|
"mean_token_accuracy": 0.749046934582293,
|
|
"num_tokens": 19277169.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 1.1670682085677981,
|
|
"epoch": 0.4389240265183266,
|
|
"grad_norm": 0.04084308072924614,
|
|
"learning_rate": 2.914572864321608e-06,
|
|
"loss": 1.1813165664672851,
|
|
"mean_token_accuracy": 0.7435223584994674,
|
|
"num_tokens": 20442936.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 1.1455160442739725,
|
|
"epoch": 0.46330869465823366,
|
|
"grad_norm": 0.08265366405248642,
|
|
"learning_rate": 2.7889447236180903e-06,
|
|
"loss": 1.1532262802124023,
|
|
"mean_token_accuracy": 0.7507004836574197,
|
|
"num_tokens": 21546055.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 1.1505778279155492,
|
|
"epoch": 0.4876933627981407,
|
|
"grad_norm": 0.04568405821919441,
|
|
"learning_rate": 2.663316582914573e-06,
|
|
"loss": 1.1755168914794922,
|
|
"mean_token_accuracy": 0.744980787485838,
|
|
"num_tokens": 22707770.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 1.133826574496925,
|
|
"epoch": 0.5120780309380477,
|
|
"grad_norm": 0.07941383123397827,
|
|
"learning_rate": 2.5376884422110553e-06,
|
|
"loss": 1.1708141326904298,
|
|
"mean_token_accuracy": 0.7456702718511223,
|
|
"num_tokens": 23850765.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 1.1377525204792618,
|
|
"epoch": 0.5364626990779547,
|
|
"grad_norm": 0.0786183699965477,
|
|
"learning_rate": 2.412060301507538e-06,
|
|
"loss": 1.1719804763793946,
|
|
"mean_token_accuracy": 0.7453695390373468,
|
|
"num_tokens": 24969640.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 1.0979079050943255,
|
|
"epoch": 0.5608473672178618,
|
|
"grad_norm": 0.037526968866586685,
|
|
"learning_rate": 2.28643216080402e-06,
|
|
"loss": 1.1441015243530273,
|
|
"mean_token_accuracy": 0.7491984859108924,
|
|
"num_tokens": 26118771.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 1.107094001956284,
|
|
"epoch": 0.5852320353577688,
|
|
"grad_norm": 0.03871579468250275,
|
|
"learning_rate": 2.1608040201005025e-06,
|
|
"loss": 1.1631497383117675,
|
|
"mean_token_accuracy": 0.7486034173518419,
|
|
"num_tokens": 27275023.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 1.0854291021823883,
|
|
"epoch": 0.6096167034976758,
|
|
"grad_norm": 0.04375020042061806,
|
|
"learning_rate": 2.035175879396985e-06,
|
|
"loss": 1.1235612869262694,
|
|
"mean_token_accuracy": 0.7546378022059799,
|
|
"num_tokens": 28333140.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 1.0757445141673088,
|
|
"epoch": 0.6340013716375829,
|
|
"grad_norm": 0.05372092127799988,
|
|
"learning_rate": 1.9095477386934674e-06,
|
|
"loss": 1.1169811248779298,
|
|
"mean_token_accuracy": 0.7564450742676854,
|
|
"num_tokens": 29418762.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 1.0732924519106746,
|
|
"epoch": 0.6583860397774899,
|
|
"grad_norm": 0.058155521750450134,
|
|
"learning_rate": 1.7839195979899499e-06,
|
|
"loss": 1.1255494117736817,
|
|
"mean_token_accuracy": 0.7540135690942407,
|
|
"num_tokens": 30506833.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 1.0920044222846628,
|
|
"epoch": 0.6827707079173969,
|
|
"grad_norm": 0.03951919823884964,
|
|
"learning_rate": 1.6582914572864323e-06,
|
|
"loss": 1.1335023880004882,
|
|
"mean_token_accuracy": 0.7506223158910871,
|
|
"num_tokens": 31612624.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 1.0659992009401322,
|
|
"epoch": 0.7071553760573039,
|
|
"grad_norm": 0.04476330056786537,
|
|
"learning_rate": 1.5326633165829146e-06,
|
|
"loss": 1.1336694717407227,
|
|
"mean_token_accuracy": 0.7525387002155185,
|
|
"num_tokens": 32760933.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 1.052399990335107,
|
|
"epoch": 0.731540044197211,
|
|
"grad_norm": 0.039937425404787064,
|
|
"learning_rate": 1.407035175879397e-06,
|
|
"loss": 1.109424877166748,
|
|
"mean_token_accuracy": 0.7570657260715962,
|
|
"num_tokens": 33900534.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 1.0555859578773379,
|
|
"epoch": 0.755924712337118,
|
|
"grad_norm": 0.05961156636476517,
|
|
"learning_rate": 1.2814070351758795e-06,
|
|
"loss": 1.1106980323791504,
|
|
"mean_token_accuracy": 0.7565701389685273,
|
|
"num_tokens": 35022809.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 1.0802918665111065,
|
|
"epoch": 0.780309380477025,
|
|
"grad_norm": 0.035128556191921234,
|
|
"learning_rate": 1.155778894472362e-06,
|
|
"loss": 1.1406271934509278,
|
|
"mean_token_accuracy": 0.7501922188326716,
|
|
"num_tokens": 36188396.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 1.0649904919788242,
|
|
"epoch": 0.8046940486169321,
|
|
"grad_norm": 0.0338631235063076,
|
|
"learning_rate": 1.0301507537688443e-06,
|
|
"loss": 1.1402740478515625,
|
|
"mean_token_accuracy": 0.7498057916760444,
|
|
"num_tokens": 37320320.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 1.0344580493867397,
|
|
"epoch": 0.8290787167568391,
|
|
"grad_norm": 0.05243472009897232,
|
|
"learning_rate": 9.045226130653267e-07,
|
|
"loss": 1.0902214050292969,
|
|
"mean_token_accuracy": 0.7581806099042296,
|
|
"num_tokens": 38349591.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 1.045315737091005,
|
|
"epoch": 0.8534633848967462,
|
|
"grad_norm": 0.03376954793930054,
|
|
"learning_rate": 7.788944723618092e-07,
|
|
"loss": 1.0944804191589355,
|
|
"mean_token_accuracy": 0.7578815678134561,
|
|
"num_tokens": 39480077.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 1.036611631512642,
|
|
"epoch": 0.8778480530366533,
|
|
"grad_norm": 0.04112258553504944,
|
|
"learning_rate": 6.532663316582916e-07,
|
|
"loss": 1.089322853088379,
|
|
"mean_token_accuracy": 0.7601783391088247,
|
|
"num_tokens": 40567760.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 1.0330295177176594,
|
|
"epoch": 0.9022327211765603,
|
|
"grad_norm": 0.076757051050663,
|
|
"learning_rate": 5.276381909547739e-07,
|
|
"loss": 1.0854530334472656,
|
|
"mean_token_accuracy": 0.759020702727139,
|
|
"num_tokens": 41667104.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 1.055363328382373,
|
|
"epoch": 0.9266173893164673,
|
|
"grad_norm": 0.059090178459882736,
|
|
"learning_rate": 4.0201005025125634e-07,
|
|
"loss": 1.124905490875244,
|
|
"mean_token_accuracy": 0.7546801719814539,
|
|
"num_tokens": 42823783.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 1.0368478525429965,
|
|
"epoch": 0.9510020574563743,
|
|
"grad_norm": 0.08474253863096237,
|
|
"learning_rate": 2.7638190954773876e-07,
|
|
"loss": 1.1043139457702638,
|
|
"mean_token_accuracy": 0.7581638781353831,
|
|
"num_tokens": 43905335.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 1.0361423827707767,
|
|
"epoch": 0.9753867255962814,
|
|
"grad_norm": 0.037209831178188324,
|
|
"learning_rate": 1.5075376884422112e-07,
|
|
"loss": 1.1020542144775392,
|
|
"mean_token_accuracy": 0.7570645421743393,
|
|
"num_tokens": 45063169.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 1.0295690370723605,
|
|
"epoch": 0.9997713937361884,
|
|
"grad_norm": 0.052957672625780106,
|
|
"learning_rate": 2.512562814070352e-08,
|
|
"loss": 1.1035842895507812,
|
|
"mean_token_accuracy": 0.7561511231586338,
|
|
"num_tokens": 46179529.0,
|
|
"step": 410
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 411,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 7.721169677367378e+17,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|