495 lines
14 KiB
JSON
495 lines
14 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2326,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 1.0907272757589817,
|
|
"epoch": 0.021503064186646596,
|
|
"grad_norm": 6.21875,
|
|
"learning_rate": 6.309012875536481e-06,
|
|
"loss": 2.1228,
|
|
"mean_token_accuracy": 0.6076477284729481,
|
|
"num_tokens": 13032106.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 1.508835320174694,
|
|
"epoch": 0.04300612837329319,
|
|
"grad_norm": 0.9296875,
|
|
"learning_rate": 1.2746781115879827e-05,
|
|
"loss": 1.5598,
|
|
"mean_token_accuracy": 0.6578908252716065,
|
|
"num_tokens": 26059780.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 1.3828416666388512,
|
|
"epoch": 0.0645091925599398,
|
|
"grad_norm": 0.86328125,
|
|
"learning_rate": 1.918454935622318e-05,
|
|
"loss": 1.3879,
|
|
"mean_token_accuracy": 0.6857959526777268,
|
|
"num_tokens": 39088700.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 1.3144192987680434,
|
|
"epoch": 0.08601225674658639,
|
|
"grad_norm": 0.875,
|
|
"learning_rate": 2.5622317596566525e-05,
|
|
"loss": 1.3154,
|
|
"mean_token_accuracy": 0.6985341481864452,
|
|
"num_tokens": 52119410.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 1.271382558941841,
|
|
"epoch": 0.10751532093323299,
|
|
"grad_norm": 0.80859375,
|
|
"learning_rate": 2.9995674446943393e-05,
|
|
"loss": 1.271,
|
|
"mean_token_accuracy": 0.706343949586153,
|
|
"num_tokens": 65159605.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 1.2296087518334389,
|
|
"epoch": 0.1290183851198796,
|
|
"grad_norm": 0.83984375,
|
|
"learning_rate": 2.992645465135663e-05,
|
|
"loss": 1.2279,
|
|
"mean_token_accuracy": 0.7139623883366585,
|
|
"num_tokens": 78193377.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 1.2088195663690566,
|
|
"epoch": 0.15052144930652617,
|
|
"grad_norm": 0.85546875,
|
|
"learning_rate": 2.977320103566463e-05,
|
|
"loss": 1.2063,
|
|
"mean_token_accuracy": 0.7178937835991382,
|
|
"num_tokens": 91226428.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 1.1856484353542327,
|
|
"epoch": 0.17202451349317277,
|
|
"grad_norm": 0.859375,
|
|
"learning_rate": 2.953677639596146e-05,
|
|
"loss": 1.1826,
|
|
"mean_token_accuracy": 0.7218748706579209,
|
|
"num_tokens": 104254404.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 1.170458752065897,
|
|
"epoch": 0.19352757767981937,
|
|
"grad_norm": 0.796875,
|
|
"learning_rate": 2.9218511769395955e-05,
|
|
"loss": 1.1672,
|
|
"mean_token_accuracy": 0.7247805140912533,
|
|
"num_tokens": 117293915.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 1.1519357265532018,
|
|
"epoch": 0.21503064186646598,
|
|
"grad_norm": 0.82421875,
|
|
"learning_rate": 2.8820198940621553e-05,
|
|
"loss": 1.1476,
|
|
"mean_token_accuracy": 0.7280966921150684,
|
|
"num_tokens": 130328746.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 1.1446182508766651,
|
|
"epoch": 0.23653370605311258,
|
|
"grad_norm": 0.78125,
|
|
"learning_rate": 2.8344080354303162e-05,
|
|
"loss": 1.1415,
|
|
"mean_token_accuracy": 0.729242740124464,
|
|
"num_tokens": 143360031.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"entropy": 1.1355690321326255,
|
|
"epoch": 0.2580367702397592,
|
|
"grad_norm": 0.796875,
|
|
"learning_rate": 2.7792836490472105e-05,
|
|
"loss": 1.1315,
|
|
"mean_token_accuracy": 0.7310809700191021,
|
|
"num_tokens": 156389192.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"entropy": 1.1251171430945397,
|
|
"epoch": 0.2795398344264058,
|
|
"grad_norm": 0.76953125,
|
|
"learning_rate": 2.716957077380419e-05,
|
|
"loss": 1.1206,
|
|
"mean_token_accuracy": 0.7328316581249237,
|
|
"num_tokens": 169427250.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"entropy": 1.1196351052820683,
|
|
"epoch": 0.30104289861305233,
|
|
"grad_norm": 0.7734375,
|
|
"learning_rate": 2.6477792101779463e-05,
|
|
"loss": 1.1157,
|
|
"mean_token_accuracy": 0.7339280897378921,
|
|
"num_tokens": 182461636.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"entropy": 1.1190296991169453,
|
|
"epoch": 0.32254596279969894,
|
|
"grad_norm": 0.7890625,
|
|
"learning_rate": 2.5721395090087677e-05,
|
|
"loss": 1.1145,
|
|
"mean_token_accuracy": 0.7338955473899841,
|
|
"num_tokens": 195499557.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"entropy": 1.1153910259902478,
|
|
"epoch": 0.34404902698634554,
|
|
"grad_norm": 0.765625,
|
|
"learning_rate": 2.4904638146495e-05,
|
|
"loss": 1.1109,
|
|
"mean_token_accuracy": 0.7346864821016789,
|
|
"num_tokens": 208534790.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"entropy": 1.1082513900101185,
|
|
"epoch": 0.36555209117299214,
|
|
"grad_norm": 0.765625,
|
|
"learning_rate": 2.403211949661299e-05,
|
|
"loss": 1.1038,
|
|
"mean_token_accuracy": 0.7359914369881153,
|
|
"num_tokens": 221571806.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"entropy": 1.101485371440649,
|
|
"epoch": 0.38705515535963875,
|
|
"grad_norm": 0.70703125,
|
|
"learning_rate": 2.3108751296541346e-05,
|
|
"loss": 1.097,
|
|
"mean_token_accuracy": 0.7371249902248382,
|
|
"num_tokens": 234607758.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"entropy": 1.0892982496321202,
|
|
"epoch": 0.40855821954628535,
|
|
"grad_norm": 0.7734375,
|
|
"learning_rate": 2.2139731978126626e-05,
|
|
"loss": 1.0846,
|
|
"mean_token_accuracy": 0.739790627360344,
|
|
"num_tokens": 247639114.0,
|
|
"step": 950
|
|
},
|
|
{
|
|
"entropy": 1.0900326637923718,
|
|
"epoch": 0.43006128373293195,
|
|
"grad_norm": 0.765625,
|
|
"learning_rate": 2.113051698252911e-05,
|
|
"loss": 1.0854,
|
|
"mean_token_accuracy": 0.7392391939461231,
|
|
"num_tokens": 260666114.0,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"entropy": 1.0910632169246675,
|
|
"epoch": 0.45156434791957856,
|
|
"grad_norm": 0.7109375,
|
|
"learning_rate": 2.008678804686377e-05,
|
|
"loss": 1.0864,
|
|
"mean_token_accuracy": 0.7390980970859528,
|
|
"num_tokens": 273699261.0,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"entropy": 1.092719831764698,
|
|
"epoch": 0.47306741210622516,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 1.901442121682713e-05,
|
|
"loss": 1.0876,
|
|
"mean_token_accuracy": 0.7389003723859787,
|
|
"num_tokens": 286736280.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"entropy": 1.0873091377317905,
|
|
"epoch": 0.4945704762928717,
|
|
"grad_norm": 0.73828125,
|
|
"learning_rate": 1.7919453765394382e-05,
|
|
"loss": 1.0825,
|
|
"mean_token_accuracy": 0.739827173948288,
|
|
"num_tokens": 299773014.0,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"entropy": 1.083502656519413,
|
|
"epoch": 0.5160735404795184,
|
|
"grad_norm": 0.73046875,
|
|
"learning_rate": 1.6808050203829845e-05,
|
|
"loss": 1.0787,
|
|
"mean_token_accuracy": 0.7405482211709022,
|
|
"num_tokens": 312808336.0,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"entropy": 1.0772461792826653,
|
|
"epoch": 0.537576604666165,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 1.568646757636395e-05,
|
|
"loss": 1.0718,
|
|
"mean_token_accuracy": 0.7420317196846008,
|
|
"num_tokens": 325846521.0,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"entropy": 1.0821722888946532,
|
|
"epoch": 0.5590796688528116,
|
|
"grad_norm": 0.6953125,
|
|
"learning_rate": 1.4561020233922692e-05,
|
|
"loss": 1.0772,
|
|
"mean_token_accuracy": 0.7409076395630837,
|
|
"num_tokens": 338879341.0,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"entropy": 1.0741630932688713,
|
|
"epoch": 0.5805827330394582,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1.3438044285228563e-05,
|
|
"loss": 1.0692,
|
|
"mean_token_accuracy": 0.7425356175005436,
|
|
"num_tokens": 351915071.0,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"entropy": 1.076338079869747,
|
|
"epoch": 0.6020857972261047,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 1.232386192540825e-05,
|
|
"loss": 1.0712,
|
|
"mean_token_accuracy": 0.7421206159889698,
|
|
"num_tokens": 364953019.0,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"entropy": 1.0808810468018055,
|
|
"epoch": 0.6235888614127513,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1.1224745842931732e-05,
|
|
"loss": 1.0753,
|
|
"mean_token_accuracy": 0.7412997713685036,
|
|
"num_tokens": 377991282.0,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"entropy": 1.0769454421103,
|
|
"epoch": 0.6450919255993979,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 1.0146883905267009e-05,
|
|
"loss": 1.0717,
|
|
"mean_token_accuracy": 0.7417641612887382,
|
|
"num_tokens": 391033088.0,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"entropy": 1.0731029373407364,
|
|
"epoch": 0.6665949897860445,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 9.096344322064908e-06,
|
|
"loss": 1.0682,
|
|
"mean_token_accuracy": 0.7425531581044197,
|
|
"num_tokens": 404065596.0,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"entropy": 1.0717635145783424,
|
|
"epoch": 0.6880980539726911,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 8.07904148200046e-06,
|
|
"loss": 1.0666,
|
|
"mean_token_accuracy": 0.7429507914185524,
|
|
"num_tokens": 417096085.0,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"entropy": 1.077366787046194,
|
|
"epoch": 0.7096011181593377,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 7.100702655604489e-06,
|
|
"loss": 1.0724,
|
|
"mean_token_accuracy": 0.7418400295078754,
|
|
"num_tokens": 430123506.0,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"entropy": 1.0732380370795727,
|
|
"epoch": 0.7311041823459843,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 6.166835751543826e-06,
|
|
"loss": 1.0676,
|
|
"mean_token_accuracy": 0.7426895283162593,
|
|
"num_tokens": 443151833.0,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"entropy": 1.0706148628890515,
|
|
"epoch": 0.7526072465326309,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 5.28269830787766e-06,
|
|
"loss": 1.0658,
|
|
"mean_token_accuracy": 0.7431967754662037,
|
|
"num_tokens": 456178484.0,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"entropy": 1.0802401515841484,
|
|
"epoch": 0.7741103107192775,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 4.453267892864809e-06,
|
|
"loss": 1.0743,
|
|
"mean_token_accuracy": 0.7413974392414093,
|
|
"num_tokens": 469218541.0,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"entropy": 1.0747664365172387,
|
|
"epoch": 0.7956133749059241,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 3.683214081961266e-06,
|
|
"loss": 1.0698,
|
|
"mean_token_accuracy": 0.7424948561191559,
|
|
"num_tokens": 482256952.0,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"entropy": 1.0749107849597932,
|
|
"epoch": 0.8171164390925707,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 2.976872168773471e-06,
|
|
"loss": 1.0692,
|
|
"mean_token_accuracy": 0.7422900800406933,
|
|
"num_tokens": 495284119.0,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"entropy": 1.0713467453420162,
|
|
"epoch": 0.8386195032792173,
|
|
"grad_norm": 0.70703125,
|
|
"learning_rate": 2.33821875797092e-06,
|
|
"loss": 1.066,
|
|
"mean_token_accuracy": 0.7431659936904907,
|
|
"num_tokens": 508317474.0,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"entropy": 1.07583929002285,
|
|
"epoch": 0.8601225674658639,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 1.7708493775665642e-06,
|
|
"loss": 1.0708,
|
|
"mean_token_accuracy": 0.7418897579610347,
|
|
"num_tokens": 521344568.0,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"entropy": 1.0714419484138489,
|
|
"epoch": 0.8816256316525105,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 1.277958236604662e-06,
|
|
"loss": 1.0659,
|
|
"mean_token_accuracy": 0.7432458332180977,
|
|
"num_tokens": 534372019.0,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"entropy": 1.0746202796697617,
|
|
"epoch": 0.9031286958391571,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 8.623202422174543e-07,
|
|
"loss": 1.0702,
|
|
"mean_token_accuracy": 0.7424204003810883,
|
|
"num_tokens": 547410540.0,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"entropy": 1.0706361505389212,
|
|
"epoch": 0.9246317600258037,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 5.262753772920398e-07,
|
|
"loss": 1.0649,
|
|
"mean_token_accuracy": 0.7435105523467064,
|
|
"num_tokens": 560450920.0,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"entropy": 1.0749036824703218,
|
|
"epoch": 0.9461348242124503,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 2.717155266990046e-07,
|
|
"loss": 1.0687,
|
|
"mean_token_accuracy": 0.7426265163719654,
|
|
"num_tokens": 573486528.0,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"entropy": 1.079463709741831,
|
|
"epoch": 0.9676378883990969,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 1.000738262492562e-07,
|
|
"loss": 1.0738,
|
|
"mean_token_accuracy": 0.7416864347457885,
|
|
"num_tokens": 586528234.0,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"entropy": 1.076693961173296,
|
|
"epoch": 0.9891409525857434,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 1.2316594342917098e-08,
|
|
"loss": 1.0718,
|
|
"mean_token_accuracy": 0.7420642673969269,
|
|
"num_tokens": 599568450.0,
|
|
"step": 2300
|
|
}
|
|
],
|
|
"logging_steps": 50,
|
|
"max_steps": 2326,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 5.125906217801089e+18,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|