3008 lines
117 KiB
JSON
3008 lines
117 KiB
JSON
{
|
|
"best_global_step": 4842,
|
|
"best_metric": 2.7643399238586426,
|
|
"best_model_checkpoint": "/vol/moe-200m-16exp-streaming/checkpoint-4842",
|
|
"epoch": 0.6,
|
|
"eval_steps": 807,
|
|
"global_step": 4842,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.006195786864931847,
|
|
"expert_0_frac": 0.06268728524446487,
|
|
"expert_10_frac": 0.05253516510128975,
|
|
"expert_11_frac": 0.06409072130918503,
|
|
"expert_12_frac": 0.06890930235385895,
|
|
"expert_13_frac": 0.06048846244812012,
|
|
"expert_14_frac": 0.060645461082458496,
|
|
"expert_15_frac": 0.06372006982564926,
|
|
"expert_1_frac": 0.061552755534648895,
|
|
"expert_2_frac": 0.05120125040411949,
|
|
"expert_3_frac": 0.0811033621430397,
|
|
"expert_4_frac": 0.06206492707133293,
|
|
"expert_5_frac": 0.05403707176446915,
|
|
"expert_6_frac": 0.07373557239770889,
|
|
"expert_7_frac": 0.06307204812765121,
|
|
"expert_8_frac": 0.06395463645458221,
|
|
"expert_9_frac": 0.056201934814453125,
|
|
"expert_frac_max": 0.0811033621430397,
|
|
"expert_frac_min": 0.05120125040411949,
|
|
"expert_frac_std": 0.00760139524936676,
|
|
"grad_norm": 1.535045862197876,
|
|
"learning_rate": 3.647642679900744e-05,
|
|
"loss": 10.6442,
|
|
"router_aux_loss": 2.0115064537525176,
|
|
"router_entropy": 2.7032830785711606,
|
|
"router_entropy_norm": 0.9750032728934159,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.012391573729863693,
|
|
"expert_0_frac": 0.06215367838740349,
|
|
"expert_10_frac": 0.06141716241836548,
|
|
"expert_11_frac": 0.06268706917762756,
|
|
"expert_12_frac": 0.06260079890489578,
|
|
"expert_13_frac": 0.06265205889940262,
|
|
"expert_14_frac": 0.0631578266620636,
|
|
"expert_15_frac": 0.06170821189880371,
|
|
"expert_1_frac": 0.06163309887051582,
|
|
"expert_2_frac": 0.06367384642362595,
|
|
"expert_3_frac": 0.06248243525624275,
|
|
"expert_4_frac": 0.06296652555465698,
|
|
"expert_5_frac": 0.06345007568597794,
|
|
"expert_6_frac": 0.061480410397052765,
|
|
"expert_7_frac": 0.06333854794502258,
|
|
"expert_8_frac": 0.06238511577248573,
|
|
"expert_9_frac": 0.06221316009759903,
|
|
"expert_frac_max": 0.06367384642362595,
|
|
"expert_frac_min": 0.06141716241836548,
|
|
"expert_frac_std": 0.0007075144094415009,
|
|
"grad_norm": 1.2961140871047974,
|
|
"learning_rate": 7.369727047146401e-05,
|
|
"loss": 9.5611,
|
|
"router_aux_loss": 2.003570516258478,
|
|
"router_entropy": 2.6998490092903378,
|
|
"router_entropy_norm": 0.9737646942130378,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.01858736059479554,
|
|
"expert_0_frac": 0.06209799647331238,
|
|
"expert_10_frac": 0.0627819076180458,
|
|
"expert_11_frac": 0.06212802231311798,
|
|
"expert_12_frac": 0.06245502457022667,
|
|
"expert_13_frac": 0.06238865107297897,
|
|
"expert_14_frac": 0.062404513359069824,
|
|
"expert_15_frac": 0.06227085739374161,
|
|
"expert_1_frac": 0.06282500177621841,
|
|
"expert_2_frac": 0.06209854781627655,
|
|
"expert_3_frac": 0.06289317458868027,
|
|
"expert_4_frac": 0.062445469200611115,
|
|
"expert_5_frac": 0.06195049360394478,
|
|
"expert_6_frac": 0.0627024918794632,
|
|
"expert_7_frac": 0.06268659979104996,
|
|
"expert_8_frac": 0.06310692429542542,
|
|
"expert_9_frac": 0.06276433914899826,
|
|
"expert_frac_max": 0.06310692429542542,
|
|
"expert_frac_min": 0.06195049360394478,
|
|
"expert_frac_std": 0.0003362393472343683,
|
|
"grad_norm": 0.5107415318489075,
|
|
"learning_rate": 0.00011091811414392058,
|
|
"loss": 8.448,
|
|
"router_aux_loss": 2.0038693606853486,
|
|
"router_entropy": 2.6251225757847227,
|
|
"router_entropy_norm": 0.9468128304525704,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.024783147459727387,
|
|
"expert_0_frac": 0.06296655535697937,
|
|
"expert_10_frac": 0.06291854381561279,
|
|
"expert_11_frac": 0.06274035573005676,
|
|
"expert_12_frac": 0.06212460994720459,
|
|
"expert_13_frac": 0.06220207363367081,
|
|
"expert_14_frac": 0.0623633973300457,
|
|
"expert_15_frac": 0.06261631846427917,
|
|
"expert_1_frac": 0.062234967947006226,
|
|
"expert_2_frac": 0.06269345432519913,
|
|
"expert_3_frac": 0.0623706579208374,
|
|
"expert_4_frac": 0.06255213916301727,
|
|
"expert_5_frac": 0.06247646361589432,
|
|
"expert_6_frac": 0.06270141154527664,
|
|
"expert_7_frac": 0.06233640015125275,
|
|
"expert_8_frac": 0.06229843944311142,
|
|
"expert_9_frac": 0.06240423396229744,
|
|
"expert_frac_max": 0.06296655535697937,
|
|
"expert_frac_min": 0.06212460994720459,
|
|
"expert_frac_std": 0.0002526248281355947,
|
|
"grad_norm": 0.5576089024543762,
|
|
"learning_rate": 0.00014813895781637715,
|
|
"loss": 7.7918,
|
|
"router_aux_loss": 2.0020354767143727,
|
|
"router_entropy": 2.575498506637911,
|
|
"router_entropy_norm": 0.9289147308358613,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.030978934324659233,
|
|
"expert_0_frac": 0.06254477053880692,
|
|
"expert_10_frac": 0.06230742111802101,
|
|
"expert_11_frac": 0.06271008402109146,
|
|
"expert_12_frac": 0.06274610757827759,
|
|
"expert_13_frac": 0.06249750405550003,
|
|
"expert_14_frac": 0.0624377578496933,
|
|
"expert_15_frac": 0.06226474419236183,
|
|
"expert_1_frac": 0.062305860221385956,
|
|
"expert_2_frac": 0.06292720884084702,
|
|
"expert_3_frac": 0.0619979128241539,
|
|
"expert_4_frac": 0.06242801249027252,
|
|
"expert_5_frac": 0.06247518211603165,
|
|
"expert_6_frac": 0.06265301257371902,
|
|
"expert_7_frac": 0.062180500477552414,
|
|
"expert_8_frac": 0.06276927888393402,
|
|
"expert_9_frac": 0.06275466829538345,
|
|
"expert_frac_max": 0.06292720884084702,
|
|
"expert_frac_min": 0.0619979128241539,
|
|
"expert_frac_std": 0.00025086093228310347,
|
|
"grad_norm": 0.7355545163154602,
|
|
"learning_rate": 0.0001853598014888337,
|
|
"loss": 7.1734,
|
|
"router_aux_loss": 2.00110543563962,
|
|
"router_entropy": 2.563297350977858,
|
|
"router_entropy_norm": 0.924514094144893,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.03717472118959108,
|
|
"expert_0_frac": 0.062286507338285446,
|
|
"expert_10_frac": 0.06241067498922348,
|
|
"expert_11_frac": 0.06250958144664764,
|
|
"expert_12_frac": 0.06269891560077667,
|
|
"expert_13_frac": 0.06233363598585129,
|
|
"expert_14_frac": 0.06255818158388138,
|
|
"expert_15_frac": 0.06255821138620377,
|
|
"expert_1_frac": 0.06232194975018501,
|
|
"expert_2_frac": 0.06297454982995987,
|
|
"expert_3_frac": 0.062439851462841034,
|
|
"expert_4_frac": 0.0625506192445755,
|
|
"expert_5_frac": 0.06263585388660431,
|
|
"expert_6_frac": 0.06239321082830429,
|
|
"expert_7_frac": 0.06247292459011078,
|
|
"expert_8_frac": 0.062302496284246445,
|
|
"expert_9_frac": 0.06255285441875458,
|
|
"expert_frac_max": 0.06297454982995987,
|
|
"expert_frac_min": 0.062286507338285446,
|
|
"expert_frac_std": 0.00017571607895661145,
|
|
"grad_norm": 0.7089310884475708,
|
|
"learning_rate": 0.0002225806451612903,
|
|
"loss": 6.616,
|
|
"router_aux_loss": 2.000896415114403,
|
|
"router_entropy": 2.5682343946397306,
|
|
"router_entropy_norm": 0.9262947562468021,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.04337050805452292,
|
|
"expert_0_frac": 0.06265521794557571,
|
|
"expert_10_frac": 0.06244245171546936,
|
|
"expert_11_frac": 0.06217898055911064,
|
|
"expert_12_frac": 0.06211438030004501,
|
|
"expert_13_frac": 0.06232894957065582,
|
|
"expert_14_frac": 0.06268642842769623,
|
|
"expert_15_frac": 0.062405042350292206,
|
|
"expert_1_frac": 0.06251642853021622,
|
|
"expert_2_frac": 0.06277015805244446,
|
|
"expert_3_frac": 0.06272362172603607,
|
|
"expert_4_frac": 0.06245952099561691,
|
|
"expert_5_frac": 0.06266545504331589,
|
|
"expert_6_frac": 0.06226136162877083,
|
|
"expert_7_frac": 0.06263869255781174,
|
|
"expert_8_frac": 0.06252796202898026,
|
|
"expert_9_frac": 0.06262536346912384,
|
|
"expert_frac_max": 0.06277015805244446,
|
|
"expert_frac_min": 0.06211438030004501,
|
|
"expert_frac_std": 0.00019952424918301404,
|
|
"grad_norm": 0.9155972003936768,
|
|
"learning_rate": 0.00025980148883374685,
|
|
"loss": 6.2012,
|
|
"router_aux_loss": 2.0007203175127506,
|
|
"router_entropy": 2.564730280985435,
|
|
"router_entropy_norm": 0.9250309143988612,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.04956629491945477,
|
|
"expert_0_frac": 0.06231584772467613,
|
|
"expert_10_frac": 0.06259948015213013,
|
|
"expert_11_frac": 0.06227356567978859,
|
|
"expert_12_frac": 0.0622558556497097,
|
|
"expert_13_frac": 0.06256835162639618,
|
|
"expert_14_frac": 0.06262826919555664,
|
|
"expert_15_frac": 0.062333691865205765,
|
|
"expert_1_frac": 0.06247744336724281,
|
|
"expert_2_frac": 0.062451884150505066,
|
|
"expert_3_frac": 0.06259673088788986,
|
|
"expert_4_frac": 0.062348831444978714,
|
|
"expert_5_frac": 0.06258165836334229,
|
|
"expert_6_frac": 0.06247778609395027,
|
|
"expert_7_frac": 0.06272567808628082,
|
|
"expert_8_frac": 0.06258414685726166,
|
|
"expert_9_frac": 0.06278080493211746,
|
|
"expert_frac_max": 0.06278080493211746,
|
|
"expert_frac_min": 0.0622558556497097,
|
|
"expert_frac_std": 0.00015969296509865671,
|
|
"grad_norm": 0.5280140042304993,
|
|
"learning_rate": 0.00029702233250620343,
|
|
"loss": 5.8599,
|
|
"router_aux_loss": 2.000566345155239,
|
|
"router_entropy": 2.5548819594830277,
|
|
"router_entropy_norm": 0.9214788832507104,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.055762081784386616,
|
|
"expert_0_frac": 0.06241057068109512,
|
|
"expert_10_frac": 0.06253797560930252,
|
|
"expert_11_frac": 0.062333617359399796,
|
|
"expert_12_frac": 0.06225384399294853,
|
|
"expert_13_frac": 0.062445636838674545,
|
|
"expert_14_frac": 0.062473755329847336,
|
|
"expert_15_frac": 0.062299177050590515,
|
|
"expert_1_frac": 0.062411118298769,
|
|
"expert_2_frac": 0.06231886148452759,
|
|
"expert_3_frac": 0.06262814998626709,
|
|
"expert_4_frac": 0.0625888928771019,
|
|
"expert_5_frac": 0.06266138702630997,
|
|
"expert_6_frac": 0.062470629811286926,
|
|
"expert_7_frac": 0.06291308254003525,
|
|
"expert_8_frac": 0.06242077797651291,
|
|
"expert_9_frac": 0.0628325492143631,
|
|
"expert_frac_max": 0.06291308254003525,
|
|
"expert_frac_min": 0.06225384399294853,
|
|
"expert_frac_std": 0.00018639311019796878,
|
|
"grad_norm": 0.6422482132911682,
|
|
"learning_rate": 0.00029997335517404445,
|
|
"loss": 5.571,
|
|
"router_aux_loss": 2.0004627220332623,
|
|
"router_entropy": 2.5414530691007773,
|
|
"router_entropy_norm": 0.9166354348609319,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.061957868649318466,
|
|
"expert_0_frac": 0.06254314631223679,
|
|
"expert_10_frac": 0.062319688498973846,
|
|
"expert_11_frac": 0.06252313405275345,
|
|
"expert_12_frac": 0.062282487750053406,
|
|
"expert_13_frac": 0.06232669949531555,
|
|
"expert_14_frac": 0.06230971962213516,
|
|
"expert_15_frac": 0.06244329735636711,
|
|
"expert_1_frac": 0.0624513179063797,
|
|
"expert_2_frac": 0.062395911663770676,
|
|
"expert_3_frac": 0.06261033564805984,
|
|
"expert_4_frac": 0.06252606958150864,
|
|
"expert_5_frac": 0.06252062320709229,
|
|
"expert_6_frac": 0.06251340359449387,
|
|
"expert_7_frac": 0.0628308430314064,
|
|
"expert_8_frac": 0.06259947270154953,
|
|
"expert_9_frac": 0.06280387192964554,
|
|
"expert_frac_max": 0.0628308430314064,
|
|
"expert_frac_min": 0.062282487750053406,
|
|
"expert_frac_std": 0.00016125103866215795,
|
|
"grad_norm": 0.8889395594596863,
|
|
"learning_rate": 0.00029988396298538336,
|
|
"loss": 5.3343,
|
|
"router_aux_loss": 2.000360265225172,
|
|
"router_entropy": 2.5253191737085583,
|
|
"router_entropy_norm": 0.9108163621427879,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.06815365551425032,
|
|
"expert_0_frac": 0.062405213713645935,
|
|
"expert_10_frac": 0.06261207163333893,
|
|
"expert_11_frac": 0.06235455349087715,
|
|
"expert_12_frac": 0.06230912730097771,
|
|
"expert_13_frac": 0.06251537799835205,
|
|
"expert_14_frac": 0.062493376433849335,
|
|
"expert_15_frac": 0.06233787536621094,
|
|
"expert_1_frac": 0.06240617856383324,
|
|
"expert_2_frac": 0.06229650601744652,
|
|
"expert_3_frac": 0.062494538724422455,
|
|
"expert_4_frac": 0.06250625848770142,
|
|
"expert_5_frac": 0.06260968744754791,
|
|
"expert_6_frac": 0.06259311735630035,
|
|
"expert_7_frac": 0.06282127648591995,
|
|
"expert_8_frac": 0.0624445416033268,
|
|
"expert_9_frac": 0.0628003180027008,
|
|
"expert_frac_max": 0.06282127648591995,
|
|
"expert_frac_min": 0.06229650601744652,
|
|
"expert_frac_std": 0.00015772653569001704,
|
|
"grad_norm": 0.5919621586799622,
|
|
"learning_rate": 0.00029973165947582927,
|
|
"loss": 5.1628,
|
|
"router_aux_loss": 2.0003816688060763,
|
|
"router_entropy": 2.508501745586594,
|
|
"router_entropy_norm": 0.9047507571047719,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.07434944237918216,
|
|
"expert_0_frac": 0.06238816678524017,
|
|
"expert_10_frac": 0.062436092644929886,
|
|
"expert_11_frac": 0.06248912960290909,
|
|
"expert_12_frac": 0.06231667846441269,
|
|
"expert_13_frac": 0.06243254989385605,
|
|
"expert_14_frac": 0.06253865361213684,
|
|
"expert_15_frac": 0.06226829066872597,
|
|
"expert_1_frac": 0.06243155896663666,
|
|
"expert_2_frac": 0.06230618804693222,
|
|
"expert_3_frac": 0.06257857382297516,
|
|
"expert_4_frac": 0.06255097687244415,
|
|
"expert_5_frac": 0.06232619285583496,
|
|
"expert_6_frac": 0.062459323555231094,
|
|
"expert_7_frac": 0.06313398480415344,
|
|
"expert_8_frac": 0.06253261864185333,
|
|
"expert_9_frac": 0.06281103938817978,
|
|
"expert_frac_max": 0.06313398480415344,
|
|
"expert_frac_min": 0.06226829066872597,
|
|
"expert_frac_std": 0.00021489191567525268,
|
|
"grad_norm": 0.6850535273551941,
|
|
"learning_rate": 0.0002995165085722677,
|
|
"loss": 5.0051,
|
|
"router_aux_loss": 2.0002849200367927,
|
|
"router_entropy": 2.4902097184211014,
|
|
"router_entropy_norm": 0.8981533028849062,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.080545229244114,
|
|
"expert_0_frac": 0.06241001933813095,
|
|
"expert_10_frac": 0.06275635212659836,
|
|
"expert_11_frac": 0.06241869181394577,
|
|
"expert_12_frac": 0.06214861199259758,
|
|
"expert_13_frac": 0.062489695847034454,
|
|
"expert_14_frac": 0.06236067786812782,
|
|
"expert_15_frac": 0.06246155500411987,
|
|
"expert_1_frac": 0.062489550560712814,
|
|
"expert_2_frac": 0.06250055134296417,
|
|
"expert_3_frac": 0.06265998631715775,
|
|
"expert_4_frac": 0.06249619275331497,
|
|
"expert_5_frac": 0.06254275888204575,
|
|
"expert_6_frac": 0.0625753328204155,
|
|
"expert_7_frac": 0.06273800879716873,
|
|
"expert_8_frac": 0.06239880248904228,
|
|
"expert_9_frac": 0.06255324184894562,
|
|
"expert_frac_max": 0.06275635212659836,
|
|
"expert_frac_min": 0.06214861199259758,
|
|
"expert_frac_std": 0.0001478375488659367,
|
|
"grad_norm": 0.5297462940216064,
|
|
"learning_rate": 0.0002992386005807413,
|
|
"loss": 4.8795,
|
|
"router_aux_loss": 2.0003453426063063,
|
|
"router_entropy": 2.472862062752247,
|
|
"router_entropy_norm": 0.8918964586837799,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.08674101610904585,
|
|
"expert_0_frac": 0.06242035701870918,
|
|
"expert_10_frac": 0.06262128055095673,
|
|
"expert_11_frac": 0.06240040436387062,
|
|
"expert_12_frac": 0.06226507946848869,
|
|
"expert_13_frac": 0.062457650899887085,
|
|
"expert_14_frac": 0.06226836517453194,
|
|
"expert_15_frac": 0.06225299835205078,
|
|
"expert_1_frac": 0.06253058463335037,
|
|
"expert_2_frac": 0.062489911913871765,
|
|
"expert_3_frac": 0.06253976374864578,
|
|
"expert_4_frac": 0.06257753819227219,
|
|
"expert_5_frac": 0.062373727560043335,
|
|
"expert_6_frac": 0.06251810491085052,
|
|
"expert_7_frac": 0.06306448578834534,
|
|
"expert_8_frac": 0.062572181224823,
|
|
"expert_9_frac": 0.06264758855104446,
|
|
"expert_frac_max": 0.06306448578834534,
|
|
"expert_frac_min": 0.06225299835205078,
|
|
"expert_frac_std": 0.00019547491683624685,
|
|
"grad_norm": 0.5590657591819763,
|
|
"learning_rate": 0.00029889805214854495,
|
|
"loss": 4.7772,
|
|
"router_aux_loss": 2.0003161118924617,
|
|
"router_entropy": 2.454501044129332,
|
|
"router_entropy_norm": 0.8852741210555425,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.09293680297397769,
|
|
"expert_0_frac": 0.062205202877521515,
|
|
"expert_10_frac": 0.06256791949272156,
|
|
"expert_11_frac": 0.06266215443611145,
|
|
"expert_12_frac": 0.062390055507421494,
|
|
"expert_13_frac": 0.06255996972322464,
|
|
"expert_14_frac": 0.06251850724220276,
|
|
"expert_15_frac": 0.06232086196541786,
|
|
"expert_1_frac": 0.06241616606712341,
|
|
"expert_2_frac": 0.062427859753370285,
|
|
"expert_3_frac": 0.06252457201480865,
|
|
"expert_4_frac": 0.06252304464578629,
|
|
"expert_5_frac": 0.06245594844222069,
|
|
"expert_6_frac": 0.06254767626523972,
|
|
"expert_7_frac": 0.06271296739578247,
|
|
"expert_8_frac": 0.06264864653348923,
|
|
"expert_9_frac": 0.06251845508813858,
|
|
"expert_frac_max": 0.06271296739578247,
|
|
"expert_frac_min": 0.062205202877521515,
|
|
"expert_frac_std": 0.00012943513866048306,
|
|
"grad_norm": 0.49839961528778076,
|
|
"learning_rate": 0.00029849500621526514,
|
|
"loss": 4.6889,
|
|
"router_aux_loss": 2.0003180173039437,
|
|
"router_entropy": 2.4365353710452715,
|
|
"router_entropy_norm": 0.8787943741893909,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.09913258983890955,
|
|
"expert_0_frac": 0.06250069290399551,
|
|
"expert_10_frac": 0.06257562339305878,
|
|
"expert_11_frac": 0.06249527633190155,
|
|
"expert_12_frac": 0.0622064545750618,
|
|
"expert_13_frac": 0.06257451325654984,
|
|
"expert_14_frac": 0.06233534589409828,
|
|
"expert_15_frac": 0.06241418421268463,
|
|
"expert_1_frac": 0.06264199316501617,
|
|
"expert_2_frac": 0.06249883770942688,
|
|
"expert_3_frac": 0.06250212341547012,
|
|
"expert_4_frac": 0.0625748336315155,
|
|
"expert_5_frac": 0.062375910580158234,
|
|
"expert_6_frac": 0.06260207295417786,
|
|
"expert_7_frac": 0.06282484531402588,
|
|
"expert_8_frac": 0.06244645640254021,
|
|
"expert_9_frac": 0.062430866062641144,
|
|
"expert_frac_max": 0.06282484531402588,
|
|
"expert_frac_min": 0.0622064545750618,
|
|
"expert_frac_std": 0.0001406399969710037,
|
|
"grad_norm": 0.5385776162147522,
|
|
"learning_rate": 0.0002980296319527835,
|
|
"loss": 4.6125,
|
|
"router_aux_loss": 2.000329566299915,
|
|
"router_entropy": 2.4200733024130265,
|
|
"router_entropy_norm": 0.8728569379947625,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.1,
|
|
"eval_loss": 3.9063076972961426,
|
|
"eval_runtime": 34.3694,
|
|
"eval_samples_per_second": 50.277,
|
|
"eval_steps_per_second": 6.285,
|
|
"expert_0_frac": 0.06256858259439468,
|
|
"expert_10_frac": 0.06268621981143951,
|
|
"expert_11_frac": 0.06197136640548706,
|
|
"expert_12_frac": 0.062151338905096054,
|
|
"expert_13_frac": 0.06156676262617111,
|
|
"expert_14_frac": 0.062240924686193466,
|
|
"expert_15_frac": 0.06298670917749405,
|
|
"expert_1_frac": 0.0629330724477768,
|
|
"expert_2_frac": 0.06259055435657501,
|
|
"expert_3_frac": 0.06298832595348358,
|
|
"expert_4_frac": 0.062575563788414,
|
|
"expert_5_frac": 0.06270726025104523,
|
|
"expert_6_frac": 0.06232661381363869,
|
|
"expert_7_frac": 0.062258243560791016,
|
|
"expert_8_frac": 0.06328871101140976,
|
|
"expert_9_frac": 0.0621596984565258,
|
|
"expert_frac_max": 0.06328871101140976,
|
|
"expert_frac_min": 0.06156676262617111,
|
|
"expert_frac_std": 0.0004415576404426247,
|
|
"router_aux_loss": 2.000288353824034,
|
|
"router_entropy": 2.413784189740332,
|
|
"router_entropy_norm": 0.8705886200786405,
|
|
"step": 807
|
|
},
|
|
{
|
|
"epoch": 0.10532837670384139,
|
|
"expert_0_frac": 0.06222734600305557,
|
|
"expert_10_frac": 0.0625881776213646,
|
|
"expert_11_frac": 0.06228402629494667,
|
|
"expert_12_frac": 0.06230998784303665,
|
|
"expert_13_frac": 0.06273674964904785,
|
|
"expert_14_frac": 0.06251049786806107,
|
|
"expert_15_frac": 0.062339041382074356,
|
|
"expert_1_frac": 0.06267312169075012,
|
|
"expert_2_frac": 0.06258376687765121,
|
|
"expert_3_frac": 0.06246174871921539,
|
|
"expert_4_frac": 0.06258857995271683,
|
|
"expert_5_frac": 0.06243951991200447,
|
|
"expert_6_frac": 0.06259816139936447,
|
|
"expert_7_frac": 0.06267885863780975,
|
|
"expert_8_frac": 0.06258304417133331,
|
|
"expert_9_frac": 0.062397364526987076,
|
|
"expert_frac_max": 0.06273674964904785,
|
|
"expert_frac_min": 0.06222734600305557,
|
|
"expert_frac_std": 0.00015404434816446155,
|
|
"grad_norm": 0.5590858459472656,
|
|
"learning_rate": 0.0002975021246942697,
|
|
"loss": 4.5451,
|
|
"router_aux_loss": 2.00037628342939,
|
|
"router_entropy": 2.4037045948784943,
|
|
"router_entropy_norm": 0.8669531746983047,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.11152416356877323,
|
|
"expert_0_frac": 0.06237448379397392,
|
|
"expert_10_frac": 0.06260498613119125,
|
|
"expert_11_frac": 0.06246993690729141,
|
|
"expert_12_frac": 0.06227007135748863,
|
|
"expert_13_frac": 0.06224007532000542,
|
|
"expert_14_frac": 0.06248144805431366,
|
|
"expert_15_frac": 0.06225846707820892,
|
|
"expert_1_frac": 0.06252281367778778,
|
|
"expert_2_frac": 0.06251576542854309,
|
|
"expert_3_frac": 0.06272197514772415,
|
|
"expert_4_frac": 0.06268062442541122,
|
|
"expert_5_frac": 0.06230660900473595,
|
|
"expert_6_frac": 0.06255774199962616,
|
|
"expert_7_frac": 0.06274865567684174,
|
|
"expert_8_frac": 0.06259787082672119,
|
|
"expert_9_frac": 0.06264849752187729,
|
|
"expert_frac_max": 0.06274865567684174,
|
|
"expert_frac_min": 0.06224007532000542,
|
|
"expert_frac_std": 0.00016805788618512452,
|
|
"grad_norm": 0.511800229549408,
|
|
"learning_rate": 0.0002969127058521934,
|
|
"loss": 4.4903,
|
|
"router_aux_loss": 2.00034979224205,
|
|
"router_entropy": 2.3888362085570893,
|
|
"router_entropy_norm": 0.8615905378953266,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.11771995043370508,
|
|
"expert_0_frac": 0.06221815198659897,
|
|
"expert_10_frac": 0.06260684877634048,
|
|
"expert_11_frac": 0.06247435510158539,
|
|
"expert_12_frac": 0.062245357781648636,
|
|
"expert_13_frac": 0.06263848394155502,
|
|
"expert_14_frac": 0.06225128471851349,
|
|
"expert_15_frac": 0.06243528053164482,
|
|
"expert_1_frac": 0.06256310641765594,
|
|
"expert_2_frac": 0.0625167116522789,
|
|
"expert_3_frac": 0.0626605972647667,
|
|
"expert_4_frac": 0.06259691715240479,
|
|
"expert_5_frac": 0.06243862956762314,
|
|
"expert_6_frac": 0.0625225305557251,
|
|
"expert_7_frac": 0.0627313181757927,
|
|
"expert_8_frac": 0.06271545588970184,
|
|
"expert_9_frac": 0.06238499656319618,
|
|
"expert_frac_max": 0.0627313181757927,
|
|
"expert_frac_min": 0.06221815198659897,
|
|
"expert_frac_std": 0.00016298388072755188,
|
|
"grad_norm": 0.5196658372879028,
|
|
"learning_rate": 0.00029626162282539014,
|
|
"loss": 4.4278,
|
|
"router_aux_loss": 2.0002928851544857,
|
|
"router_entropy": 2.3760699312140545,
|
|
"router_entropy_norm": 0.8569860766419742,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.12391573729863693,
|
|
"expert_0_frac": 0.062374427914619446,
|
|
"expert_10_frac": 0.06274916976690292,
|
|
"expert_11_frac": 0.0625402107834816,
|
|
"expert_12_frac": 0.06249481812119484,
|
|
"expert_13_frac": 0.06247967481613159,
|
|
"expert_14_frac": 0.062247175723314285,
|
|
"expert_15_frac": 0.0623476505279541,
|
|
"expert_1_frac": 0.06259554624557495,
|
|
"expert_2_frac": 0.062494710087776184,
|
|
"expert_3_frac": 0.062440454959869385,
|
|
"expert_4_frac": 0.06264761835336685,
|
|
"expert_5_frac": 0.062286071479320526,
|
|
"expert_6_frac": 0.06263621896505356,
|
|
"expert_7_frac": 0.06261817365884781,
|
|
"expert_8_frac": 0.06239589303731918,
|
|
"expert_9_frac": 0.06265220791101456,
|
|
"expert_frac_max": 0.06274916976690292,
|
|
"expert_frac_min": 0.062247175723314285,
|
|
"expert_frac_std": 0.0001446436217520386,
|
|
"grad_norm": 0.5525915026664734,
|
|
"learning_rate": 0.00029554914889521987,
|
|
"loss": 4.3662,
|
|
"router_aux_loss": 2.0003440357744693,
|
|
"router_entropy": 2.3614482231934866,
|
|
"router_entropy_norm": 0.8517124102293243,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.13011152416356878,
|
|
"expert_0_frac": 0.06243520230054855,
|
|
"expert_10_frac": 0.062535859644413,
|
|
"expert_11_frac": 0.06255455315113068,
|
|
"expert_12_frac": 0.06228669732809067,
|
|
"expert_13_frac": 0.06247691437602043,
|
|
"expert_14_frac": 0.0624634250998497,
|
|
"expert_15_frac": 0.06220926344394684,
|
|
"expert_1_frac": 0.062498826533555984,
|
|
"expert_2_frac": 0.06245774030685425,
|
|
"expert_3_frac": 0.06270050257444382,
|
|
"expert_4_frac": 0.06269151717424393,
|
|
"expert_5_frac": 0.06252426654100418,
|
|
"expert_6_frac": 0.06250670552253723,
|
|
"expert_7_frac": 0.062534399330616,
|
|
"expert_8_frac": 0.06266549229621887,
|
|
"expert_9_frac": 0.06245866417884827,
|
|
"expert_frac_max": 0.06270050257444382,
|
|
"expert_frac_min": 0.06220926344394684,
|
|
"expert_frac_std": 0.00012908343342132866,
|
|
"grad_norm": 0.5066197514533997,
|
|
"learning_rate": 0.00029477558311086135,
|
|
"loss": 4.3252,
|
|
"router_aux_loss": 2.000340888053179,
|
|
"router_entropy": 2.3474042297899724,
|
|
"router_entropy_norm": 0.8466471103199426,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.13630731102850063,
|
|
"expert_0_frac": 0.06234085187315941,
|
|
"expert_10_frac": 0.06262367963790894,
|
|
"expert_11_frac": 0.062475550919771194,
|
|
"expert_12_frac": 0.06233123317360878,
|
|
"expert_13_frac": 0.06273722648620605,
|
|
"expert_14_frac": 0.062331944704055786,
|
|
"expert_15_frac": 0.06239987164735794,
|
|
"expert_1_frac": 0.0627233237028122,
|
|
"expert_2_frac": 0.06237931549549103,
|
|
"expert_3_frac": 0.06261306256055832,
|
|
"expert_4_frac": 0.06256913393735886,
|
|
"expert_5_frac": 0.06224743649363518,
|
|
"expert_6_frac": 0.0626690536737442,
|
|
"expert_7_frac": 0.06252788007259369,
|
|
"expert_8_frac": 0.0625937357544899,
|
|
"expert_9_frac": 0.06243671476840973,
|
|
"expert_frac_max": 0.06273722648620605,
|
|
"expert_frac_min": 0.06224743649363518,
|
|
"expert_frac_std": 0.0001532289170427248,
|
|
"grad_norm": 0.6175411939620972,
|
|
"learning_rate": 0.0002939412501637914,
|
|
"loss": 4.2875,
|
|
"router_aux_loss": 2.000367415100336,
|
|
"router_entropy": 2.33591702952981,
|
|
"router_entropy_norm": 0.8425039786076838,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.14250309789343246,
|
|
"expert_0_frac": 0.06229696422815323,
|
|
"expert_10_frac": 0.06262972950935364,
|
|
"expert_11_frac": 0.062427107244729996,
|
|
"expert_12_frac": 0.06219984218478203,
|
|
"expert_13_frac": 0.06254806369543076,
|
|
"expert_14_frac": 0.06229260936379433,
|
|
"expert_15_frac": 0.062417853623628616,
|
|
"expert_1_frac": 0.06263617426156998,
|
|
"expert_2_frac": 0.06263332813978195,
|
|
"expert_3_frac": 0.06255980581045151,
|
|
"expert_4_frac": 0.06264110654592514,
|
|
"expert_5_frac": 0.06253870576620102,
|
|
"expert_6_frac": 0.06249291077256203,
|
|
"expert_7_frac": 0.06253132969141006,
|
|
"expert_8_frac": 0.06262201815843582,
|
|
"expert_9_frac": 0.06253247708082199,
|
|
"expert_frac_max": 0.06264110654592514,
|
|
"expert_frac_min": 0.06219984218478203,
|
|
"expert_frac_std": 0.00013755267718806863,
|
|
"grad_norm": 0.5854677557945251,
|
|
"learning_rate": 0.00029304650025150096,
|
|
"loss": 4.2579,
|
|
"router_aux_loss": 2.00031850785017,
|
|
"router_entropy": 2.323672557796041,
|
|
"router_entropy_norm": 0.8380877189455305,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.14869888475836432,
|
|
"expert_0_frac": 0.06235099211335182,
|
|
"expert_10_frac": 0.06260727345943451,
|
|
"expert_11_frac": 0.06260436028242111,
|
|
"expert_12_frac": 0.06245154142379761,
|
|
"expert_13_frac": 0.06255651265382767,
|
|
"expert_14_frac": 0.062305666506290436,
|
|
"expert_15_frac": 0.06230290234088898,
|
|
"expert_1_frac": 0.06253120303153992,
|
|
"expert_2_frac": 0.06256424635648727,
|
|
"expert_3_frac": 0.06265949457883835,
|
|
"expert_4_frac": 0.0625964030623436,
|
|
"expert_5_frac": 0.0623132660984993,
|
|
"expert_6_frac": 0.06253556907176971,
|
|
"expert_7_frac": 0.06253494322299957,
|
|
"expert_8_frac": 0.06250181794166565,
|
|
"expert_9_frac": 0.06258382648229599,
|
|
"expert_frac_max": 0.06265949457883835,
|
|
"expert_frac_min": 0.06230290234088898,
|
|
"expert_frac_std": 0.00011859627556987107,
|
|
"grad_norm": 0.46354418992996216,
|
|
"learning_rate": 0.00029209170893050536,
|
|
"loss": 4.2219,
|
|
"router_aux_loss": 2.0003797383606434,
|
|
"router_entropy": 2.3109171950817107,
|
|
"router_entropy_norm": 0.8334871943123543,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.15489467162329615,
|
|
"expert_0_frac": 0.062493301928043365,
|
|
"expert_10_frac": 0.06252740323543549,
|
|
"expert_11_frac": 0.06238264963030815,
|
|
"expert_12_frac": 0.062336668372154236,
|
|
"expert_13_frac": 0.062471043318510056,
|
|
"expert_14_frac": 0.062253303825855255,
|
|
"expert_15_frac": 0.06223702430725098,
|
|
"expert_1_frac": 0.06259184330701828,
|
|
"expert_2_frac": 0.06251081079244614,
|
|
"expert_3_frac": 0.0626404732465744,
|
|
"expert_4_frac": 0.06271036714315414,
|
|
"expert_5_frac": 0.06241315230727196,
|
|
"expert_6_frac": 0.06260694563388824,
|
|
"expert_7_frac": 0.06258156895637512,
|
|
"expert_8_frac": 0.06270106136798859,
|
|
"expert_9_frac": 0.06254240870475769,
|
|
"expert_frac_max": 0.06271036714315414,
|
|
"expert_frac_min": 0.06223702430725098,
|
|
"expert_frac_std": 0.00014433629985433072,
|
|
"grad_norm": 0.44044944643974304,
|
|
"learning_rate": 0.00029107727695871084,
|
|
"loss": 4.1886,
|
|
"router_aux_loss": 2.0003322799503804,
|
|
"router_entropy": 2.300990598698457,
|
|
"router_entropy_norm": 0.8299069314685977,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.161090458488228,
|
|
"expert_0_frac": 0.062284037470817566,
|
|
"expert_10_frac": 0.06267246603965759,
|
|
"expert_11_frac": 0.06254246830940247,
|
|
"expert_12_frac": 0.062176335602998734,
|
|
"expert_13_frac": 0.06258271634578705,
|
|
"expert_14_frac": 0.06226383522152901,
|
|
"expert_15_frac": 0.062295541167259216,
|
|
"expert_1_frac": 0.06266821175813675,
|
|
"expert_2_frac": 0.06251491606235504,
|
|
"expert_3_frac": 0.06266596168279648,
|
|
"expert_4_frac": 0.06262225657701492,
|
|
"expert_5_frac": 0.062298376113176346,
|
|
"expert_6_frac": 0.06268490105867386,
|
|
"expert_7_frac": 0.0625143051147461,
|
|
"expert_8_frac": 0.06259044259786606,
|
|
"expert_9_frac": 0.06262325495481491,
|
|
"expert_frac_max": 0.06268490105867386,
|
|
"expert_frac_min": 0.062176335602998734,
|
|
"expert_frac_std": 0.00017450697487220168,
|
|
"grad_norm": 0.4276027977466583,
|
|
"learning_rate": 0.00029000363012720296,
|
|
"loss": 4.1441,
|
|
"router_aux_loss": 2.0003066021203995,
|
|
"router_entropy": 2.289357506732146,
|
|
"router_entropy_norm": 0.8257111804460971,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.16728624535315986,
|
|
"expert_0_frac": 0.06231163814663887,
|
|
"expert_10_frac": 0.06265905499458313,
|
|
"expert_11_frac": 0.06236410140991211,
|
|
"expert_12_frac": 0.062274329364299774,
|
|
"expert_13_frac": 0.06251786649227142,
|
|
"expert_14_frac": 0.06248300522565842,
|
|
"expert_15_frac": 0.06248318403959274,
|
|
"expert_1_frac": 0.06256091594696045,
|
|
"expert_2_frac": 0.06251896172761917,
|
|
"expert_3_frac": 0.06250979751348495,
|
|
"expert_4_frac": 0.06253128498792648,
|
|
"expert_5_frac": 0.06237177178263664,
|
|
"expert_6_frac": 0.06275496631860733,
|
|
"expert_7_frac": 0.06262163072824478,
|
|
"expert_8_frac": 0.06238657608628273,
|
|
"expert_9_frac": 0.06265093386173248,
|
|
"expert_frac_max": 0.06275496631860733,
|
|
"expert_frac_min": 0.062274329364299774,
|
|
"expert_frac_std": 0.00013369132648222148,
|
|
"grad_norm": 0.5024176836013794,
|
|
"learning_rate": 0.0002888712190815277,
|
|
"loss": 4.1224,
|
|
"router_aux_loss": 2.000337022989988,
|
|
"router_entropy": 2.2781822363535564,
|
|
"router_entropy_norm": 0.8216805536571511,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.1734820322180917,
|
|
"expert_0_frac": 0.06237514689564705,
|
|
"expert_10_frac": 0.062498342245817184,
|
|
"expert_11_frac": 0.06249630078673363,
|
|
"expert_12_frac": 0.06231970712542534,
|
|
"expert_13_frac": 0.06248846277594566,
|
|
"expert_14_frac": 0.062270451337099075,
|
|
"expert_15_frac": 0.06256610155105591,
|
|
"expert_1_frac": 0.06258639693260193,
|
|
"expert_2_frac": 0.0625196322798729,
|
|
"expert_3_frac": 0.06262469291687012,
|
|
"expert_4_frac": 0.06277842074632645,
|
|
"expert_5_frac": 0.06232196465134621,
|
|
"expert_6_frac": 0.06267911195755005,
|
|
"expert_7_frac": 0.06257634609937668,
|
|
"expert_8_frac": 0.06246921792626381,
|
|
"expert_9_frac": 0.062429722398519516,
|
|
"expert_frac_max": 0.06277842074632645,
|
|
"expert_frac_min": 0.062270451337099075,
|
|
"expert_frac_std": 0.0001370436802972108,
|
|
"grad_norm": 0.42523810267448425,
|
|
"learning_rate": 0.0002876805191325406,
|
|
"loss": 4.087,
|
|
"router_aux_loss": 2.0003104132413863,
|
|
"router_entropy": 2.2699872746318577,
|
|
"router_entropy_norm": 0.8187248459981087,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.17967781908302355,
|
|
"expert_0_frac": 0.062234066426754,
|
|
"expert_10_frac": 0.06256188452243805,
|
|
"expert_11_frac": 0.06254622340202332,
|
|
"expert_12_frac": 0.06240968033671379,
|
|
"expert_13_frac": 0.06278626620769501,
|
|
"expert_14_frac": 0.06218487024307251,
|
|
"expert_15_frac": 0.06230517849326134,
|
|
"expert_1_frac": 0.06253640353679657,
|
|
"expert_2_frac": 0.0625469759106636,
|
|
"expert_3_frac": 0.06253714859485626,
|
|
"expert_4_frac": 0.06265972554683685,
|
|
"expert_5_frac": 0.06244716793298721,
|
|
"expert_6_frac": 0.06260564923286438,
|
|
"expert_7_frac": 0.06254668533802032,
|
|
"expert_8_frac": 0.06246053799986839,
|
|
"expert_9_frac": 0.06263155490159988,
|
|
"expert_frac_max": 0.06278626620769501,
|
|
"expert_frac_min": 0.06218487024307251,
|
|
"expert_frac_std": 0.00015694495232310146,
|
|
"grad_norm": 0.39899736642837524,
|
|
"learning_rate": 0.00028643203005690286,
|
|
"loss": 4.0667,
|
|
"router_aux_loss": 2.000341858714819,
|
|
"router_entropy": 2.26084908430775,
|
|
"router_entropy_norm": 0.8154289405322862,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.18587360594795538,
|
|
"expert_0_frac": 0.062320563942193985,
|
|
"expert_10_frac": 0.06271921843290329,
|
|
"expert_11_frac": 0.06238391250371933,
|
|
"expert_12_frac": 0.06240151822566986,
|
|
"expert_13_frac": 0.06269849836826324,
|
|
"expert_14_frac": 0.062356676906347275,
|
|
"expert_15_frac": 0.06238163262605667,
|
|
"expert_1_frac": 0.0625913217663765,
|
|
"expert_2_frac": 0.06255612522363663,
|
|
"expert_3_frac": 0.0625518411397934,
|
|
"expert_4_frac": 0.06254532933235168,
|
|
"expert_5_frac": 0.06224118173122406,
|
|
"expert_6_frac": 0.06265821307897568,
|
|
"expert_7_frac": 0.06254154443740845,
|
|
"expert_8_frac": 0.06240622326731682,
|
|
"expert_9_frac": 0.06264621019363403,
|
|
"expert_frac_max": 0.06271921843290329,
|
|
"expert_frac_min": 0.06224118173122406,
|
|
"expert_frac_std": 0.00014549469051416963,
|
|
"grad_norm": 0.41512852907180786,
|
|
"learning_rate": 0.00028512627588730814,
|
|
"loss": 4.0337,
|
|
"router_aux_loss": 2.0003461696207525,
|
|
"router_entropy": 2.2523704461505014,
|
|
"router_entropy_norm": 0.8123709182265476,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.19206939281288724,
|
|
"expert_0_frac": 0.062392860651016235,
|
|
"expert_10_frac": 0.062398917973041534,
|
|
"expert_11_frac": 0.06255253404378891,
|
|
"expert_12_frac": 0.06235002726316452,
|
|
"expert_13_frac": 0.06256391853094101,
|
|
"expert_14_frac": 0.06229909881949425,
|
|
"expert_15_frac": 0.062403980642557144,
|
|
"expert_1_frac": 0.0625167191028595,
|
|
"expert_2_frac": 0.06252924352884293,
|
|
"expert_3_frac": 0.06263791769742966,
|
|
"expert_4_frac": 0.06260311603546143,
|
|
"expert_5_frac": 0.06242026016116142,
|
|
"expert_6_frac": 0.0625079870223999,
|
|
"expert_7_frac": 0.06265029311180115,
|
|
"expert_8_frac": 0.06259024143218994,
|
|
"expert_9_frac": 0.06258291751146317,
|
|
"expert_frac_max": 0.06265029311180115,
|
|
"expert_frac_min": 0.06229909881949425,
|
|
"expert_frac_std": 0.00010814578126883134,
|
|
"grad_norm": 0.5054246783256531,
|
|
"learning_rate": 0.0002837638046925287,
|
|
"loss": 4.0077,
|
|
"router_aux_loss": 2.0003621953725816,
|
|
"router_entropy": 2.2446445197115343,
|
|
"router_entropy_norm": 0.809584379286605,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.1982651796778191,
|
|
"expert_0_frac": 0.06226435676217079,
|
|
"expert_10_frac": 0.06269991397857666,
|
|
"expert_11_frac": 0.0623975433409214,
|
|
"expert_12_frac": 0.06223442777991295,
|
|
"expert_13_frac": 0.0626256987452507,
|
|
"expert_14_frac": 0.06223084405064583,
|
|
"expert_15_frac": 0.06239727884531021,
|
|
"expert_1_frac": 0.06264633685350418,
|
|
"expert_2_frac": 0.06251941621303558,
|
|
"expert_3_frac": 0.06264705210924149,
|
|
"expert_4_frac": 0.06256012618541718,
|
|
"expert_5_frac": 0.062397200614213943,
|
|
"expert_6_frac": 0.06272200495004654,
|
|
"expert_7_frac": 0.06254942715167999,
|
|
"expert_8_frac": 0.06245889142155647,
|
|
"expert_9_frac": 0.06264950335025787,
|
|
"expert_frac_max": 0.06272200495004654,
|
|
"expert_frac_min": 0.06223084405064583,
|
|
"expert_frac_std": 0.0001650626363698393,
|
|
"grad_norm": 0.5019674301147461,
|
|
"learning_rate": 0.0002823451883473724,
|
|
"loss": 3.9899,
|
|
"router_aux_loss": 2.000300102084875,
|
|
"router_entropy": 2.23768438488245,
|
|
"router_entropy_norm": 0.8070740412861453,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.2,
|
|
"eval_loss": 3.348736524581909,
|
|
"eval_runtime": 34.4759,
|
|
"eval_samples_per_second": 50.122,
|
|
"eval_steps_per_second": 6.265,
|
|
"expert_0_frac": 0.062293633818626404,
|
|
"expert_10_frac": 0.06241512671113014,
|
|
"expert_11_frac": 0.06290776282548904,
|
|
"expert_12_frac": 0.06183137372136116,
|
|
"expert_13_frac": 0.06233253329992294,
|
|
"expert_14_frac": 0.06204337999224663,
|
|
"expert_15_frac": 0.06220891326665878,
|
|
"expert_1_frac": 0.06302127987146378,
|
|
"expert_2_frac": 0.062322936952114105,
|
|
"expert_3_frac": 0.062286339700222015,
|
|
"expert_4_frac": 0.06292441487312317,
|
|
"expert_5_frac": 0.06253024935722351,
|
|
"expert_6_frac": 0.06261050701141357,
|
|
"expert_7_frac": 0.06317538768053055,
|
|
"expert_8_frac": 0.06250172108411789,
|
|
"expert_9_frac": 0.06259440630674362,
|
|
"expert_frac_max": 0.06317538768053055,
|
|
"expert_frac_min": 0.06183137372136116,
|
|
"expert_frac_std": 0.0003643674135673791,
|
|
"router_aux_loss": 2.000175356864929,
|
|
"router_entropy": 2.2309101776191684,
|
|
"router_entropy_norm": 0.8046307624799728,
|
|
"step": 1614
|
|
},
|
|
{
|
|
"epoch": 0.20446096654275092,
|
|
"expert_0_frac": 0.06241779029369354,
|
|
"expert_10_frac": 0.06254168599843979,
|
|
"expert_11_frac": 0.06247587502002716,
|
|
"expert_12_frac": 0.062394991517066956,
|
|
"expert_13_frac": 0.06247551366686821,
|
|
"expert_14_frac": 0.06237455829977989,
|
|
"expert_15_frac": 0.06222521886229515,
|
|
"expert_1_frac": 0.06245426833629608,
|
|
"expert_2_frac": 0.06267520040273666,
|
|
"expert_3_frac": 0.06251148879528046,
|
|
"expert_4_frac": 0.06266897916793823,
|
|
"expert_5_frac": 0.06242360174655914,
|
|
"expert_6_frac": 0.06273163110017776,
|
|
"expert_7_frac": 0.06260820478200912,
|
|
"expert_8_frac": 0.06258939206600189,
|
|
"expert_9_frac": 0.062431611120700836,
|
|
"expert_frac_max": 0.06273163110017776,
|
|
"expert_frac_min": 0.06222521886229515,
|
|
"expert_frac_std": 0.0001309664803557098,
|
|
"grad_norm": 0.44987165927886963,
|
|
"learning_rate": 0.00028087102229264795,
|
|
"loss": 3.9507,
|
|
"router_aux_loss": 2.0002999247776136,
|
|
"router_entropy": 2.228416964687683,
|
|
"router_entropy_norm": 0.8037315259969391,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.21065675340768278,
|
|
"expert_0_frac": 0.062353696674108505,
|
|
"expert_10_frac": 0.0626240074634552,
|
|
"expert_11_frac": 0.06262946128845215,
|
|
"expert_12_frac": 0.062432609498500824,
|
|
"expert_13_frac": 0.06261216104030609,
|
|
"expert_14_frac": 0.06230517104268074,
|
|
"expert_15_frac": 0.06234478950500488,
|
|
"expert_1_frac": 0.06255390495061874,
|
|
"expert_2_frac": 0.062430791556835175,
|
|
"expert_3_frac": 0.06261441111564636,
|
|
"expert_4_frac": 0.06257495284080505,
|
|
"expert_5_frac": 0.06230694428086281,
|
|
"expert_6_frac": 0.06264862418174744,
|
|
"expert_7_frac": 0.0625498816370964,
|
|
"expert_8_frac": 0.0624498575925827,
|
|
"expert_9_frac": 0.0625687763094902,
|
|
"expert_frac_max": 0.06264862418174744,
|
|
"expert_frac_min": 0.06230517104268074,
|
|
"expert_frac_std": 0.00012334466737229377,
|
|
"grad_norm": 0.4263043701648712,
|
|
"learning_rate": 0.00027934192528523803,
|
|
"loss": 3.9304,
|
|
"router_aux_loss": 2.0003829108178617,
|
|
"router_entropy": 2.2245351150631905,
|
|
"router_entropy_norm": 0.8023314446962562,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.21685254027261464,
|
|
"expert_0_frac": 0.06229822337627411,
|
|
"expert_10_frac": 0.06234228238463402,
|
|
"expert_11_frac": 0.062404099851846695,
|
|
"expert_12_frac": 0.06231901794672012,
|
|
"expert_13_frac": 0.06277892738580704,
|
|
"expert_14_frac": 0.06221300736069679,
|
|
"expert_15_frac": 0.062457721680402756,
|
|
"expert_1_frac": 0.06249331310391426,
|
|
"expert_2_frac": 0.062426842749118805,
|
|
"expert_3_frac": 0.06263561546802521,
|
|
"expert_4_frac": 0.0626637190580368,
|
|
"expert_5_frac": 0.062459539622068405,
|
|
"expert_6_frac": 0.0625869482755661,
|
|
"expert_7_frac": 0.06273391097784042,
|
|
"expert_8_frac": 0.06258323788642883,
|
|
"expert_9_frac": 0.06260362267494202,
|
|
"expert_frac_max": 0.06277892738580704,
|
|
"expert_frac_min": 0.06221300736069679,
|
|
"expert_frac_std": 0.00016363832401111722,
|
|
"grad_norm": 0.44618475437164307,
|
|
"learning_rate": 0.0002777585391383867,
|
|
"loss": 3.9089,
|
|
"router_aux_loss": 2.0003176471590995,
|
|
"router_entropy": 2.216816434636712,
|
|
"router_entropy_norm": 0.7995475192028844,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.22304832713754646,
|
|
"expert_0_frac": 0.06226443871855736,
|
|
"expert_10_frac": 0.06262083351612091,
|
|
"expert_11_frac": 0.06259326636791229,
|
|
"expert_12_frac": 0.0622016042470932,
|
|
"expert_13_frac": 0.06253740191459656,
|
|
"expert_14_frac": 0.06229192763566971,
|
|
"expert_15_frac": 0.06249629333615303,
|
|
"expert_1_frac": 0.06274900585412979,
|
|
"expert_2_frac": 0.06252726167440414,
|
|
"expert_3_frac": 0.0625016838312149,
|
|
"expert_4_frac": 0.06247502192854881,
|
|
"expert_5_frac": 0.0623481310904026,
|
|
"expert_6_frac": 0.06267069280147552,
|
|
"expert_7_frac": 0.06268630176782608,
|
|
"expert_8_frac": 0.06241276115179062,
|
|
"expert_9_frac": 0.06262339651584625,
|
|
"expert_frac_max": 0.06274900585412979,
|
|
"expert_frac_min": 0.0622016042470932,
|
|
"expert_frac_std": 0.000160344410687685,
|
|
"grad_norm": 0.38775309920310974,
|
|
"learning_rate": 0.00027612152845230865,
|
|
"loss": 3.8949,
|
|
"router_aux_loss": 2.000342970043421,
|
|
"router_entropy": 2.211577178960045,
|
|
"router_entropy_norm": 0.7976578571572152,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.22924411400247832,
|
|
"expert_0_frac": 0.06244306638836861,
|
|
"expert_10_frac": 0.0625232681632042,
|
|
"expert_11_frac": 0.06246243789792061,
|
|
"expert_12_frac": 0.06222465634346008,
|
|
"expert_13_frac": 0.06267709285020828,
|
|
"expert_14_frac": 0.06221655011177063,
|
|
"expert_15_frac": 0.06233378127217293,
|
|
"expert_1_frac": 0.0626348927617073,
|
|
"expert_2_frac": 0.0624520517885685,
|
|
"expert_3_frac": 0.06252284348011017,
|
|
"expert_4_frac": 0.06256033480167389,
|
|
"expert_5_frac": 0.06249850243330002,
|
|
"expert_6_frac": 0.06259974837303162,
|
|
"expert_7_frac": 0.06249947473406792,
|
|
"expert_8_frac": 0.06257827579975128,
|
|
"expert_9_frac": 0.06277304142713547,
|
|
"expert_frac_max": 0.06277304142713547,
|
|
"expert_frac_min": 0.06221655011177063,
|
|
"expert_frac_std": 0.0001495839678682387,
|
|
"grad_norm": 0.4576256275177002,
|
|
"learning_rate": 0.00027443158033523435,
|
|
"loss": 3.8763,
|
|
"router_aux_loss": 2.0003334523737433,
|
|
"router_entropy": 2.206269159093499,
|
|
"router_entropy_norm": 0.7957433936726137,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.23543990086741015,
|
|
"expert_0_frac": 0.06226146221160889,
|
|
"expert_10_frac": 0.062378235161304474,
|
|
"expert_11_frac": 0.06265708804130554,
|
|
"expert_12_frac": 0.06239369139075279,
|
|
"expert_13_frac": 0.06258013844490051,
|
|
"expert_14_frac": 0.062292423099279404,
|
|
"expert_15_frac": 0.0625077486038208,
|
|
"expert_1_frac": 0.06268229335546494,
|
|
"expert_2_frac": 0.06261877715587616,
|
|
"expert_3_frac": 0.06257469952106476,
|
|
"expert_4_frac": 0.06252875179052353,
|
|
"expert_5_frac": 0.062346141785383224,
|
|
"expert_6_frac": 0.06263620406389236,
|
|
"expert_7_frac": 0.06256420910358429,
|
|
"expert_8_frac": 0.06239767745137215,
|
|
"expert_9_frac": 0.06258046627044678,
|
|
"expert_frac_max": 0.06268229335546494,
|
|
"expert_frac_min": 0.06226146221160889,
|
|
"expert_frac_std": 0.0001352312829112634,
|
|
"grad_norm": 0.44768062233924866,
|
|
"learning_rate": 0.00027268940411500763,
|
|
"loss": 3.864,
|
|
"router_aux_loss": 2.0003784953057764,
|
|
"router_entropy": 2.1981928451855977,
|
|
"router_entropy_norm": 0.7928304791667157,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.241635687732342,
|
|
"expert_0_frac": 0.062274038791656494,
|
|
"expert_10_frac": 0.06261645257472992,
|
|
"expert_11_frac": 0.0625358372926712,
|
|
"expert_12_frac": 0.06225775554776192,
|
|
"expert_13_frac": 0.0624920092523098,
|
|
"expert_14_frac": 0.0622897669672966,
|
|
"expert_15_frac": 0.06222781166434288,
|
|
"expert_1_frac": 0.06258921325206757,
|
|
"expert_2_frac": 0.062490224838256836,
|
|
"expert_3_frac": 0.06263462454080582,
|
|
"expert_4_frac": 0.06267813593149185,
|
|
"expert_5_frac": 0.06240127235651016,
|
|
"expert_6_frac": 0.06273884326219559,
|
|
"expert_7_frac": 0.06260263919830322,
|
|
"expert_8_frac": 0.06257489323616028,
|
|
"expert_9_frac": 0.06259652227163315,
|
|
"expert_frac_max": 0.06273884326219559,
|
|
"expert_frac_min": 0.06222781166434288,
|
|
"expert_frac_std": 0.00016188451263587922,
|
|
"grad_norm": 0.44072648882865906,
|
|
"learning_rate": 0.00027089573104135695,
|
|
"loss": 3.8464,
|
|
"router_aux_loss": 2.0002842231094835,
|
|
"router_entropy": 2.1940508898099265,
|
|
"router_entropy_norm": 0.7913365845466996,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.24783147459727387,
|
|
"expert_0_frac": 0.06231727823615074,
|
|
"expert_10_frac": 0.06263195723295212,
|
|
"expert_11_frac": 0.06249087676405907,
|
|
"expert_12_frac": 0.06225622817873955,
|
|
"expert_13_frac": 0.06269091367721558,
|
|
"expert_14_frac": 0.06215570494532585,
|
|
"expert_15_frac": 0.062470827251672745,
|
|
"expert_1_frac": 0.0627078041434288,
|
|
"expert_2_frac": 0.06257681548595428,
|
|
"expert_3_frac": 0.06265929341316223,
|
|
"expert_4_frac": 0.06238482892513275,
|
|
"expert_5_frac": 0.06231342628598213,
|
|
"expert_6_frac": 0.06271243840456009,
|
|
"expert_7_frac": 0.06250765919685364,
|
|
"expert_8_frac": 0.06244330108165741,
|
|
"expert_9_frac": 0.0626806691288948,
|
|
"expert_frac_max": 0.06271243840456009,
|
|
"expert_frac_min": 0.06215570494532585,
|
|
"expert_frac_std": 0.00017695999122224748,
|
|
"grad_norm": 0.45407283306121826,
|
|
"learning_rate": 0.00026905131397896557,
|
|
"loss": 3.8192,
|
|
"router_aux_loss": 2.0003489625453947,
|
|
"router_entropy": 2.189426625718673,
|
|
"router_entropy_norm": 0.7896687338286466,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.2540272614622057,
|
|
"expert_0_frac": 0.062279440462589264,
|
|
"expert_10_frac": 0.06258825212717056,
|
|
"expert_11_frac": 0.0625229924917221,
|
|
"expert_12_frac": 0.06232941523194313,
|
|
"expert_13_frac": 0.062425460666418076,
|
|
"expert_14_frac": 0.06236887723207474,
|
|
"expert_15_frac": 0.06254032999277115,
|
|
"expert_1_frac": 0.0624471940100193,
|
|
"expert_2_frac": 0.06263547390699387,
|
|
"expert_3_frac": 0.06265461444854736,
|
|
"expert_4_frac": 0.06267198920249939,
|
|
"expert_5_frac": 0.062366899102926254,
|
|
"expert_6_frac": 0.062466420233249664,
|
|
"expert_7_frac": 0.06264328211545944,
|
|
"expert_8_frac": 0.06244220957159996,
|
|
"expert_9_frac": 0.06261716783046722,
|
|
"expert_frac_max": 0.06267198920249939,
|
|
"expert_frac_min": 0.062279440462589264,
|
|
"expert_frac_std": 0.00012688770948443562,
|
|
"grad_norm": 0.4619823694229126,
|
|
"learning_rate": 0.0002671569270914689,
|
|
"loss": 3.8077,
|
|
"router_aux_loss": 2.000316036939621,
|
|
"router_entropy": 2.1830097007751466,
|
|
"router_entropy_norm": 0.787354317380201,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.26022304832713755,
|
|
"expert_0_frac": 0.06231517344713211,
|
|
"expert_10_frac": 0.06260181963443756,
|
|
"expert_11_frac": 0.0624566413462162,
|
|
"expert_12_frac": 0.06214236095547676,
|
|
"expert_13_frac": 0.0626717060804367,
|
|
"expert_14_frac": 0.062233492732048035,
|
|
"expert_15_frac": 0.06237335875630379,
|
|
"expert_1_frac": 0.0626189112663269,
|
|
"expert_2_frac": 0.06243986263871193,
|
|
"expert_3_frac": 0.0627051368355751,
|
|
"expert_4_frac": 0.06272777915000916,
|
|
"expert_5_frac": 0.06241276115179062,
|
|
"expert_6_frac": 0.062677301466465,
|
|
"expert_7_frac": 0.06261099129915237,
|
|
"expert_8_frac": 0.06242264434695244,
|
|
"expert_9_frac": 0.06259007751941681,
|
|
"expert_frac_max": 0.06272777915000916,
|
|
"expert_frac_min": 0.06214236095547676,
|
|
"expert_frac_std": 0.00017664371989667416,
|
|
"grad_norm": 0.5377533435821533,
|
|
"learning_rate": 0.000265213365516512,
|
|
"loss": 3.7954,
|
|
"router_aux_loss": 2.0003011628985403,
|
|
"router_entropy": 2.1784792892634868,
|
|
"router_entropy_norm": 0.7857203168249365,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.2664188351920694,
|
|
"expert_0_frac": 0.06247631832957268,
|
|
"expert_10_frac": 0.06265552341938019,
|
|
"expert_11_frac": 0.062469732016325,
|
|
"expert_12_frac": 0.062348317354917526,
|
|
"expert_13_frac": 0.062486276030540466,
|
|
"expert_14_frac": 0.062405217438936234,
|
|
"expert_15_frac": 0.062300167977809906,
|
|
"expert_1_frac": 0.06241647154092789,
|
|
"expert_2_frac": 0.0623706579208374,
|
|
"expert_3_frac": 0.06280349940061569,
|
|
"expert_4_frac": 0.06258194893598557,
|
|
"expert_5_frac": 0.062391288578510284,
|
|
"expert_6_frac": 0.06257753819227219,
|
|
"expert_7_frac": 0.06254401057958603,
|
|
"expert_8_frac": 0.06254838407039642,
|
|
"expert_9_frac": 0.06262466311454773,
|
|
"expert_frac_max": 0.06280349940061569,
|
|
"expert_frac_min": 0.062300167977809906,
|
|
"expert_frac_std": 0.00013113862951286137,
|
|
"grad_norm": 0.4972023367881775,
|
|
"learning_rate": 0.00026322144503200404,
|
|
"loss": 3.7692,
|
|
"router_aux_loss": 2.000367202758789,
|
|
"router_entropy": 2.1743140166749555,
|
|
"router_entropy_norm": 0.7842180122980803,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.27261462205700127,
|
|
"expert_0_frac": 0.06236778944730759,
|
|
"expert_10_frac": 0.06253974139690399,
|
|
"expert_11_frac": 0.06243103742599487,
|
|
"expert_12_frac": 0.06224328652024269,
|
|
"expert_13_frac": 0.06255636364221573,
|
|
"expert_14_frac": 0.062425754964351654,
|
|
"expert_15_frac": 0.062366798520088196,
|
|
"expert_1_frac": 0.06278834491968155,
|
|
"expert_2_frac": 0.06251846253871918,
|
|
"expert_3_frac": 0.06271340698003769,
|
|
"expert_4_frac": 0.06249046325683594,
|
|
"expert_5_frac": 0.062309712171554565,
|
|
"expert_6_frac": 0.06265412271022797,
|
|
"expert_7_frac": 0.06254371255636215,
|
|
"expert_8_frac": 0.06236147880554199,
|
|
"expert_9_frac": 0.06268955022096634,
|
|
"expert_frac_max": 0.06278834491968155,
|
|
"expert_frac_min": 0.06224328652024269,
|
|
"expert_frac_std": 0.00015540422464255244,
|
|
"grad_norm": 0.4365726709365845,
|
|
"learning_rate": 0.00026118200171370843,
|
|
"loss": 3.7575,
|
|
"router_aux_loss": 2.000355373620987,
|
|
"router_entropy": 2.167735231295228,
|
|
"router_entropy_norm": 0.7818452170374789,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.2788104089219331,
|
|
"expert_0_frac": 0.062455914914608,
|
|
"expert_10_frac": 0.06251836568117142,
|
|
"expert_11_frac": 0.06251351535320282,
|
|
"expert_12_frac": 0.062291666865348816,
|
|
"expert_13_frac": 0.06252916157245636,
|
|
"expert_14_frac": 0.06226165220141411,
|
|
"expert_15_frac": 0.06260184198617935,
|
|
"expert_1_frac": 0.06268971413373947,
|
|
"expert_2_frac": 0.06259042769670486,
|
|
"expert_3_frac": 0.06244368478655815,
|
|
"expert_4_frac": 0.062420859932899475,
|
|
"expert_5_frac": 0.062372010201215744,
|
|
"expert_6_frac": 0.0626329779624939,
|
|
"expert_7_frac": 0.0627329871058464,
|
|
"expert_8_frac": 0.062414951622486115,
|
|
"expert_9_frac": 0.06253029406070709,
|
|
"expert_frac_max": 0.0627329871058464,
|
|
"expert_frac_min": 0.06226165220141411,
|
|
"expert_frac_std": 0.00013266490714158863,
|
|
"grad_norm": 0.5881329774856567,
|
|
"learning_rate": 0.0002590958915843143,
|
|
"loss": 3.7527,
|
|
"router_aux_loss": 2.000336780399084,
|
|
"router_entropy": 2.1648418973137935,
|
|
"router_entropy_norm": 0.7808016673908161,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.2850061957868649,
|
|
"expert_0_frac": 0.062241166830062866,
|
|
"expert_10_frac": 0.06262774020433426,
|
|
"expert_11_frac": 0.06260697543621063,
|
|
"expert_12_frac": 0.06218230724334717,
|
|
"expert_13_frac": 0.06255532056093216,
|
|
"expert_14_frac": 0.06231023371219635,
|
|
"expert_15_frac": 0.06251320242881775,
|
|
"expert_1_frac": 0.06266742944717407,
|
|
"expert_2_frac": 0.06255898624658585,
|
|
"expert_3_frac": 0.0625995546579361,
|
|
"expert_4_frac": 0.06255882233381271,
|
|
"expert_5_frac": 0.06240781396627426,
|
|
"expert_6_frac": 0.06258196383714676,
|
|
"expert_7_frac": 0.06254728883504868,
|
|
"expert_8_frac": 0.062440671026706696,
|
|
"expert_9_frac": 0.06260056048631668,
|
|
"expert_frac_max": 0.06266742944717407,
|
|
"expert_frac_min": 0.06218230724334717,
|
|
"expert_frac_std": 0.00014396726328413934,
|
|
"grad_norm": 0.4369165897369385,
|
|
"learning_rate": 0.0002569639902541347,
|
|
"loss": 3.7403,
|
|
"router_aux_loss": 2.000332098007202,
|
|
"router_entropy": 2.160193564010163,
|
|
"router_entropy_norm": 0.7791251355394294,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.29120198265179675,
|
|
"expert_0_frac": 0.06230125576257706,
|
|
"expert_10_frac": 0.06242264062166214,
|
|
"expert_11_frac": 0.0626748725771904,
|
|
"expert_12_frac": 0.06223186478018761,
|
|
"expert_13_frac": 0.06251568347215652,
|
|
"expert_14_frac": 0.06230219826102257,
|
|
"expert_15_frac": 0.06251835078001022,
|
|
"expert_1_frac": 0.06252612918615341,
|
|
"expert_2_frac": 0.06243795529007912,
|
|
"expert_3_frac": 0.06267717480659485,
|
|
"expert_4_frac": 0.06260242313146591,
|
|
"expert_5_frac": 0.06240590661764145,
|
|
"expert_6_frac": 0.06257519125938416,
|
|
"expert_7_frac": 0.06247491389513016,
|
|
"expert_8_frac": 0.06258118152618408,
|
|
"expert_9_frac": 0.06275228410959244,
|
|
"expert_frac_max": 0.06275228410959244,
|
|
"expert_frac_min": 0.06223186478018761,
|
|
"expert_frac_std": 0.00014593535161111504,
|
|
"grad_norm": 0.4153628349304199,
|
|
"learning_rate": 0.00025478719255358405,
|
|
"loss": 3.7196,
|
|
"router_aux_loss": 2.00037807777524,
|
|
"router_entropy": 2.156891989496847,
|
|
"router_entropy_norm": 0.7779343442450578,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.29739776951672864,
|
|
"expert_0_frac": 0.06231791898608208,
|
|
"expert_10_frac": 0.06267321854829788,
|
|
"expert_11_frac": 0.06255703419446945,
|
|
"expert_12_frac": 0.062291208654642105,
|
|
"expert_13_frac": 0.06265942007303238,
|
|
"expert_14_frac": 0.06218227371573448,
|
|
"expert_15_frac": 0.06219065561890602,
|
|
"expert_1_frac": 0.06256262212991714,
|
|
"expert_2_frac": 0.06257721036672592,
|
|
"expert_3_frac": 0.0626627653837204,
|
|
"expert_4_frac": 0.062482550740242004,
|
|
"expert_5_frac": 0.0623604841530323,
|
|
"expert_6_frac": 0.0626610815525055,
|
|
"expert_7_frac": 0.06265909224748611,
|
|
"expert_8_frac": 0.06247072294354439,
|
|
"expert_9_frac": 0.06269176304340363,
|
|
"expert_frac_max": 0.06269176304340363,
|
|
"expert_frac_min": 0.06218227371573448,
|
|
"expert_frac_std": 0.00017834350001066923,
|
|
"grad_norm": 0.41617467999458313,
|
|
"learning_rate": 0.00025256641215758743,
|
|
"loss": 3.7003,
|
|
"router_aux_loss": 2.0003033436834814,
|
|
"router_entropy": 2.1526932566737136,
|
|
"router_entropy_norm": 0.7764199714895698,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.3,
|
|
"eval_loss": 3.081298589706421,
|
|
"eval_runtime": 33.3012,
|
|
"eval_samples_per_second": 51.89,
|
|
"eval_steps_per_second": 6.486,
|
|
"expert_0_frac": 0.06206047907471657,
|
|
"expert_10_frac": 0.06263051182031631,
|
|
"expert_11_frac": 0.06254921853542328,
|
|
"expert_12_frac": 0.06210346519947052,
|
|
"expert_13_frac": 0.06262776255607605,
|
|
"expert_14_frac": 0.06231609359383583,
|
|
"expert_15_frac": 0.06274214386940002,
|
|
"expert_1_frac": 0.062285520136356354,
|
|
"expert_2_frac": 0.06221441179513931,
|
|
"expert_3_frac": 0.06259218603372574,
|
|
"expert_4_frac": 0.06259699165821075,
|
|
"expert_5_frac": 0.06273031234741211,
|
|
"expert_6_frac": 0.06256095319986343,
|
|
"expert_7_frac": 0.06282997876405716,
|
|
"expert_8_frac": 0.06255850940942764,
|
|
"expert_9_frac": 0.06260145455598831,
|
|
"expert_frac_max": 0.06282997876405716,
|
|
"expert_frac_min": 0.06206047907471657,
|
|
"expert_frac_std": 0.00023144215811043978,
|
|
"router_aux_loss": 2.000319122620251,
|
|
"router_entropy": 2.150703725823458,
|
|
"router_entropy_norm": 0.7757023999167298,
|
|
"step": 2421
|
|
},
|
|
{
|
|
"epoch": 0.30359355638166047,
|
|
"expert_0_frac": 0.06226431205868721,
|
|
"expert_10_frac": 0.06251043826341629,
|
|
"expert_11_frac": 0.062482383102178574,
|
|
"expert_12_frac": 0.06242340803146362,
|
|
"expert_13_frac": 0.06245507672429085,
|
|
"expert_14_frac": 0.062295958399772644,
|
|
"expert_15_frac": 0.062416043132543564,
|
|
"expert_1_frac": 0.06262195110321045,
|
|
"expert_2_frac": 0.06257570534944534,
|
|
"expert_3_frac": 0.0626882016658783,
|
|
"expert_4_frac": 0.06279187649488449,
|
|
"expert_5_frac": 0.06238429993391037,
|
|
"expert_6_frac": 0.06260958313941956,
|
|
"expert_7_frac": 0.0625062808394432,
|
|
"expert_8_frac": 0.0622023344039917,
|
|
"expert_9_frac": 0.06277214735746384,
|
|
"expert_frac_max": 0.06279187649488449,
|
|
"expert_frac_min": 0.0622023344039917,
|
|
"expert_frac_std": 0.0001716299884719774,
|
|
"grad_norm": 0.41244012117385864,
|
|
"learning_rate": 0.00025030258120208086,
|
|
"loss": 3.6924,
|
|
"router_aux_loss": 2.0003207174354585,
|
|
"router_entropy": 2.1494769665316262,
|
|
"router_entropy_norm": 0.7752599400300574,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.3097893432465923,
|
|
"expert_0_frac": 0.062158700078725815,
|
|
"expert_10_frac": 0.06262535601854324,
|
|
"expert_11_frac": 0.06255626678466797,
|
|
"expert_12_frac": 0.0621267706155777,
|
|
"expert_13_frac": 0.06274285167455673,
|
|
"expert_14_frac": 0.062286049127578735,
|
|
"expert_15_frac": 0.06258541345596313,
|
|
"expert_1_frac": 0.06256241351366043,
|
|
"expert_2_frac": 0.0624278262257576,
|
|
"expert_3_frac": 0.062509685754776,
|
|
"expert_4_frac": 0.0625128522515297,
|
|
"expert_5_frac": 0.06246746703982353,
|
|
"expert_6_frac": 0.0625898465514183,
|
|
"expert_7_frac": 0.06269817799329758,
|
|
"expert_8_frac": 0.06244150176644325,
|
|
"expert_9_frac": 0.06270884722471237,
|
|
"expert_frac_max": 0.06274285167455673,
|
|
"expert_frac_min": 0.0621267706155777,
|
|
"expert_frac_std": 0.00018101956811733544,
|
|
"grad_norm": 0.4232514798641205,
|
|
"learning_rate": 0.0002479966498927624,
|
|
"loss": 3.6834,
|
|
"router_aux_loss": 2.0003105348348615,
|
|
"router_entropy": 2.1456227505579593,
|
|
"router_entropy_norm": 0.7738698254621263,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.3159851301115242,
|
|
"expert_0_frac": 0.06242232769727707,
|
|
"expert_10_frac": 0.06251971423625946,
|
|
"expert_11_frac": 0.06261476129293442,
|
|
"expert_12_frac": 0.062365610152482986,
|
|
"expert_13_frac": 0.06250380724668503,
|
|
"expert_14_frac": 0.06229201331734657,
|
|
"expert_15_frac": 0.06255565583705902,
|
|
"expert_1_frac": 0.06260047852993011,
|
|
"expert_2_frac": 0.06252625584602356,
|
|
"expert_3_frac": 0.06255336105823517,
|
|
"expert_4_frac": 0.06245260313153267,
|
|
"expert_5_frac": 0.06221557781100273,
|
|
"expert_6_frac": 0.06264305114746094,
|
|
"expert_7_frac": 0.06258053332567215,
|
|
"expert_8_frac": 0.0624939501285553,
|
|
"expert_9_frac": 0.0626603215932846,
|
|
"expert_frac_max": 0.0626603215932846,
|
|
"expert_frac_min": 0.06221557781100273,
|
|
"expert_frac_std": 0.00012476296979002655,
|
|
"grad_norm": 0.4126286208629608,
|
|
"learning_rate": 0.00024564958610625925,
|
|
"loss": 3.6738,
|
|
"router_aux_loss": 2.000373025536537,
|
|
"router_entropy": 2.1407304053753613,
|
|
"router_entropy_norm": 0.7721052849288136,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.322180916976456,
|
|
"expert_0_frac": 0.06229957565665245,
|
|
"expert_10_frac": 0.06251021474599838,
|
|
"expert_11_frac": 0.06240880489349365,
|
|
"expert_12_frac": 0.0621674545109272,
|
|
"expert_13_frac": 0.062565378844738,
|
|
"expert_14_frac": 0.06240019574761391,
|
|
"expert_15_frac": 0.06233181059360504,
|
|
"expert_1_frac": 0.06257034838199615,
|
|
"expert_2_frac": 0.06248115003108978,
|
|
"expert_3_frac": 0.06270527094602585,
|
|
"expert_4_frac": 0.06253506988286972,
|
|
"expert_5_frac": 0.06253746896982193,
|
|
"expert_6_frac": 0.06264340132474899,
|
|
"expert_7_frac": 0.06273256987333298,
|
|
"expert_8_frac": 0.062466099858284,
|
|
"expert_9_frac": 0.06264520436525345,
|
|
"expert_frac_max": 0.06273256987333298,
|
|
"expert_frac_min": 0.0621674545109272,
|
|
"expert_frac_std": 0.00015245997929014266,
|
|
"grad_norm": 0.42913854122161865,
|
|
"learning_rate": 0.00024326237498387734,
|
|
"loss": 3.6625,
|
|
"router_aux_loss": 2.0002928037941454,
|
|
"router_entropy": 2.1381007156769436,
|
|
"router_entropy_norm": 0.7711568248570675,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.32837670384138784,
|
|
"expert_0_frac": 0.062335118651390076,
|
|
"expert_10_frac": 0.06263874471187592,
|
|
"expert_11_frac": 0.06248386576771736,
|
|
"expert_12_frac": 0.06223371624946594,
|
|
"expert_13_frac": 0.0627041608095169,
|
|
"expert_14_frac": 0.062123946845531464,
|
|
"expert_15_frac": 0.06241362914443016,
|
|
"expert_1_frac": 0.06246369332075119,
|
|
"expert_2_frac": 0.0624663308262825,
|
|
"expert_3_frac": 0.06285633146762848,
|
|
"expert_4_frac": 0.0624915212392807,
|
|
"expert_5_frac": 0.06226895749568939,
|
|
"expert_6_frac": 0.06270433217287064,
|
|
"expert_7_frac": 0.0626092478632927,
|
|
"expert_8_frac": 0.06243276596069336,
|
|
"expert_9_frac": 0.06277365982532501,
|
|
"expert_frac_max": 0.06285633146762848,
|
|
"expert_frac_min": 0.062123946845531464,
|
|
"expert_frac_std": 0.00020398269407451153,
|
|
"grad_norm": 0.4190995693206787,
|
|
"learning_rate": 0.00024083601851810444,
|
|
"loss": 3.6486,
|
|
"router_aux_loss": 2.000305945575237,
|
|
"router_entropy": 2.135682564576467,
|
|
"router_entropy_norm": 0.7702846612068731,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.3345724907063197,
|
|
"expert_0_frac": 0.062428977340459824,
|
|
"expert_10_frac": 0.06245648115873337,
|
|
"expert_11_frac": 0.06252357363700867,
|
|
"expert_12_frac": 0.06209154799580574,
|
|
"expert_13_frac": 0.0625762864947319,
|
|
"expert_14_frac": 0.062406446784734726,
|
|
"expert_15_frac": 0.06270066648721695,
|
|
"expert_1_frac": 0.0626891627907753,
|
|
"expert_2_frac": 0.06255494803190231,
|
|
"expert_3_frac": 0.06254468858242035,
|
|
"expert_4_frac": 0.062471628189086914,
|
|
"expert_5_frac": 0.06238212808966637,
|
|
"expert_6_frac": 0.062687948346138,
|
|
"expert_7_frac": 0.06255471706390381,
|
|
"expert_8_frac": 0.06231201812624931,
|
|
"expert_9_frac": 0.06261879950761795,
|
|
"expert_frac_max": 0.06270066648721695,
|
|
"expert_frac_min": 0.06209154799580574,
|
|
"expert_frac_std": 0.00015757165965624154,
|
|
"grad_norm": 0.3790879249572754,
|
|
"learning_rate": 0.0002383715351320406,
|
|
"loss": 3.6488,
|
|
"router_aux_loss": 2.000347435027361,
|
|
"router_entropy": 2.131170397289097,
|
|
"router_entropy_norm": 0.7686572408645856,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.34076827757125155,
|
|
"expert_0_frac": 0.062156982719898224,
|
|
"expert_10_frac": 0.062389545142650604,
|
|
"expert_11_frac": 0.0625583827495575,
|
|
"expert_12_frac": 0.062392160296440125,
|
|
"expert_13_frac": 0.06247055158019066,
|
|
"expert_14_frac": 0.06246820464730263,
|
|
"expert_15_frac": 0.06243617460131645,
|
|
"expert_1_frac": 0.06267351657152176,
|
|
"expert_2_frac": 0.06233975291252136,
|
|
"expert_3_frac": 0.06252095103263855,
|
|
"expert_4_frac": 0.06271501630544662,
|
|
"expert_5_frac": 0.062358953058719635,
|
|
"expert_6_frac": 0.06264780461788177,
|
|
"expert_7_frac": 0.06268636882305145,
|
|
"expert_8_frac": 0.06252342462539673,
|
|
"expert_9_frac": 0.06266223639249802,
|
|
"expert_frac_max": 0.06271501630544662,
|
|
"expert_frac_min": 0.062156982719898224,
|
|
"expert_frac_std": 0.00015419730334542692,
|
|
"grad_norm": 0.42185646295547485,
|
|
"learning_rate": 0.0002358699592519316,
|
|
"loss": 3.6241,
|
|
"router_aux_loss": 2.0003399024903774,
|
|
"router_entropy": 2.1298128960778318,
|
|
"router_entropy_norm": 0.7681676257982123,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.3469640644361834,
|
|
"expert_0_frac": 0.06238381192088127,
|
|
"expert_10_frac": 0.06263288110494614,
|
|
"expert_11_frac": 0.06245595961809158,
|
|
"expert_12_frac": 0.06230363994836807,
|
|
"expert_13_frac": 0.062405478209257126,
|
|
"expert_14_frac": 0.06207668408751488,
|
|
"expert_15_frac": 0.062493905425071716,
|
|
"expert_1_frac": 0.06252549588680267,
|
|
"expert_2_frac": 0.06255904585123062,
|
|
"expert_3_frac": 0.0626806765794754,
|
|
"expert_4_frac": 0.06250762194395065,
|
|
"expert_5_frac": 0.062433961778879166,
|
|
"expert_6_frac": 0.06261438131332397,
|
|
"expert_7_frac": 0.0627550259232521,
|
|
"expert_8_frac": 0.062472350895404816,
|
|
"expert_9_frac": 0.062699094414711,
|
|
"expert_frac_max": 0.0627550259232521,
|
|
"expert_frac_min": 0.06207668408751488,
|
|
"expert_frac_std": 0.00016724149463698268,
|
|
"grad_norm": 0.3707304000854492,
|
|
"learning_rate": 0.00023333234087298565,
|
|
"loss": 3.626,
|
|
"router_aux_loss": 2.000274879336357,
|
|
"router_entropy": 2.123708927979072,
|
|
"router_entropy_norm": 0.765966084671756,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.35315985130111527,
|
|
"expert_0_frac": 0.06237242370843887,
|
|
"expert_10_frac": 0.06265714764595032,
|
|
"expert_11_frac": 0.06253128498792648,
|
|
"expert_12_frac": 0.062280744314193726,
|
|
"expert_13_frac": 0.06251207739114761,
|
|
"expert_14_frac": 0.06227829307317734,
|
|
"expert_15_frac": 0.06258461624383926,
|
|
"expert_1_frac": 0.06235877051949501,
|
|
"expert_2_frac": 0.062398917973041534,
|
|
"expert_3_frac": 0.06260959059000015,
|
|
"expert_4_frac": 0.0624103806912899,
|
|
"expert_5_frac": 0.06252153217792511,
|
|
"expert_6_frac": 0.0626278966665268,
|
|
"expert_7_frac": 0.06266181170940399,
|
|
"expert_8_frac": 0.06252672523260117,
|
|
"expert_9_frac": 0.06266781687736511,
|
|
"expert_frac_max": 0.06266781687736511,
|
|
"expert_frac_min": 0.06227829307317734,
|
|
"expert_frac_std": 0.0001338079891866073,
|
|
"grad_norm": 0.41720008850097656,
|
|
"learning_rate": 0.0002307597451186554,
|
|
"loss": 3.6167,
|
|
"router_aux_loss": 2.000353735238314,
|
|
"router_entropy": 2.121400617274145,
|
|
"router_entropy_norm": 0.7651335375700488,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.3593556381660471,
|
|
"expert_0_frac": 0.06238774210214615,
|
|
"expert_10_frac": 0.0625099465250969,
|
|
"expert_11_frac": 0.06246226280927658,
|
|
"expert_12_frac": 0.06219741329550743,
|
|
"expert_13_frac": 0.06253630667924881,
|
|
"expert_14_frac": 0.06228252500295639,
|
|
"expert_15_frac": 0.06252117455005646,
|
|
"expert_1_frac": 0.06270123273134232,
|
|
"expert_2_frac": 0.06251953542232513,
|
|
"expert_3_frac": 0.06263242661952972,
|
|
"expert_4_frac": 0.06256052106618881,
|
|
"expert_5_frac": 0.062256649136543274,
|
|
"expert_6_frac": 0.06262441724538803,
|
|
"expert_7_frac": 0.0625799149274826,
|
|
"expert_8_frac": 0.06235365569591522,
|
|
"expert_9_frac": 0.06287430226802826,
|
|
"expert_frac_max": 0.06287430226802826,
|
|
"expert_frac_min": 0.06219741329550743,
|
|
"expert_frac_std": 0.00017500968533568084,
|
|
"grad_norm": 0.40777263045310974,
|
|
"learning_rate": 0.0002281532517935697,
|
|
"loss": 3.6094,
|
|
"router_aux_loss": 2.0003093630075455,
|
|
"router_entropy": 2.118771378012995,
|
|
"router_entropy_norm": 0.7641852399592057,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.3655514250309789,
|
|
"expert_0_frac": 0.06230409815907478,
|
|
"expert_10_frac": 0.062623530626297,
|
|
"expert_11_frac": 0.06251265108585358,
|
|
"expert_12_frac": 0.06222499907016754,
|
|
"expert_13_frac": 0.06258230656385422,
|
|
"expert_14_frac": 0.062339454889297485,
|
|
"expert_15_frac": 0.06251765042543411,
|
|
"expert_1_frac": 0.06265218555927277,
|
|
"expert_2_frac": 0.06247617304325104,
|
|
"expert_3_frac": 0.06254806369543076,
|
|
"expert_4_frac": 0.06239999458193779,
|
|
"expert_5_frac": 0.06245831772685051,
|
|
"expert_6_frac": 0.06256891787052155,
|
|
"expert_7_frac": 0.06273853778839111,
|
|
"expert_8_frac": 0.06234954297542572,
|
|
"expert_9_frac": 0.06270358711481094,
|
|
"expert_frac_max": 0.06273853778839111,
|
|
"expert_frac_min": 0.06222499907016754,
|
|
"expert_frac_std": 0.00014737600577063859,
|
|
"grad_norm": 0.42828845977783203,
|
|
"learning_rate": 0.0002255139549303036,
|
|
"loss": 3.586,
|
|
"router_aux_loss": 2.0003345051407813,
|
|
"router_entropy": 2.1164697188759845,
|
|
"router_entropy_norm": 0.7633550919035104,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.37174721189591076,
|
|
"expert_0_frac": 0.06223268806934357,
|
|
"expert_10_frac": 0.06252967566251755,
|
|
"expert_11_frac": 0.06271190196275711,
|
|
"expert_12_frac": 0.062454599887132645,
|
|
"expert_13_frac": 0.062481142580509186,
|
|
"expert_14_frac": 0.06233172118663788,
|
|
"expert_15_frac": 0.06249425560235977,
|
|
"expert_1_frac": 0.06263557076454163,
|
|
"expert_2_frac": 0.062410738319158554,
|
|
"expert_3_frac": 0.06252335011959076,
|
|
"expert_4_frac": 0.06250470131635666,
|
|
"expert_5_frac": 0.06235555186867714,
|
|
"expert_6_frac": 0.06261074542999268,
|
|
"expert_7_frac": 0.062659852206707,
|
|
"expert_8_frac": 0.062338899821043015,
|
|
"expert_9_frac": 0.06272463500499725,
|
|
"expert_frac_max": 0.06272463500499725,
|
|
"expert_frac_min": 0.06223268806934357,
|
|
"expert_frac_std": 0.00014376308536157012,
|
|
"grad_norm": 0.3720088005065918,
|
|
"learning_rate": 0.00022284296233017614,
|
|
"loss": 3.5771,
|
|
"router_aux_loss": 2.0003691291809083,
|
|
"router_entropy": 2.1153419841329257,
|
|
"router_entropy_norm": 0.7629483475731981,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.37794299876084264,
|
|
"expert_0_frac": 0.06234049052000046,
|
|
"expert_10_frac": 0.06246494501829147,
|
|
"expert_11_frac": 0.06251261383295059,
|
|
"expert_12_frac": 0.062142081558704376,
|
|
"expert_13_frac": 0.06254368275403976,
|
|
"expert_14_frac": 0.06233340874314308,
|
|
"expert_15_frac": 0.06253020465373993,
|
|
"expert_1_frac": 0.06254199147224426,
|
|
"expert_2_frac": 0.062429092824459076,
|
|
"expert_3_frac": 0.06272594630718231,
|
|
"expert_4_frac": 0.06251217424869537,
|
|
"expert_5_frac": 0.062371477484703064,
|
|
"expert_6_frac": 0.06272263824939728,
|
|
"expert_7_frac": 0.06261076778173447,
|
|
"expert_8_frac": 0.0625164806842804,
|
|
"expert_9_frac": 0.06270202994346619,
|
|
"expert_frac_max": 0.06272594630718231,
|
|
"expert_frac_min": 0.062142081558704376,
|
|
"expert_frac_std": 0.00015555151912849396,
|
|
"grad_norm": 0.38100388646125793,
|
|
"learning_rate": 0.00022014139509826905,
|
|
"loss": 3.5893,
|
|
"router_aux_loss": 2.000323775559664,
|
|
"router_entropy": 2.1117512386168045,
|
|
"router_entropy_norm": 0.7616532598858975,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.38413878562577447,
|
|
"expert_0_frac": 0.06226764991879463,
|
|
"expert_10_frac": 0.06236816570162773,
|
|
"expert_11_frac": 0.06274646520614624,
|
|
"expert_12_frac": 0.06223607435822487,
|
|
"expert_13_frac": 0.06262464821338654,
|
|
"expert_14_frac": 0.06223903223872185,
|
|
"expert_15_frac": 0.06242969632148743,
|
|
"expert_1_frac": 0.06258812546730042,
|
|
"expert_2_frac": 0.06244523450732231,
|
|
"expert_3_frac": 0.06271171569824219,
|
|
"expert_4_frac": 0.06243456155061722,
|
|
"expert_5_frac": 0.06234009563922882,
|
|
"expert_6_frac": 0.062552310526371,
|
|
"expert_7_frac": 0.0627344399690628,
|
|
"expert_8_frac": 0.0625125989317894,
|
|
"expert_9_frac": 0.06276921182870865,
|
|
"expert_frac_max": 0.06276921182870865,
|
|
"expert_frac_min": 0.06223607435822487,
|
|
"expert_frac_std": 0.00018368911696597934,
|
|
"grad_norm": 0.39150938391685486,
|
|
"learning_rate": 0.0002174103871728613,
|
|
"loss": 3.5581,
|
|
"router_aux_loss": 2.0003552556037905,
|
|
"router_entropy": 2.108792674889167,
|
|
"router_entropy_norm": 0.7605861835813934,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.3903345724907063,
|
|
"expert_0_frac": 0.0623362734913826,
|
|
"expert_10_frac": 0.06251577287912369,
|
|
"expert_11_frac": 0.06251625716686249,
|
|
"expert_12_frac": 0.062346331775188446,
|
|
"expert_13_frac": 0.06256897002458572,
|
|
"expert_14_frac": 0.0623774528503418,
|
|
"expert_15_frac": 0.062373917549848557,
|
|
"expert_1_frac": 0.062472108751535416,
|
|
"expert_2_frac": 0.06244368851184845,
|
|
"expert_3_frac": 0.06261617690324783,
|
|
"expert_4_frac": 0.06247147545218468,
|
|
"expert_5_frac": 0.06246494501829147,
|
|
"expert_6_frac": 0.06274916231632233,
|
|
"expert_7_frac": 0.06245803460478783,
|
|
"expert_8_frac": 0.06257476657629013,
|
|
"expert_9_frac": 0.06271468102931976,
|
|
"expert_frac_max": 0.06274916231632233,
|
|
"expert_frac_min": 0.0623362734913826,
|
|
"expert_frac_std": 0.00012154950672993436,
|
|
"grad_norm": 0.4003765881061554,
|
|
"learning_rate": 0.00021465108484947738,
|
|
"loss": 3.5526,
|
|
"router_aux_loss": 2.000348298251629,
|
|
"router_entropy": 2.1078100501249235,
|
|
"router_entropy_norm": 0.7602317766127862,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.3965303593556382,
|
|
"expert_0_frac": 0.062301527708768845,
|
|
"expert_10_frac": 0.062433645129203796,
|
|
"expert_11_frac": 0.06263574212789536,
|
|
"expert_12_frac": 0.06211092323064804,
|
|
"expert_13_frac": 0.062396157532930374,
|
|
"expert_14_frac": 0.06230778619647026,
|
|
"expert_15_frac": 0.06266193836927414,
|
|
"expert_1_frac": 0.06248924881219864,
|
|
"expert_2_frac": 0.062424615025520325,
|
|
"expert_3_frac": 0.06274488568305969,
|
|
"expert_4_frac": 0.06253775954246521,
|
|
"expert_5_frac": 0.062442515045404434,
|
|
"expert_6_frac": 0.0626094862818718,
|
|
"expert_7_frac": 0.06269068270921707,
|
|
"expert_8_frac": 0.062457650899887085,
|
|
"expert_9_frac": 0.06275545060634613,
|
|
"expert_frac_max": 0.06275545060634613,
|
|
"expert_frac_min": 0.06211092323064804,
|
|
"expert_frac_std": 0.0001776602875906974,
|
|
"grad_norm": 0.37601521611213684,
|
|
"learning_rate": 0.00021186464629974848,
|
|
"loss": 3.5602,
|
|
"router_aux_loss": 2.000331805050373,
|
|
"router_entropy": 2.104159056780239,
|
|
"router_entropy_norm": 0.7589149591146124,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.4,
|
|
"eval_loss": 2.932450771331787,
|
|
"eval_runtime": 33.668,
|
|
"eval_samples_per_second": 51.325,
|
|
"eval_steps_per_second": 6.416,
|
|
"expert_0_frac": 0.06264324486255646,
|
|
"expert_10_frac": 0.06252216547727585,
|
|
"expert_11_frac": 0.06269601732492447,
|
|
"expert_12_frac": 0.06214261054992676,
|
|
"expert_13_frac": 0.0626559853553772,
|
|
"expert_14_frac": 0.062081582844257355,
|
|
"expert_15_frac": 0.06226176396012306,
|
|
"expert_1_frac": 0.06273604184389114,
|
|
"expert_2_frac": 0.06268617510795593,
|
|
"expert_3_frac": 0.06241494044661522,
|
|
"expert_4_frac": 0.062481921166181564,
|
|
"expert_5_frac": 0.062450990080833435,
|
|
"expert_6_frac": 0.06252512335777283,
|
|
"expert_7_frac": 0.06260444968938828,
|
|
"expert_8_frac": 0.06249772384762764,
|
|
"expert_9_frac": 0.06259922683238983,
|
|
"expert_frac_max": 0.06273604184389114,
|
|
"expert_frac_min": 0.062081582844257355,
|
|
"expert_frac_std": 0.00019408830848988146,
|
|
"router_aux_loss": 2.000305411506848,
|
|
"router_entropy": 2.1031951366897568,
|
|
"router_entropy_norm": 0.7585672984310244,
|
|
"step": 3228
|
|
},
|
|
{
|
|
"epoch": 0.40272614622057,
|
|
"expert_0_frac": 0.062352053821086884,
|
|
"expert_10_frac": 0.0625324547290802,
|
|
"expert_11_frac": 0.06247243285179138,
|
|
"expert_12_frac": 0.06233026087284088,
|
|
"expert_13_frac": 0.06254485249519348,
|
|
"expert_14_frac": 0.06235339492559433,
|
|
"expert_15_frac": 0.06269770115613937,
|
|
"expert_1_frac": 0.062410831451416016,
|
|
"expert_2_frac": 0.06258723139762878,
|
|
"expert_3_frac": 0.062351156026124954,
|
|
"expert_4_frac": 0.062363799661397934,
|
|
"expert_5_frac": 0.06258995085954666,
|
|
"expert_6_frac": 0.06269625574350357,
|
|
"expert_7_frac": 0.06271589547395706,
|
|
"expert_8_frac": 0.06235557794570923,
|
|
"expert_9_frac": 0.06264617294073105,
|
|
"expert_frac_max": 0.06271589547395706,
|
|
"expert_frac_min": 0.06233026087284088,
|
|
"expert_frac_std": 0.00014301776536740363,
|
|
"grad_norm": 0.38073164224624634,
|
|
"learning_rate": 0.00020905224108528923,
|
|
"loss": 3.5458,
|
|
"router_aux_loss": 2.000334331935102,
|
|
"router_entropy": 2.10179190532389,
|
|
"router_entropy_norm": 0.7580611896978355,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.40892193308550184,
|
|
"expert_0_frac": 0.062395963817834854,
|
|
"expert_10_frac": 0.06260203570127487,
|
|
"expert_11_frac": 0.06264384835958481,
|
|
"expert_12_frac": 0.06235386058688164,
|
|
"expert_13_frac": 0.06258896738290787,
|
|
"expert_14_frac": 0.0623982809484005,
|
|
"expert_15_frac": 0.06242229789495468,
|
|
"expert_1_frac": 0.06261791288852692,
|
|
"expert_2_frac": 0.06222543120384216,
|
|
"expert_3_frac": 0.06262610852718353,
|
|
"expert_4_frac": 0.06244849041104317,
|
|
"expert_5_frac": 0.062327079474925995,
|
|
"expert_6_frac": 0.06249148026108742,
|
|
"expert_7_frac": 0.0627094954252243,
|
|
"expert_8_frac": 0.06250277161598206,
|
|
"expert_9_frac": 0.06264599412679672,
|
|
"expert_frac_max": 0.0627094954252243,
|
|
"expert_frac_min": 0.06222543120384216,
|
|
"expert_frac_std": 0.00013919193588662893,
|
|
"grad_norm": 0.36956146359443665,
|
|
"learning_rate": 0.00020621504966679337,
|
|
"loss": 3.5394,
|
|
"router_aux_loss": 2.000408248752356,
|
|
"router_entropy": 2.1004575939973193,
|
|
"router_entropy_norm": 0.7575799386143741,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.41511771995043373,
|
|
"expert_0_frac": 0.06248762831091881,
|
|
"expert_10_frac": 0.06243772804737091,
|
|
"expert_11_frac": 0.06252548098564148,
|
|
"expert_12_frac": 0.06232022866606712,
|
|
"expert_13_frac": 0.06252936273813248,
|
|
"expert_14_frac": 0.062245216220617294,
|
|
"expert_15_frac": 0.06240775063633919,
|
|
"expert_1_frac": 0.06255979090929031,
|
|
"expert_2_frac": 0.06250588595867157,
|
|
"expert_3_frac": 0.06262928992509842,
|
|
"expert_4_frac": 0.062493931502103806,
|
|
"expert_5_frac": 0.06225007027387619,
|
|
"expert_6_frac": 0.06273949146270752,
|
|
"expert_7_frac": 0.06255599111318588,
|
|
"expert_8_frac": 0.06259478628635406,
|
|
"expert_9_frac": 0.06271740049123764,
|
|
"expert_frac_max": 0.06273949146270752,
|
|
"expert_frac_min": 0.062245216220617294,
|
|
"expert_frac_std": 0.00014395613106898963,
|
|
"grad_norm": 0.4224052131175995,
|
|
"learning_rate": 0.00020335426290855494,
|
|
"loss": 3.5179,
|
|
"router_aux_loss": 2.0003286020457747,
|
|
"router_entropy": 2.0987680555631716,
|
|
"router_entropy_norm": 0.7569705664342901,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.42131350681536556,
|
|
"expert_0_frac": 0.06220756098628044,
|
|
"expert_10_frac": 0.06257414072751999,
|
|
"expert_11_frac": 0.06239807978272438,
|
|
"expert_12_frac": 0.06225958094000816,
|
|
"expert_13_frac": 0.062421929091215134,
|
|
"expert_14_frac": 0.062199681997299194,
|
|
"expert_15_frac": 0.062410350888967514,
|
|
"expert_1_frac": 0.06256657093763351,
|
|
"expert_2_frac": 0.062459468841552734,
|
|
"expert_3_frac": 0.06267154961824417,
|
|
"expert_4_frac": 0.06252565234899521,
|
|
"expert_5_frac": 0.06251086294651031,
|
|
"expert_6_frac": 0.06265921145677567,
|
|
"expert_7_frac": 0.06275542080402374,
|
|
"expert_8_frac": 0.0625537633895874,
|
|
"expert_9_frac": 0.06282620877027512,
|
|
"expert_frac_max": 0.06282620877027512,
|
|
"expert_frac_min": 0.062199681997299194,
|
|
"expert_frac_std": 0.00018251939036417753,
|
|
"grad_norm": 0.38169124722480774,
|
|
"learning_rate": 0.00020047108157862255,
|
|
"loss": 3.5252,
|
|
"router_aux_loss": 2.0002493160963057,
|
|
"router_entropy": 2.0952287341778475,
|
|
"router_entropy_norm": 0.7556940260816102,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.4275092936802974,
|
|
"expert_0_frac": 0.06231572479009628,
|
|
"expert_10_frac": 0.06259610503911972,
|
|
"expert_11_frac": 0.06259749084711075,
|
|
"expert_12_frac": 0.06229960918426514,
|
|
"expert_13_frac": 0.06249750778079033,
|
|
"expert_14_frac": 0.06224007532000542,
|
|
"expert_15_frac": 0.062429435551166534,
|
|
"expert_1_frac": 0.06246008723974228,
|
|
"expert_2_frac": 0.06253749132156372,
|
|
"expert_3_frac": 0.06280416995286942,
|
|
"expert_4_frac": 0.06255106627941132,
|
|
"expert_5_frac": 0.062294136732816696,
|
|
"expert_6_frac": 0.06257584691047668,
|
|
"expert_7_frac": 0.06268903613090515,
|
|
"expert_8_frac": 0.06247768551111221,
|
|
"expert_9_frac": 0.06263455748558044,
|
|
"expert_frac_max": 0.06280416995286942,
|
|
"expert_frac_min": 0.06224007532000542,
|
|
"expert_frac_std": 0.00015620443446096033,
|
|
"grad_norm": 0.37302330136299133,
|
|
"learning_rate": 0.0001975667158447968,
|
|
"loss": 3.5342,
|
|
"router_aux_loss": 2.000293505489826,
|
|
"router_entropy": 2.0930864307905237,
|
|
"router_entropy_norm": 0.7549213534633673,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.43370508054522927,
|
|
"expert_0_frac": 0.062250856310129166,
|
|
"expert_10_frac": 0.06260839104652405,
|
|
"expert_11_frac": 0.06263632327318192,
|
|
"expert_12_frac": 0.06220966950058937,
|
|
"expert_13_frac": 0.0625518411397934,
|
|
"expert_14_frac": 0.06223663315176964,
|
|
"expert_15_frac": 0.0625864714384079,
|
|
"expert_1_frac": 0.06254573166370392,
|
|
"expert_2_frac": 0.062443047761917114,
|
|
"expert_3_frac": 0.06261158734560013,
|
|
"expert_4_frac": 0.062505804002285,
|
|
"expert_5_frac": 0.06240115687251091,
|
|
"expert_6_frac": 0.06271927803754807,
|
|
"expert_7_frac": 0.06251521408557892,
|
|
"expert_8_frac": 0.0624253936111927,
|
|
"expert_9_frac": 0.0627526193857193,
|
|
"expert_frac_max": 0.0627526193857193,
|
|
"expert_frac_min": 0.06220966950058937,
|
|
"expert_frac_std": 0.00016360751760657877,
|
|
"grad_norm": 0.39231953024864197,
|
|
"learning_rate": 0.00019464238476668252,
|
|
"loss": 3.5022,
|
|
"router_aux_loss": 2.000298661738634,
|
|
"router_entropy": 2.092628412457804,
|
|
"router_entropy_norm": 0.7547561582690545,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.4399008674101611,
|
|
"expert_0_frac": 0.062313877046108246,
|
|
"expert_10_frac": 0.06249168515205383,
|
|
"expert_11_frac": 0.06265368312597275,
|
|
"expert_12_frac": 0.062291502952575684,
|
|
"expert_13_frac": 0.06255083531141281,
|
|
"expert_14_frac": 0.06223292276263237,
|
|
"expert_15_frac": 0.06240782514214516,
|
|
"expert_1_frac": 0.06274271756410599,
|
|
"expert_2_frac": 0.0625762939453125,
|
|
"expert_3_frac": 0.06271133571863174,
|
|
"expert_4_frac": 0.06232191249728203,
|
|
"expert_5_frac": 0.062295157462358475,
|
|
"expert_6_frac": 0.06254404038190842,
|
|
"expert_7_frac": 0.06275083869695663,
|
|
"expert_8_frac": 0.062450066208839417,
|
|
"expert_9_frac": 0.06266532838344574,
|
|
"expert_frac_max": 0.06275083869695663,
|
|
"expert_frac_min": 0.06223292276263237,
|
|
"expert_frac_std": 0.00017549331823829561,
|
|
"grad_norm": 0.37481606006622314,
|
|
"learning_rate": 0.0001916993157840083,
|
|
"loss": 3.503,
|
|
"router_aux_loss": 2.0003165489435197,
|
|
"router_entropy": 2.089450304582715,
|
|
"router_entropy_norm": 0.7536098981513543,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.44609665427509293,
|
|
"expert_0_frac": 0.06234418973326683,
|
|
"expert_10_frac": 0.062443748116493225,
|
|
"expert_11_frac": 0.06271863728761673,
|
|
"expert_12_frac": 0.06224535033106804,
|
|
"expert_13_frac": 0.062424443662166595,
|
|
"expert_14_frac": 0.062366239726543427,
|
|
"expert_15_frac": 0.06260008364915848,
|
|
"expert_1_frac": 0.06257687509059906,
|
|
"expert_2_frac": 0.0623343363404274,
|
|
"expert_3_frac": 0.0625820904970169,
|
|
"expert_4_frac": 0.06246402859687805,
|
|
"expert_5_frac": 0.06257043033838272,
|
|
"expert_6_frac": 0.0626128613948822,
|
|
"expert_7_frac": 0.06262767314910889,
|
|
"expert_8_frac": 0.06241843104362488,
|
|
"expert_9_frac": 0.06267061084508896,
|
|
"expert_frac_max": 0.06271863728761673,
|
|
"expert_frac_min": 0.06224535033106804,
|
|
"expert_frac_std": 0.00013780524022877216,
|
|
"grad_norm": 0.39895886182785034,
|
|
"learning_rate": 0.00018873874420142906,
|
|
"loss": 3.4977,
|
|
"router_aux_loss": 2.000358630567789,
|
|
"router_entropy": 2.087255741332968,
|
|
"router_entropy_norm": 0.7528183767720226,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.45229244114002476,
|
|
"expert_0_frac": 0.0623842254281044,
|
|
"expert_10_frac": 0.06250423938035965,
|
|
"expert_11_frac": 0.06252159178256989,
|
|
"expert_12_frac": 0.06225087121129036,
|
|
"expert_13_frac": 0.06249937787652016,
|
|
"expert_14_frac": 0.06226740777492523,
|
|
"expert_15_frac": 0.06252508610486984,
|
|
"expert_1_frac": 0.06258075684309006,
|
|
"expert_2_frac": 0.06247315928339958,
|
|
"expert_3_frac": 0.06254542618989944,
|
|
"expert_4_frac": 0.06261557340621948,
|
|
"expert_5_frac": 0.06244013085961342,
|
|
"expert_6_frac": 0.06266670674085617,
|
|
"expert_7_frac": 0.06256407499313354,
|
|
"expert_8_frac": 0.062476083636283875,
|
|
"expert_9_frac": 0.06268531084060669,
|
|
"expert_frac_max": 0.06268531084060669,
|
|
"expert_frac_min": 0.06225087121129036,
|
|
"expert_frac_std": 0.00012213051377329975,
|
|
"grad_norm": 0.37105000019073486,
|
|
"learning_rate": 0.0001857619126700273,
|
|
"loss": 3.4968,
|
|
"router_aux_loss": 2.0002850928902625,
|
|
"router_entropy": 2.085349079382916,
|
|
"router_entropy_norm": 0.7521306938370246,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.45848822800495664,
|
|
"expert_0_frac": 0.062216244637966156,
|
|
"expert_10_frac": 0.06255313754081726,
|
|
"expert_11_frac": 0.06254009902477264,
|
|
"expert_12_frac": 0.06221632659435272,
|
|
"expert_13_frac": 0.06249389052391052,
|
|
"expert_14_frac": 0.06222785636782646,
|
|
"expert_15_frac": 0.06247757747769356,
|
|
"expert_1_frac": 0.06254442036151886,
|
|
"expert_2_frac": 0.06252530217170715,
|
|
"expert_3_frac": 0.06265315413475037,
|
|
"expert_4_frac": 0.06257827579975128,
|
|
"expert_5_frac": 0.062381528317928314,
|
|
"expert_6_frac": 0.06273956596851349,
|
|
"expert_7_frac": 0.06278820335865021,
|
|
"expert_8_frac": 0.06248488277196884,
|
|
"expert_9_frac": 0.06257954239845276,
|
|
"expert_frac_max": 0.06278820335865021,
|
|
"expert_frac_min": 0.062216244637966156,
|
|
"expert_frac_std": 0.0001699533750070259,
|
|
"grad_norm": 0.42140495777130127,
|
|
"learning_rate": 0.00018277007066573098,
|
|
"loss": 3.4851,
|
|
"router_aux_loss": 2.0002259370684623,
|
|
"router_entropy": 2.0829126729443668,
|
|
"router_entropy_norm": 0.7512519459654033,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.4646840148698885,
|
|
"expert_0_frac": 0.06227515637874603,
|
|
"expert_10_frac": 0.06233804672956467,
|
|
"expert_11_frac": 0.06253945827484131,
|
|
"expert_12_frac": 0.06221473962068558,
|
|
"expert_13_frac": 0.06275515258312225,
|
|
"expert_14_frac": 0.06235675886273384,
|
|
"expert_15_frac": 0.06251177936792374,
|
|
"expert_1_frac": 0.06242473050951958,
|
|
"expert_2_frac": 0.06246200203895569,
|
|
"expert_3_frac": 0.06270095705986023,
|
|
"expert_4_frac": 0.06234190613031387,
|
|
"expert_5_frac": 0.06254827976226807,
|
|
"expert_6_frac": 0.06270807236433029,
|
|
"expert_7_frac": 0.06254412978887558,
|
|
"expert_8_frac": 0.06251411885023117,
|
|
"expert_9_frac": 0.0627647191286087,
|
|
"expert_frac_max": 0.0627647191286087,
|
|
"expert_frac_min": 0.06221473962068558,
|
|
"expert_frac_std": 0.00017045202548615634,
|
|
"grad_norm": 0.39511430263519287,
|
|
"learning_rate": 0.0001797644739648662,
|
|
"loss": 3.4754,
|
|
"router_aux_loss": 2.0003580312430858,
|
|
"router_entropy": 2.081217644525071,
|
|
"router_entropy_norm": 0.7506405936917324,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.4708798017348203,
|
|
"expert_0_frac": 0.062218230217695236,
|
|
"expert_10_frac": 0.06246109679341316,
|
|
"expert_11_frac": 0.06242981180548668,
|
|
"expert_12_frac": 0.06220239773392677,
|
|
"expert_13_frac": 0.06252938508987427,
|
|
"expert_14_frac": 0.06232874095439911,
|
|
"expert_15_frac": 0.06253427267074585,
|
|
"expert_1_frac": 0.06251819431781769,
|
|
"expert_2_frac": 0.06247918680310249,
|
|
"expert_3_frac": 0.06275278329849243,
|
|
"expert_4_frac": 0.062442347407341,
|
|
"expert_5_frac": 0.06229308620095253,
|
|
"expert_6_frac": 0.0625835508108139,
|
|
"expert_7_frac": 0.06279103457927704,
|
|
"expert_8_frac": 0.06255204975605011,
|
|
"expert_9_frac": 0.06288384646177292,
|
|
"expert_frac_max": 0.06288384646177292,
|
|
"expert_frac_min": 0.06220239773392677,
|
|
"expert_frac_std": 0.00019292465003672987,
|
|
"grad_norm": 0.39034244418144226,
|
|
"learning_rate": 0.00017674638411706612,
|
|
"loss": 3.4804,
|
|
"router_aux_loss": 2.0002825504541395,
|
|
"router_entropy": 2.0786126318449774,
|
|
"router_entropy_norm": 0.7497010339729764,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.4770755885997522,
|
|
"expert_0_frac": 0.06214878335595131,
|
|
"expert_10_frac": 0.06230198219418526,
|
|
"expert_11_frac": 0.06274259835481644,
|
|
"expert_12_frac": 0.06231401488184929,
|
|
"expert_13_frac": 0.0625581219792366,
|
|
"expert_14_frac": 0.062250979244709015,
|
|
"expert_15_frac": 0.06247100979089737,
|
|
"expert_1_frac": 0.06267211586236954,
|
|
"expert_2_frac": 0.06251274049282074,
|
|
"expert_3_frac": 0.06262460350990295,
|
|
"expert_4_frac": 0.06242990493774414,
|
|
"expert_5_frac": 0.06250366568565369,
|
|
"expert_6_frac": 0.06266610324382782,
|
|
"expert_7_frac": 0.06275343149900436,
|
|
"expert_8_frac": 0.06240144744515419,
|
|
"expert_9_frac": 0.06264853477478027,
|
|
"expert_frac_max": 0.06275343149900436,
|
|
"expert_frac_min": 0.06214878335595131,
|
|
"expert_frac_std": 0.00018185534281656146,
|
|
"grad_norm": 0.36116084456443787,
|
|
"learning_rate": 0.00017371706791575632,
|
|
"loss": 3.474,
|
|
"router_aux_loss": 2.000293119698763,
|
|
"router_entropy": 2.077186061926186,
|
|
"router_entropy_norm": 0.749186507636146,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.483271375464684,
|
|
"expert_0_frac": 0.06240856274962425,
|
|
"expert_10_frac": 0.06241278350353241,
|
|
"expert_11_frac": 0.06253360211849213,
|
|
"expert_12_frac": 0.06221204996109009,
|
|
"expert_13_frac": 0.06244863197207451,
|
|
"expert_14_frac": 0.062473393976688385,
|
|
"expert_15_frac": 0.06247289106249809,
|
|
"expert_1_frac": 0.06267870217561722,
|
|
"expert_2_frac": 0.06242802366614342,
|
|
"expert_3_frac": 0.06274519115686417,
|
|
"expert_4_frac": 0.062426675111055374,
|
|
"expert_5_frac": 0.06239253655076027,
|
|
"expert_6_frac": 0.0625360757112503,
|
|
"expert_7_frac": 0.062477219849824905,
|
|
"expert_8_frac": 0.06259272992610931,
|
|
"expert_9_frac": 0.06276094913482666,
|
|
"expert_frac_max": 0.06276094913482666,
|
|
"expert_frac_min": 0.06221204996109009,
|
|
"expert_frac_std": 0.00014065694995224476,
|
|
"grad_norm": 0.39785218238830566,
|
|
"learning_rate": 0.00017067779686643893,
|
|
"loss": 3.4499,
|
|
"router_aux_loss": 2.0003442023694515,
|
|
"router_entropy": 2.0774265551939606,
|
|
"router_entropy_norm": 0.7492732472473423,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.48946716232961585,
|
|
"expert_0_frac": 0.06253805756568909,
|
|
"expert_10_frac": 0.062361087650060654,
|
|
"expert_11_frac": 0.06251271814107895,
|
|
"expert_12_frac": 0.06228022277355194,
|
|
"expert_13_frac": 0.062415316700935364,
|
|
"expert_14_frac": 0.06231295317411423,
|
|
"expert_15_frac": 0.062470030039548874,
|
|
"expert_1_frac": 0.06256447732448578,
|
|
"expert_2_frac": 0.06246081367135048,
|
|
"expert_3_frac": 0.0626181960105896,
|
|
"expert_4_frac": 0.06245271861553192,
|
|
"expert_5_frac": 0.06253720074892044,
|
|
"expert_6_frac": 0.06264904141426086,
|
|
"expert_7_frac": 0.06271599978208542,
|
|
"expert_8_frac": 0.06236222758889198,
|
|
"expert_9_frac": 0.06274896115064621,
|
|
"expert_frac_max": 0.06274896115064621,
|
|
"expert_frac_min": 0.06228022277355194,
|
|
"expert_frac_std": 0.00013793223479297012,
|
|
"grad_norm": 0.36815178394317627,
|
|
"learning_rate": 0.00016762984665299937,
|
|
"loss": 3.4589,
|
|
"router_aux_loss": 2.000360932648182,
|
|
"router_entropy": 2.0749111812810104,
|
|
"router_entropy_norm": 0.7483660178797937,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.49566294919454773,
|
|
"expert_0_frac": 0.062341488897800446,
|
|
"expert_10_frac": 0.0626181811094284,
|
|
"expert_11_frac": 0.06261951476335526,
|
|
"expert_12_frac": 0.06203712150454521,
|
|
"expert_13_frac": 0.06248367950320244,
|
|
"expert_14_frac": 0.06227276101708412,
|
|
"expert_15_frac": 0.06273195147514343,
|
|
"expert_1_frac": 0.062674380838871,
|
|
"expert_2_frac": 0.06232789531350136,
|
|
"expert_3_frac": 0.06270692497491837,
|
|
"expert_4_frac": 0.06244327127933502,
|
|
"expert_5_frac": 0.06220940500497818,
|
|
"expert_6_frac": 0.06272995471954346,
|
|
"expert_7_frac": 0.06274814158678055,
|
|
"expert_8_frac": 0.06237934157252312,
|
|
"expert_9_frac": 0.06267600506544113,
|
|
"expert_frac_max": 0.06274814158678055,
|
|
"expert_frac_min": 0.06203712150454521,
|
|
"expert_frac_std": 0.00021967220527585596,
|
|
"grad_norm": 0.36331707239151,
|
|
"learning_rate": 0.00016457449660225917,
|
|
"loss": 3.456,
|
|
"router_aux_loss": 2.000295414030552,
|
|
"router_entropy": 2.0727119484667975,
|
|
"router_entropy_norm": 0.7475728123110874,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.5,
|
|
"eval_loss": 2.835195779800415,
|
|
"eval_runtime": 33.137,
|
|
"eval_samples_per_second": 52.147,
|
|
"eval_steps_per_second": 6.518,
|
|
"expert_0_frac": 0.06223877891898155,
|
|
"expert_10_frac": 0.06225793808698654,
|
|
"expert_11_frac": 0.06256310641765594,
|
|
"expert_12_frac": 0.06226780265569687,
|
|
"expert_13_frac": 0.0626240149140358,
|
|
"expert_14_frac": 0.06233023852109909,
|
|
"expert_15_frac": 0.06236087530851364,
|
|
"expert_1_frac": 0.06266454607248306,
|
|
"expert_2_frac": 0.062462449073791504,
|
|
"expert_3_frac": 0.06261497735977173,
|
|
"expert_4_frac": 0.06259915977716446,
|
|
"expert_5_frac": 0.06261510401964188,
|
|
"expert_6_frac": 0.06256020814180374,
|
|
"expert_7_frac": 0.06260185688734055,
|
|
"expert_8_frac": 0.06254878640174866,
|
|
"expert_9_frac": 0.06269016861915588,
|
|
"expert_frac_max": 0.06269016861915588,
|
|
"expert_frac_min": 0.06223877891898155,
|
|
"expert_frac_std": 0.00015612253628205508,
|
|
"router_aux_loss": 2.0002878901270247,
|
|
"router_entropy": 2.0733980220131243,
|
|
"router_entropy_norm": 0.7478202610368301,
|
|
"step": 4035
|
|
},
|
|
{
|
|
"epoch": 0.5018587360594795,
|
|
"expert_0_frac": 0.06223723292350769,
|
|
"expert_10_frac": 0.06237826868891716,
|
|
"expert_11_frac": 0.06259053945541382,
|
|
"expert_12_frac": 0.06238409876823425,
|
|
"expert_13_frac": 0.06224346160888672,
|
|
"expert_14_frac": 0.06210483983159065,
|
|
"expert_15_frac": 0.06264093518257141,
|
|
"expert_1_frac": 0.06250996142625809,
|
|
"expert_2_frac": 0.06256218254566193,
|
|
"expert_3_frac": 0.06271038204431534,
|
|
"expert_4_frac": 0.06232079491019249,
|
|
"expert_5_frac": 0.06244299188256264,
|
|
"expert_6_frac": 0.06276946514844894,
|
|
"expert_7_frac": 0.06290285289287567,
|
|
"expert_8_frac": 0.06239217147231102,
|
|
"expert_9_frac": 0.06280983984470367,
|
|
"expert_frac_max": 0.06290285289287567,
|
|
"expert_frac_min": 0.06210483983159065,
|
|
"expert_frac_std": 0.00022707448806613684,
|
|
"grad_norm": 0.38191738724708557,
|
|
"learning_rate": 0.0001615130291469992,
|
|
"loss": 3.4508,
|
|
"router_aux_loss": 2.0002857352296513,
|
|
"router_entropy": 2.071026981290844,
|
|
"router_entropy_norm": 0.7469650888638851,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.5080545229244114,
|
|
"expert_0_frac": 0.06229468062520027,
|
|
"expert_10_frac": 0.06243317574262619,
|
|
"expert_11_frac": 0.06253965944051743,
|
|
"expert_12_frac": 0.06219393387436867,
|
|
"expert_13_frac": 0.06261800229549408,
|
|
"expert_14_frac": 0.06220797449350357,
|
|
"expert_15_frac": 0.06253494322299957,
|
|
"expert_1_frac": 0.06259635090827942,
|
|
"expert_2_frac": 0.062486693263053894,
|
|
"expert_3_frac": 0.062488868832588196,
|
|
"expert_4_frac": 0.062300536781549454,
|
|
"expert_5_frac": 0.06251173466444016,
|
|
"expert_6_frac": 0.0627284049987793,
|
|
"expert_7_frac": 0.06274477392435074,
|
|
"expert_8_frac": 0.06251753866672516,
|
|
"expert_9_frac": 0.0628027394413948,
|
|
"expert_frac_max": 0.0628027394413948,
|
|
"expert_frac_min": 0.06219393387436867,
|
|
"expert_frac_std": 0.00018181400082539767,
|
|
"grad_norm": 0.3982459008693695,
|
|
"learning_rate": 0.00015844672928768015,
|
|
"loss": 3.4419,
|
|
"router_aux_loss": 2.0002834285795688,
|
|
"router_entropy": 2.069342819737891,
|
|
"router_entropy_norm": 0.7463576559837599,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.5142503097893433,
|
|
"expert_0_frac": 0.06240158900618553,
|
|
"expert_10_frac": 0.06245036423206329,
|
|
"expert_11_frac": 0.06251361221075058,
|
|
"expert_12_frac": 0.062208451330661774,
|
|
"expert_13_frac": 0.0623130165040493,
|
|
"expert_14_frac": 0.06229441985487938,
|
|
"expert_15_frac": 0.06245220825076103,
|
|
"expert_1_frac": 0.06258202344179153,
|
|
"expert_2_frac": 0.06249164044857025,
|
|
"expert_3_frac": 0.06279388070106506,
|
|
"expert_4_frac": 0.06255616247653961,
|
|
"expert_5_frac": 0.0623348131775856,
|
|
"expert_6_frac": 0.06263235211372375,
|
|
"expert_7_frac": 0.06268365681171417,
|
|
"expert_8_frac": 0.06251730024814606,
|
|
"expert_9_frac": 0.06277452409267426,
|
|
"expert_frac_max": 0.06279388070106506,
|
|
"expert_frac_min": 0.062208451330661774,
|
|
"expert_frac_std": 0.0001687915064394474,
|
|
"grad_norm": 0.42543739080429077,
|
|
"learning_rate": 0.0001553768840530846,
|
|
"loss": 3.4514,
|
|
"router_aux_loss": 2.0002649518847466,
|
|
"router_entropy": 2.0675219970072307,
|
|
"router_entropy_norm": 0.7457009330027945,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.5204460966542751,
|
|
"expert_0_frac": 0.062248531728982925,
|
|
"expert_10_frac": 0.06262663751840591,
|
|
"expert_11_frac": 0.06247673183679581,
|
|
"expert_12_frac": 0.062475770711898804,
|
|
"expert_13_frac": 0.06257357448339462,
|
|
"expert_14_frac": 0.06223734840750694,
|
|
"expert_15_frac": 0.06242147088050842,
|
|
"expert_1_frac": 0.06263577193021774,
|
|
"expert_2_frac": 0.06257237493991852,
|
|
"expert_3_frac": 0.06253638863563538,
|
|
"expert_4_frac": 0.06244934722781181,
|
|
"expert_5_frac": 0.06250207126140594,
|
|
"expert_6_frac": 0.06273172795772552,
|
|
"expert_7_frac": 0.0626993179321289,
|
|
"expert_8_frac": 0.062292035669088364,
|
|
"expert_9_frac": 0.06252092123031616,
|
|
"expert_frac_max": 0.06273172795772552,
|
|
"expert_frac_min": 0.06223734840750694,
|
|
"expert_frac_std": 0.0001473423035349697,
|
|
"grad_norm": 0.3881041407585144,
|
|
"learning_rate": 0.00015230478196010788,
|
|
"loss": 3.4374,
|
|
"router_aux_loss": 2.000279816389084,
|
|
"router_entropy": 2.0661936601996422,
|
|
"router_entropy_norm": 0.74522183677156,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.5266418835192069,
|
|
"expert_0_frac": 0.062292248010635376,
|
|
"expert_10_frac": 0.0625143051147461,
|
|
"expert_11_frac": 0.06272810697555542,
|
|
"expert_12_frac": 0.062146659940481186,
|
|
"expert_13_frac": 0.06232764571905136,
|
|
"expert_14_frac": 0.06244494393467903,
|
|
"expert_15_frac": 0.06246032193303108,
|
|
"expert_1_frac": 0.06257320195436478,
|
|
"expert_2_frac": 0.06242893263697624,
|
|
"expert_3_frac": 0.06263642758131027,
|
|
"expert_4_frac": 0.06243368983268738,
|
|
"expert_5_frac": 0.06238401681184769,
|
|
"expert_6_frac": 0.06271476298570633,
|
|
"expert_7_frac": 0.06273151189088821,
|
|
"expert_8_frac": 0.06250733137130737,
|
|
"expert_9_frac": 0.06267590820789337,
|
|
"expert_frac_max": 0.06273151189088821,
|
|
"expert_frac_min": 0.062146659940481186,
|
|
"expert_frac_std": 0.00016973471792880446,
|
|
"grad_norm": 0.3913940191268921,
|
|
"learning_rate": 0.00014923171247292443,
|
|
"loss": 3.4388,
|
|
"router_aux_loss": 2.000279443114996,
|
|
"router_entropy": 2.0648219058538477,
|
|
"router_entropy_norm": 0.7447270809735611,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.5328376703841388,
|
|
"expert_0_frac": 0.062424205243587494,
|
|
"expert_10_frac": 0.06258242577314377,
|
|
"expert_11_frac": 0.062486547976732254,
|
|
"expert_12_frac": 0.06218972057104111,
|
|
"expert_13_frac": 0.06255097687244415,
|
|
"expert_14_frac": 0.06233542412519455,
|
|
"expert_15_frac": 0.06249724701046944,
|
|
"expert_1_frac": 0.062383346259593964,
|
|
"expert_2_frac": 0.062382712960243225,
|
|
"expert_3_frac": 0.0627749040722847,
|
|
"expert_4_frac": 0.0625402182340622,
|
|
"expert_5_frac": 0.06236400082707405,
|
|
"expert_6_frac": 0.06257309019565582,
|
|
"expert_7_frac": 0.06259342283010483,
|
|
"expert_8_frac": 0.06245896965265274,
|
|
"expert_9_frac": 0.0628628209233284,
|
|
"expert_frac_max": 0.0628628209233284,
|
|
"expert_frac_min": 0.06218972057104111,
|
|
"expert_frac_std": 0.0001651518396101892,
|
|
"grad_norm": 0.3753291070461273,
|
|
"learning_rate": 0.0001461589654617563,
|
|
"loss": 3.4265,
|
|
"router_aux_loss": 2.000321349054575,
|
|
"router_entropy": 2.0635299008215466,
|
|
"router_entropy_norm": 0.744261088660335,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.5390334572490706,
|
|
"expert_0_frac": 0.06218123808503151,
|
|
"expert_10_frac": 0.0625486820936203,
|
|
"expert_11_frac": 0.06267302483320236,
|
|
"expert_12_frac": 0.06233431026339531,
|
|
"expert_13_frac": 0.06256947666406631,
|
|
"expert_14_frac": 0.062171194702386856,
|
|
"expert_15_frac": 0.06244954094290733,
|
|
"expert_1_frac": 0.0626150518655777,
|
|
"expert_2_frac": 0.062484242022037506,
|
|
"expert_3_frac": 0.06249242275953293,
|
|
"expert_4_frac": 0.06238124892115593,
|
|
"expert_5_frac": 0.06239190697669983,
|
|
"expert_6_frac": 0.06276813894510269,
|
|
"expert_7_frac": 0.06265375018119812,
|
|
"expert_8_frac": 0.06257021427154541,
|
|
"expert_9_frac": 0.06271558254957199,
|
|
"expert_frac_max": 0.06276813894510269,
|
|
"expert_frac_min": 0.062171194702386856,
|
|
"expert_frac_std": 0.0001757133868522942,
|
|
"grad_norm": 0.35662841796875,
|
|
"learning_rate": 0.00014308783066147178,
|
|
"loss": 3.4153,
|
|
"router_aux_loss": 2.000260810405016,
|
|
"router_entropy": 2.0623221289614837,
|
|
"router_entropy_norm": 0.7438254770420755,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.5452292441140025,
|
|
"expert_0_frac": 0.062258683145046234,
|
|
"expert_10_frac": 0.062456462532281876,
|
|
"expert_11_frac": 0.06260629743337631,
|
|
"expert_12_frac": 0.0621049627661705,
|
|
"expert_13_frac": 0.062417272478342056,
|
|
"expert_14_frac": 0.06238047406077385,
|
|
"expert_15_frac": 0.06264665722846985,
|
|
"expert_1_frac": 0.0626235082745552,
|
|
"expert_2_frac": 0.06254547834396362,
|
|
"expert_3_frac": 0.0626249685883522,
|
|
"expert_4_frac": 0.06244637444615364,
|
|
"expert_5_frac": 0.06245977059006691,
|
|
"expert_6_frac": 0.06254879385232925,
|
|
"expert_7_frac": 0.06266503781080246,
|
|
"expert_8_frac": 0.06235170364379883,
|
|
"expert_9_frac": 0.06286357343196869,
|
|
"expert_frac_max": 0.06286357343196869,
|
|
"expert_frac_min": 0.0621049627661705,
|
|
"expert_frac_std": 0.00018098823784384876,
|
|
"grad_norm": 0.36747997999191284,
|
|
"learning_rate": 0.00014001959713024002,
|
|
"loss": 3.4252,
|
|
"router_aux_loss": 2.000273561477661,
|
|
"router_entropy": 2.0592385633910695,
|
|
"router_entropy_norm": 0.7427133158529023,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.5514250309789344,
|
|
"expert_0_frac": 0.06231154128909111,
|
|
"expert_10_frac": 0.06250350177288055,
|
|
"expert_11_frac": 0.06245879456400871,
|
|
"expert_12_frac": 0.06217548996210098,
|
|
"expert_13_frac": 0.062260840088129044,
|
|
"expert_14_frac": 0.06228562816977501,
|
|
"expert_15_frac": 0.06255662441253662,
|
|
"expert_1_frac": 0.0626593753695488,
|
|
"expert_2_frac": 0.062435850501060486,
|
|
"expert_3_frac": 0.06265868246555328,
|
|
"expert_4_frac": 0.0625995323061943,
|
|
"expert_5_frac": 0.06244940683245659,
|
|
"expert_6_frac": 0.06266835331916809,
|
|
"expert_7_frac": 0.06271927058696747,
|
|
"expert_8_frac": 0.06248893216252327,
|
|
"expert_9_frac": 0.06276819854974747,
|
|
"expert_frac_max": 0.06276819854974747,
|
|
"expert_frac_min": 0.06217548996210098,
|
|
"expert_frac_std": 0.00017597920668777078,
|
|
"grad_norm": 0.41151490807533264,
|
|
"learning_rate": 0.00013695555270847056,
|
|
"loss": 3.427,
|
|
"router_aux_loss": 2.0002112005651,
|
|
"router_entropy": 2.0581923769166073,
|
|
"router_entropy_norm": 0.7423359838432644,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.5576208178438662,
|
|
"expert_0_frac": 0.062337107956409454,
|
|
"expert_10_frac": 0.06262237578630447,
|
|
"expert_11_frac": 0.06275221705436707,
|
|
"expert_12_frac": 0.06221630424261093,
|
|
"expert_13_frac": 0.062493275851011276,
|
|
"expert_14_frac": 0.06234654784202576,
|
|
"expert_15_frac": 0.06257208436727524,
|
|
"expert_1_frac": 0.0625588446855545,
|
|
"expert_2_frac": 0.06234808266162872,
|
|
"expert_3_frac": 0.06260024011135101,
|
|
"expert_4_frac": 0.06243022903800011,
|
|
"expert_5_frac": 0.06240806728601456,
|
|
"expert_6_frac": 0.06267375499010086,
|
|
"expert_7_frac": 0.06255117803812027,
|
|
"expert_8_frac": 0.06241467222571373,
|
|
"expert_9_frac": 0.06267504394054413,
|
|
"expert_frac_max": 0.06275221705436707,
|
|
"expert_frac_min": 0.06221630424261093,
|
|
"expert_frac_std": 0.00014945138536859304,
|
|
"grad_norm": 0.376271516084671,
|
|
"learning_rate": 0.0001338969834782632,
|
|
"loss": 3.4129,
|
|
"router_aux_loss": 2.000335542410612,
|
|
"router_entropy": 2.0577339370921255,
|
|
"router_entropy_norm": 0.7421706366279329,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.563816604708798,
|
|
"expert_0_frac": 0.0622267946600914,
|
|
"expert_10_frac": 0.06248943507671356,
|
|
"expert_11_frac": 0.06273786723613739,
|
|
"expert_12_frac": 0.062277115881443024,
|
|
"expert_13_frac": 0.06260387599468231,
|
|
"expert_14_frac": 0.06225940212607384,
|
|
"expert_15_frac": 0.06244751811027527,
|
|
"expert_1_frac": 0.06263836473226547,
|
|
"expert_2_frac": 0.06259891390800476,
|
|
"expert_3_frac": 0.0626433789730072,
|
|
"expert_4_frac": 0.06251221895217896,
|
|
"expert_5_frac": 0.062373142689466476,
|
|
"expert_6_frac": 0.06243934854865074,
|
|
"expert_7_frac": 0.06258486211299896,
|
|
"expert_8_frac": 0.06255466490983963,
|
|
"expert_9_frac": 0.0626131221652031,
|
|
"expert_frac_max": 0.06273786723613739,
|
|
"expert_frac_min": 0.0622267946600914,
|
|
"expert_frac_std": 0.00015162151248659939,
|
|
"grad_norm": 0.39026713371276855,
|
|
"learning_rate": 0.00013084517322359637,
|
|
"loss": 3.4059,
|
|
"router_aux_loss": 2.000299784094095,
|
|
"router_entropy": 2.056280114464462,
|
|
"router_entropy_norm": 0.7416462809541173,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.5700123915737298,
|
|
"expert_0_frac": 0.06235774978995323,
|
|
"expert_10_frac": 0.06246906518936157,
|
|
"expert_11_frac": 0.06250017136335373,
|
|
"expert_12_frac": 0.062340375036001205,
|
|
"expert_13_frac": 0.06265813112258911,
|
|
"expert_14_frac": 0.062344953417778015,
|
|
"expert_15_frac": 0.06246919929981232,
|
|
"expert_1_frac": 0.0626433864235878,
|
|
"expert_2_frac": 0.0623575858771801,
|
|
"expert_3_frac": 0.06266095489263535,
|
|
"expert_4_frac": 0.06238768249750137,
|
|
"expert_5_frac": 0.06245439127087593,
|
|
"expert_6_frac": 0.06249968335032463,
|
|
"expert_7_frac": 0.0627267062664032,
|
|
"expert_8_frac": 0.06241398677229881,
|
|
"expert_9_frac": 0.06271599978208542,
|
|
"expert_frac_max": 0.0627267062664032,
|
|
"expert_frac_min": 0.062340375036001205,
|
|
"expert_frac_std": 0.00013749160279985517,
|
|
"grad_norm": 0.3813641369342804,
|
|
"learning_rate": 0.0001278014028914802,
|
|
"loss": 3.4044,
|
|
"router_aux_loss": 2.000373487025499,
|
|
"router_entropy": 2.0545436954001586,
|
|
"router_entropy_norm": 0.7410200001608734,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.5762081784386617,
|
|
"expert_0_frac": 0.06234966963529587,
|
|
"expert_10_frac": 0.062438640743494034,
|
|
"expert_11_frac": 0.06256365031003952,
|
|
"expert_12_frac": 0.062073301523923874,
|
|
"expert_13_frac": 0.06242337077856064,
|
|
"expert_14_frac": 0.062409333884716034,
|
|
"expert_15_frac": 0.06248563155531883,
|
|
"expert_1_frac": 0.06252411752939224,
|
|
"expert_2_frac": 0.06244285777211189,
|
|
"expert_3_frac": 0.06261327862739563,
|
|
"expert_4_frac": 0.06238767132163048,
|
|
"expert_5_frac": 0.06257081776857376,
|
|
"expert_6_frac": 0.06280014663934708,
|
|
"expert_7_frac": 0.06257054209709167,
|
|
"expert_8_frac": 0.06250441074371338,
|
|
"expert_9_frac": 0.06284257769584656,
|
|
"expert_frac_max": 0.06284257769584656,
|
|
"expert_frac_min": 0.062073301523923874,
|
|
"expert_frac_std": 0.0001776777789928019,
|
|
"grad_norm": 0.37938594818115234,
|
|
"learning_rate": 0.00012476695005429975,
|
|
"loss": 3.3886,
|
|
"router_aux_loss": 2.0002758315205575,
|
|
"router_entropy": 2.0531820828095078,
|
|
"router_entropy_norm": 0.7405289022278375,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.5824039653035935,
|
|
"expert_0_frac": 0.06221843138337135,
|
|
"expert_10_frac": 0.06256091594696045,
|
|
"expert_11_frac": 0.0626656711101532,
|
|
"expert_12_frac": 0.06217745691537857,
|
|
"expert_13_frac": 0.06265641748905182,
|
|
"expert_14_frac": 0.062169160693883896,
|
|
"expert_15_frac": 0.06261997669935226,
|
|
"expert_1_frac": 0.06251893937587738,
|
|
"expert_2_frac": 0.06247605010867119,
|
|
"expert_3_frac": 0.0625799298286438,
|
|
"expert_4_frac": 0.06254834681749344,
|
|
"expert_5_frac": 0.06241624429821968,
|
|
"expert_6_frac": 0.06244943290948868,
|
|
"expert_7_frac": 0.06266841292381287,
|
|
"expert_8_frac": 0.06259527057409286,
|
|
"expert_9_frac": 0.06267937272787094,
|
|
"expert_frac_max": 0.06267937272787094,
|
|
"expert_frac_min": 0.062169160693883896,
|
|
"expert_frac_std": 0.00017351898713968694,
|
|
"grad_norm": 0.36018115282058716,
|
|
"learning_rate": 0.0001217430883735755,
|
|
"loss": 3.3863,
|
|
"router_aux_loss": 2.00024845123291,
|
|
"router_entropy": 2.0513762359196943,
|
|
"router_entropy_norm": 0.7398775806397029,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.5885997521685254,
|
|
"expert_0_frac": 0.06233708932995796,
|
|
"expert_10_frac": 0.06242017820477486,
|
|
"expert_11_frac": 0.06262047588825226,
|
|
"expert_12_frac": 0.06229627877473831,
|
|
"expert_13_frac": 0.06242483854293823,
|
|
"expert_14_frac": 0.062415316700935364,
|
|
"expert_15_frac": 0.06253886222839355,
|
|
"expert_1_frac": 0.06252031028270721,
|
|
"expert_2_frac": 0.06237517297267914,
|
|
"expert_3_frac": 0.06263025850057602,
|
|
"expert_4_frac": 0.06249498575925827,
|
|
"expert_5_frac": 0.06240921840071678,
|
|
"expert_6_frac": 0.06260804086923599,
|
|
"expert_7_frac": 0.06275253742933273,
|
|
"expert_8_frac": 0.06233108788728714,
|
|
"expert_9_frac": 0.06282538175582886,
|
|
"expert_frac_max": 0.06282538175582886,
|
|
"expert_frac_min": 0.06229627877473831,
|
|
"expert_frac_std": 0.00015369657194241881,
|
|
"grad_norm": 0.3805563449859619,
|
|
"learning_rate": 0.00011873108706536448,
|
|
"loss": 3.3834,
|
|
"router_aux_loss": 2.0003303153812886,
|
|
"router_entropy": 2.049379334114492,
|
|
"router_entropy_norm": 0.739157350556826,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.5947955390334573,
|
|
"expert_0_frac": 0.0621938481926918,
|
|
"expert_10_frac": 0.06241048872470856,
|
|
"expert_11_frac": 0.06265752017498016,
|
|
"expert_12_frac": 0.062160950154066086,
|
|
"expert_13_frac": 0.06252051889896393,
|
|
"expert_14_frac": 0.06239565089344978,
|
|
"expert_15_frac": 0.06259883940219879,
|
|
"expert_1_frac": 0.06257259845733643,
|
|
"expert_2_frac": 0.06241338327527046,
|
|
"expert_3_frac": 0.06267257779836655,
|
|
"expert_4_frac": 0.06234891340136528,
|
|
"expert_5_frac": 0.06232818588614464,
|
|
"expert_6_frac": 0.06263818591833115,
|
|
"expert_7_frac": 0.06266766041517258,
|
|
"expert_8_frac": 0.06272201240062714,
|
|
"expert_9_frac": 0.06269869208335876,
|
|
"expert_frac_max": 0.06272201240062714,
|
|
"expert_frac_min": 0.062160950154066086,
|
|
"expert_frac_std": 0.00018137702136300504,
|
|
"grad_norm": 0.37568414211273193,
|
|
"learning_rate": 0.00011573221036752828,
|
|
"loss": 3.397,
|
|
"router_aux_loss": 2.000270184874535,
|
|
"router_entropy": 2.046731663371126,
|
|
"router_entropy_norm": 0.7382024051939856,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.6,
|
|
"eval_loss": 2.7643399238586426,
|
|
"eval_runtime": 34.8236,
|
|
"eval_samples_per_second": 49.622,
|
|
"eval_steps_per_second": 6.203,
|
|
"expert_0_frac": 0.06227129325270653,
|
|
"expert_10_frac": 0.06248059123754501,
|
|
"expert_11_frac": 0.06245836243033409,
|
|
"expert_12_frac": 0.062425948679447174,
|
|
"expert_13_frac": 0.062481872737407684,
|
|
"expert_14_frac": 0.06234771013259888,
|
|
"expert_15_frac": 0.062448468059301376,
|
|
"expert_1_frac": 0.06265126913785934,
|
|
"expert_2_frac": 0.062485091388225555,
|
|
"expert_3_frac": 0.06254459172487259,
|
|
"expert_4_frac": 0.06255730241537094,
|
|
"expert_5_frac": 0.06245243921875954,
|
|
"expert_6_frac": 0.06256099790334702,
|
|
"expert_7_frac": 0.06259247660636902,
|
|
"expert_8_frac": 0.06239942088723183,
|
|
"expert_9_frac": 0.06284219026565552,
|
|
"expert_frac_max": 0.06284219026565552,
|
|
"expert_frac_min": 0.06227129325270653,
|
|
"expert_frac_std": 0.00013069015403743833,
|
|
"router_aux_loss": 2.0002922967479035,
|
|
"router_entropy": 2.0467925748757057,
|
|
"router_entropy_norm": 0.7382243743753832,
|
|
"step": 4842
|
|
}
|
|
],
|
|
"logging_steps": 50,
|
|
"max_steps": 8070,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 9223372036854775807,
|
|
"save_steps": 1614,
|
|
"stateful_callbacks": {
|
|
"EarlyStoppingCallback": {
|
|
"args": {
|
|
"early_stopping_patience": 3,
|
|
"early_stopping_threshold": 0.0
|
|
},
|
|
"attributes": {
|
|
"early_stopping_patience_counter": 0
|
|
}
|
|
},
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 7.141687525260657e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|