{ "best_global_step": 4842, "best_metric": 2.7643399238586426, "best_model_checkpoint": "/vol/moe-200m-16exp-streaming/checkpoint-4842", "epoch": 0.6, "eval_steps": 807, "global_step": 4842, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.006195786864931847, "expert_0_frac": 0.06268728524446487, "expert_10_frac": 0.05253516510128975, "expert_11_frac": 0.06409072130918503, "expert_12_frac": 0.06890930235385895, "expert_13_frac": 0.06048846244812012, "expert_14_frac": 0.060645461082458496, "expert_15_frac": 0.06372006982564926, "expert_1_frac": 0.061552755534648895, "expert_2_frac": 0.05120125040411949, "expert_3_frac": 0.0811033621430397, "expert_4_frac": 0.06206492707133293, "expert_5_frac": 0.05403707176446915, "expert_6_frac": 0.07373557239770889, "expert_7_frac": 0.06307204812765121, "expert_8_frac": 0.06395463645458221, "expert_9_frac": 0.056201934814453125, "expert_frac_max": 0.0811033621430397, "expert_frac_min": 0.05120125040411949, "expert_frac_std": 0.00760139524936676, "grad_norm": 1.535045862197876, "learning_rate": 3.647642679900744e-05, "loss": 10.6442, "router_aux_loss": 2.0115064537525176, "router_entropy": 2.7032830785711606, "router_entropy_norm": 0.9750032728934159, "step": 50 }, { "epoch": 0.012391573729863693, "expert_0_frac": 0.06215367838740349, "expert_10_frac": 0.06141716241836548, "expert_11_frac": 0.06268706917762756, "expert_12_frac": 0.06260079890489578, "expert_13_frac": 0.06265205889940262, "expert_14_frac": 0.0631578266620636, "expert_15_frac": 0.06170821189880371, "expert_1_frac": 0.06163309887051582, "expert_2_frac": 0.06367384642362595, "expert_3_frac": 0.06248243525624275, "expert_4_frac": 0.06296652555465698, "expert_5_frac": 0.06345007568597794, "expert_6_frac": 0.061480410397052765, "expert_7_frac": 0.06333854794502258, "expert_8_frac": 0.06238511577248573, "expert_9_frac": 0.06221316009759903, "expert_frac_max": 0.06367384642362595, "expert_frac_min": 0.06141716241836548, "expert_frac_std": 0.0007075144094415009, "grad_norm": 1.2961140871047974, "learning_rate": 7.369727047146401e-05, "loss": 9.5611, "router_aux_loss": 2.003570516258478, "router_entropy": 2.6998490092903378, "router_entropy_norm": 0.9737646942130378, "step": 100 }, { "epoch": 0.01858736059479554, "expert_0_frac": 0.06209799647331238, "expert_10_frac": 0.0627819076180458, "expert_11_frac": 0.06212802231311798, "expert_12_frac": 0.06245502457022667, "expert_13_frac": 0.06238865107297897, "expert_14_frac": 0.062404513359069824, "expert_15_frac": 0.06227085739374161, "expert_1_frac": 0.06282500177621841, "expert_2_frac": 0.06209854781627655, "expert_3_frac": 0.06289317458868027, "expert_4_frac": 0.062445469200611115, "expert_5_frac": 0.06195049360394478, "expert_6_frac": 0.0627024918794632, "expert_7_frac": 0.06268659979104996, "expert_8_frac": 0.06310692429542542, "expert_9_frac": 0.06276433914899826, "expert_frac_max": 0.06310692429542542, "expert_frac_min": 0.06195049360394478, "expert_frac_std": 0.0003362393472343683, "grad_norm": 0.5107415318489075, "learning_rate": 0.00011091811414392058, "loss": 8.448, "router_aux_loss": 2.0038693606853486, "router_entropy": 2.6251225757847227, "router_entropy_norm": 0.9468128304525704, "step": 150 }, { "epoch": 0.024783147459727387, "expert_0_frac": 0.06296655535697937, "expert_10_frac": 0.06291854381561279, "expert_11_frac": 0.06274035573005676, "expert_12_frac": 0.06212460994720459, "expert_13_frac": 0.06220207363367081, "expert_14_frac": 0.0623633973300457, "expert_15_frac": 0.06261631846427917, "expert_1_frac": 0.062234967947006226, "expert_2_frac": 0.06269345432519913, "expert_3_frac": 0.0623706579208374, "expert_4_frac": 0.06255213916301727, "expert_5_frac": 0.06247646361589432, "expert_6_frac": 0.06270141154527664, "expert_7_frac": 0.06233640015125275, "expert_8_frac": 0.06229843944311142, "expert_9_frac": 0.06240423396229744, "expert_frac_max": 0.06296655535697937, "expert_frac_min": 0.06212460994720459, "expert_frac_std": 0.0002526248281355947, "grad_norm": 0.5576089024543762, "learning_rate": 0.00014813895781637715, "loss": 7.7918, "router_aux_loss": 2.0020354767143727, "router_entropy": 2.575498506637911, "router_entropy_norm": 0.9289147308358613, "step": 200 }, { "epoch": 0.030978934324659233, "expert_0_frac": 0.06254477053880692, "expert_10_frac": 0.06230742111802101, "expert_11_frac": 0.06271008402109146, "expert_12_frac": 0.06274610757827759, "expert_13_frac": 0.06249750405550003, "expert_14_frac": 0.0624377578496933, "expert_15_frac": 0.06226474419236183, "expert_1_frac": 0.062305860221385956, "expert_2_frac": 0.06292720884084702, "expert_3_frac": 0.0619979128241539, "expert_4_frac": 0.06242801249027252, "expert_5_frac": 0.06247518211603165, "expert_6_frac": 0.06265301257371902, "expert_7_frac": 0.062180500477552414, "expert_8_frac": 0.06276927888393402, "expert_9_frac": 0.06275466829538345, "expert_frac_max": 0.06292720884084702, "expert_frac_min": 0.0619979128241539, "expert_frac_std": 0.00025086093228310347, "grad_norm": 0.7355545163154602, "learning_rate": 0.0001853598014888337, "loss": 7.1734, "router_aux_loss": 2.00110543563962, "router_entropy": 2.563297350977858, "router_entropy_norm": 0.924514094144893, "step": 250 }, { "epoch": 0.03717472118959108, "expert_0_frac": 0.062286507338285446, "expert_10_frac": 0.06241067498922348, "expert_11_frac": 0.06250958144664764, "expert_12_frac": 0.06269891560077667, "expert_13_frac": 0.06233363598585129, "expert_14_frac": 0.06255818158388138, "expert_15_frac": 0.06255821138620377, "expert_1_frac": 0.06232194975018501, "expert_2_frac": 0.06297454982995987, "expert_3_frac": 0.062439851462841034, "expert_4_frac": 0.0625506192445755, "expert_5_frac": 0.06263585388660431, "expert_6_frac": 0.06239321082830429, "expert_7_frac": 0.06247292459011078, "expert_8_frac": 0.062302496284246445, "expert_9_frac": 0.06255285441875458, "expert_frac_max": 0.06297454982995987, "expert_frac_min": 0.062286507338285446, "expert_frac_std": 0.00017571607895661145, "grad_norm": 0.7089310884475708, "learning_rate": 0.0002225806451612903, "loss": 6.616, "router_aux_loss": 2.000896415114403, "router_entropy": 2.5682343946397306, "router_entropy_norm": 0.9262947562468021, "step": 300 }, { "epoch": 0.04337050805452292, "expert_0_frac": 0.06265521794557571, "expert_10_frac": 0.06244245171546936, "expert_11_frac": 0.06217898055911064, "expert_12_frac": 0.06211438030004501, "expert_13_frac": 0.06232894957065582, "expert_14_frac": 0.06268642842769623, "expert_15_frac": 0.062405042350292206, "expert_1_frac": 0.06251642853021622, "expert_2_frac": 0.06277015805244446, "expert_3_frac": 0.06272362172603607, "expert_4_frac": 0.06245952099561691, "expert_5_frac": 0.06266545504331589, "expert_6_frac": 0.06226136162877083, "expert_7_frac": 0.06263869255781174, "expert_8_frac": 0.06252796202898026, "expert_9_frac": 0.06262536346912384, "expert_frac_max": 0.06277015805244446, "expert_frac_min": 0.06211438030004501, "expert_frac_std": 0.00019952424918301404, "grad_norm": 0.9155972003936768, "learning_rate": 0.00025980148883374685, "loss": 6.2012, "router_aux_loss": 2.0007203175127506, "router_entropy": 2.564730280985435, "router_entropy_norm": 0.9250309143988612, "step": 350 }, { "epoch": 0.04956629491945477, "expert_0_frac": 0.06231584772467613, "expert_10_frac": 0.06259948015213013, "expert_11_frac": 0.06227356567978859, "expert_12_frac": 0.0622558556497097, "expert_13_frac": 0.06256835162639618, "expert_14_frac": 0.06262826919555664, "expert_15_frac": 0.062333691865205765, "expert_1_frac": 0.06247744336724281, "expert_2_frac": 0.062451884150505066, "expert_3_frac": 0.06259673088788986, "expert_4_frac": 0.062348831444978714, "expert_5_frac": 0.06258165836334229, "expert_6_frac": 0.06247778609395027, "expert_7_frac": 0.06272567808628082, "expert_8_frac": 0.06258414685726166, "expert_9_frac": 0.06278080493211746, "expert_frac_max": 0.06278080493211746, "expert_frac_min": 0.0622558556497097, "expert_frac_std": 0.00015969296509865671, "grad_norm": 0.5280140042304993, "learning_rate": 0.00029702233250620343, "loss": 5.8599, "router_aux_loss": 2.000566345155239, "router_entropy": 2.5548819594830277, "router_entropy_norm": 0.9214788832507104, "step": 400 }, { "epoch": 0.055762081784386616, "expert_0_frac": 0.06241057068109512, "expert_10_frac": 0.06253797560930252, "expert_11_frac": 0.062333617359399796, "expert_12_frac": 0.06225384399294853, "expert_13_frac": 0.062445636838674545, "expert_14_frac": 0.062473755329847336, "expert_15_frac": 0.062299177050590515, "expert_1_frac": 0.062411118298769, "expert_2_frac": 0.06231886148452759, "expert_3_frac": 0.06262814998626709, "expert_4_frac": 0.0625888928771019, "expert_5_frac": 0.06266138702630997, "expert_6_frac": 0.062470629811286926, "expert_7_frac": 0.06291308254003525, "expert_8_frac": 0.06242077797651291, "expert_9_frac": 0.0628325492143631, "expert_frac_max": 0.06291308254003525, "expert_frac_min": 0.06225384399294853, "expert_frac_std": 0.00018639311019796878, "grad_norm": 0.6422482132911682, "learning_rate": 0.00029997335517404445, "loss": 5.571, "router_aux_loss": 2.0004627220332623, "router_entropy": 2.5414530691007773, "router_entropy_norm": 0.9166354348609319, "step": 450 }, { "epoch": 0.061957868649318466, "expert_0_frac": 0.06254314631223679, "expert_10_frac": 0.062319688498973846, "expert_11_frac": 0.06252313405275345, "expert_12_frac": 0.062282487750053406, "expert_13_frac": 0.06232669949531555, "expert_14_frac": 0.06230971962213516, "expert_15_frac": 0.06244329735636711, "expert_1_frac": 0.0624513179063797, "expert_2_frac": 0.062395911663770676, "expert_3_frac": 0.06261033564805984, "expert_4_frac": 0.06252606958150864, "expert_5_frac": 0.06252062320709229, "expert_6_frac": 0.06251340359449387, "expert_7_frac": 0.0628308430314064, "expert_8_frac": 0.06259947270154953, "expert_9_frac": 0.06280387192964554, "expert_frac_max": 0.0628308430314064, "expert_frac_min": 0.062282487750053406, "expert_frac_std": 0.00016125103866215795, "grad_norm": 0.8889395594596863, "learning_rate": 0.00029988396298538336, "loss": 5.3343, "router_aux_loss": 2.000360265225172, "router_entropy": 2.5253191737085583, "router_entropy_norm": 0.9108163621427879, "step": 500 }, { "epoch": 0.06815365551425032, "expert_0_frac": 0.062405213713645935, "expert_10_frac": 0.06261207163333893, "expert_11_frac": 0.06235455349087715, "expert_12_frac": 0.06230912730097771, "expert_13_frac": 0.06251537799835205, "expert_14_frac": 0.062493376433849335, "expert_15_frac": 0.06233787536621094, "expert_1_frac": 0.06240617856383324, "expert_2_frac": 0.06229650601744652, "expert_3_frac": 0.062494538724422455, "expert_4_frac": 0.06250625848770142, "expert_5_frac": 0.06260968744754791, "expert_6_frac": 0.06259311735630035, "expert_7_frac": 0.06282127648591995, "expert_8_frac": 0.0624445416033268, "expert_9_frac": 0.0628003180027008, "expert_frac_max": 0.06282127648591995, "expert_frac_min": 0.06229650601744652, "expert_frac_std": 0.00015772653569001704, "grad_norm": 0.5919621586799622, "learning_rate": 0.00029973165947582927, "loss": 5.1628, "router_aux_loss": 2.0003816688060763, "router_entropy": 2.508501745586594, "router_entropy_norm": 0.9047507571047719, "step": 550 }, { "epoch": 0.07434944237918216, "expert_0_frac": 0.06238816678524017, "expert_10_frac": 0.062436092644929886, "expert_11_frac": 0.06248912960290909, "expert_12_frac": 0.06231667846441269, "expert_13_frac": 0.06243254989385605, "expert_14_frac": 0.06253865361213684, "expert_15_frac": 0.06226829066872597, "expert_1_frac": 0.06243155896663666, "expert_2_frac": 0.06230618804693222, "expert_3_frac": 0.06257857382297516, "expert_4_frac": 0.06255097687244415, "expert_5_frac": 0.06232619285583496, "expert_6_frac": 0.062459323555231094, "expert_7_frac": 0.06313398480415344, "expert_8_frac": 0.06253261864185333, "expert_9_frac": 0.06281103938817978, "expert_frac_max": 0.06313398480415344, "expert_frac_min": 0.06226829066872597, "expert_frac_std": 0.00021489191567525268, "grad_norm": 0.6850535273551941, "learning_rate": 0.0002995165085722677, "loss": 5.0051, "router_aux_loss": 2.0002849200367927, "router_entropy": 2.4902097184211014, "router_entropy_norm": 0.8981533028849062, "step": 600 }, { "epoch": 0.080545229244114, "expert_0_frac": 0.06241001933813095, "expert_10_frac": 0.06275635212659836, "expert_11_frac": 0.06241869181394577, "expert_12_frac": 0.06214861199259758, "expert_13_frac": 0.062489695847034454, "expert_14_frac": 0.06236067786812782, "expert_15_frac": 0.06246155500411987, "expert_1_frac": 0.062489550560712814, "expert_2_frac": 0.06250055134296417, "expert_3_frac": 0.06265998631715775, "expert_4_frac": 0.06249619275331497, "expert_5_frac": 0.06254275888204575, "expert_6_frac": 0.0625753328204155, "expert_7_frac": 0.06273800879716873, "expert_8_frac": 0.06239880248904228, "expert_9_frac": 0.06255324184894562, "expert_frac_max": 0.06275635212659836, "expert_frac_min": 0.06214861199259758, "expert_frac_std": 0.0001478375488659367, "grad_norm": 0.5297462940216064, "learning_rate": 0.0002992386005807413, "loss": 4.8795, "router_aux_loss": 2.0003453426063063, "router_entropy": 2.472862062752247, "router_entropy_norm": 0.8918964586837799, "step": 650 }, { "epoch": 0.08674101610904585, "expert_0_frac": 0.06242035701870918, "expert_10_frac": 0.06262128055095673, "expert_11_frac": 0.06240040436387062, "expert_12_frac": 0.06226507946848869, "expert_13_frac": 0.062457650899887085, "expert_14_frac": 0.06226836517453194, "expert_15_frac": 0.06225299835205078, "expert_1_frac": 0.06253058463335037, "expert_2_frac": 0.062489911913871765, "expert_3_frac": 0.06253976374864578, "expert_4_frac": 0.06257753819227219, "expert_5_frac": 0.062373727560043335, "expert_6_frac": 0.06251810491085052, "expert_7_frac": 0.06306448578834534, "expert_8_frac": 0.062572181224823, "expert_9_frac": 0.06264758855104446, "expert_frac_max": 0.06306448578834534, "expert_frac_min": 0.06225299835205078, "expert_frac_std": 0.00019547491683624685, "grad_norm": 0.5590657591819763, "learning_rate": 0.00029889805214854495, "loss": 4.7772, "router_aux_loss": 2.0003161118924617, "router_entropy": 2.454501044129332, "router_entropy_norm": 0.8852741210555425, "step": 700 }, { "epoch": 0.09293680297397769, "expert_0_frac": 0.062205202877521515, "expert_10_frac": 0.06256791949272156, "expert_11_frac": 0.06266215443611145, "expert_12_frac": 0.062390055507421494, "expert_13_frac": 0.06255996972322464, "expert_14_frac": 0.06251850724220276, "expert_15_frac": 0.06232086196541786, "expert_1_frac": 0.06241616606712341, "expert_2_frac": 0.062427859753370285, "expert_3_frac": 0.06252457201480865, "expert_4_frac": 0.06252304464578629, "expert_5_frac": 0.06245594844222069, "expert_6_frac": 0.06254767626523972, "expert_7_frac": 0.06271296739578247, "expert_8_frac": 0.06264864653348923, "expert_9_frac": 0.06251845508813858, "expert_frac_max": 0.06271296739578247, "expert_frac_min": 0.062205202877521515, "expert_frac_std": 0.00012943513866048306, "grad_norm": 0.49839961528778076, "learning_rate": 0.00029849500621526514, "loss": 4.6889, "router_aux_loss": 2.0003180173039437, "router_entropy": 2.4365353710452715, "router_entropy_norm": 0.8787943741893909, "step": 750 }, { "epoch": 0.09913258983890955, "expert_0_frac": 0.06250069290399551, "expert_10_frac": 0.06257562339305878, "expert_11_frac": 0.06249527633190155, "expert_12_frac": 0.0622064545750618, "expert_13_frac": 0.06257451325654984, "expert_14_frac": 0.06233534589409828, "expert_15_frac": 0.06241418421268463, "expert_1_frac": 0.06264199316501617, "expert_2_frac": 0.06249883770942688, "expert_3_frac": 0.06250212341547012, "expert_4_frac": 0.0625748336315155, "expert_5_frac": 0.062375910580158234, "expert_6_frac": 0.06260207295417786, "expert_7_frac": 0.06282484531402588, "expert_8_frac": 0.06244645640254021, "expert_9_frac": 0.062430866062641144, "expert_frac_max": 0.06282484531402588, "expert_frac_min": 0.0622064545750618, "expert_frac_std": 0.0001406399969710037, "grad_norm": 0.5385776162147522, "learning_rate": 0.0002980296319527835, "loss": 4.6125, "router_aux_loss": 2.000329566299915, "router_entropy": 2.4200733024130265, "router_entropy_norm": 0.8728569379947625, "step": 800 }, { "epoch": 0.1, "eval_loss": 3.9063076972961426, "eval_runtime": 34.3694, "eval_samples_per_second": 50.277, "eval_steps_per_second": 6.285, "expert_0_frac": 0.06256858259439468, "expert_10_frac": 0.06268621981143951, "expert_11_frac": 0.06197136640548706, "expert_12_frac": 0.062151338905096054, "expert_13_frac": 0.06156676262617111, "expert_14_frac": 0.062240924686193466, "expert_15_frac": 0.06298670917749405, "expert_1_frac": 0.0629330724477768, "expert_2_frac": 0.06259055435657501, "expert_3_frac": 0.06298832595348358, "expert_4_frac": 0.062575563788414, "expert_5_frac": 0.06270726025104523, "expert_6_frac": 0.06232661381363869, "expert_7_frac": 0.062258243560791016, "expert_8_frac": 0.06328871101140976, "expert_9_frac": 0.0621596984565258, "expert_frac_max": 0.06328871101140976, "expert_frac_min": 0.06156676262617111, "expert_frac_std": 0.0004415576404426247, "router_aux_loss": 2.000288353824034, "router_entropy": 2.413784189740332, "router_entropy_norm": 0.8705886200786405, "step": 807 }, { "epoch": 0.10532837670384139, "expert_0_frac": 0.06222734600305557, "expert_10_frac": 0.0625881776213646, "expert_11_frac": 0.06228402629494667, "expert_12_frac": 0.06230998784303665, "expert_13_frac": 0.06273674964904785, "expert_14_frac": 0.06251049786806107, "expert_15_frac": 0.062339041382074356, "expert_1_frac": 0.06267312169075012, "expert_2_frac": 0.06258376687765121, "expert_3_frac": 0.06246174871921539, "expert_4_frac": 0.06258857995271683, "expert_5_frac": 0.06243951991200447, "expert_6_frac": 0.06259816139936447, "expert_7_frac": 0.06267885863780975, "expert_8_frac": 0.06258304417133331, "expert_9_frac": 0.062397364526987076, "expert_frac_max": 0.06273674964904785, "expert_frac_min": 0.06222734600305557, "expert_frac_std": 0.00015404434816446155, "grad_norm": 0.5590858459472656, "learning_rate": 0.0002975021246942697, "loss": 4.5451, "router_aux_loss": 2.00037628342939, "router_entropy": 2.4037045948784943, "router_entropy_norm": 0.8669531746983047, "step": 850 }, { "epoch": 0.11152416356877323, "expert_0_frac": 0.06237448379397392, "expert_10_frac": 0.06260498613119125, "expert_11_frac": 0.06246993690729141, "expert_12_frac": 0.06227007135748863, "expert_13_frac": 0.06224007532000542, "expert_14_frac": 0.06248144805431366, "expert_15_frac": 0.06225846707820892, "expert_1_frac": 0.06252281367778778, "expert_2_frac": 0.06251576542854309, "expert_3_frac": 0.06272197514772415, "expert_4_frac": 0.06268062442541122, "expert_5_frac": 0.06230660900473595, "expert_6_frac": 0.06255774199962616, "expert_7_frac": 0.06274865567684174, "expert_8_frac": 0.06259787082672119, "expert_9_frac": 0.06264849752187729, "expert_frac_max": 0.06274865567684174, "expert_frac_min": 0.06224007532000542, "expert_frac_std": 0.00016805788618512452, "grad_norm": 0.511800229549408, "learning_rate": 0.0002969127058521934, "loss": 4.4903, "router_aux_loss": 2.00034979224205, "router_entropy": 2.3888362085570893, "router_entropy_norm": 0.8615905378953266, "step": 900 }, { "epoch": 0.11771995043370508, "expert_0_frac": 0.06221815198659897, "expert_10_frac": 0.06260684877634048, "expert_11_frac": 0.06247435510158539, "expert_12_frac": 0.062245357781648636, "expert_13_frac": 0.06263848394155502, "expert_14_frac": 0.06225128471851349, "expert_15_frac": 0.06243528053164482, "expert_1_frac": 0.06256310641765594, "expert_2_frac": 0.0625167116522789, "expert_3_frac": 0.0626605972647667, "expert_4_frac": 0.06259691715240479, "expert_5_frac": 0.06243862956762314, "expert_6_frac": 0.0625225305557251, "expert_7_frac": 0.0627313181757927, "expert_8_frac": 0.06271545588970184, "expert_9_frac": 0.06238499656319618, "expert_frac_max": 0.0627313181757927, "expert_frac_min": 0.06221815198659897, "expert_frac_std": 0.00016298388072755188, "grad_norm": 0.5196658372879028, "learning_rate": 0.00029626162282539014, "loss": 4.4278, "router_aux_loss": 2.0002928851544857, "router_entropy": 2.3760699312140545, "router_entropy_norm": 0.8569860766419742, "step": 950 }, { "epoch": 0.12391573729863693, "expert_0_frac": 0.062374427914619446, "expert_10_frac": 0.06274916976690292, "expert_11_frac": 0.0625402107834816, "expert_12_frac": 0.06249481812119484, "expert_13_frac": 0.06247967481613159, "expert_14_frac": 0.062247175723314285, "expert_15_frac": 0.0623476505279541, "expert_1_frac": 0.06259554624557495, "expert_2_frac": 0.062494710087776184, "expert_3_frac": 0.062440454959869385, "expert_4_frac": 0.06264761835336685, "expert_5_frac": 0.062286071479320526, "expert_6_frac": 0.06263621896505356, "expert_7_frac": 0.06261817365884781, "expert_8_frac": 0.06239589303731918, "expert_9_frac": 0.06265220791101456, "expert_frac_max": 0.06274916976690292, "expert_frac_min": 0.062247175723314285, "expert_frac_std": 0.0001446436217520386, "grad_norm": 0.5525915026664734, "learning_rate": 0.00029554914889521987, "loss": 4.3662, "router_aux_loss": 2.0003440357744693, "router_entropy": 2.3614482231934866, "router_entropy_norm": 0.8517124102293243, "step": 1000 }, { "epoch": 0.13011152416356878, "expert_0_frac": 0.06243520230054855, "expert_10_frac": 0.062535859644413, "expert_11_frac": 0.06255455315113068, "expert_12_frac": 0.06228669732809067, "expert_13_frac": 0.06247691437602043, "expert_14_frac": 0.0624634250998497, "expert_15_frac": 0.06220926344394684, "expert_1_frac": 0.062498826533555984, "expert_2_frac": 0.06245774030685425, "expert_3_frac": 0.06270050257444382, "expert_4_frac": 0.06269151717424393, "expert_5_frac": 0.06252426654100418, "expert_6_frac": 0.06250670552253723, "expert_7_frac": 0.062534399330616, "expert_8_frac": 0.06266549229621887, "expert_9_frac": 0.06245866417884827, "expert_frac_max": 0.06270050257444382, "expert_frac_min": 0.06220926344394684, "expert_frac_std": 0.00012908343342132866, "grad_norm": 0.5066197514533997, "learning_rate": 0.00029477558311086135, "loss": 4.3252, "router_aux_loss": 2.000340888053179, "router_entropy": 2.3474042297899724, "router_entropy_norm": 0.8466471103199426, "step": 1050 }, { "epoch": 0.13630731102850063, "expert_0_frac": 0.06234085187315941, "expert_10_frac": 0.06262367963790894, "expert_11_frac": 0.062475550919771194, "expert_12_frac": 0.06233123317360878, "expert_13_frac": 0.06273722648620605, "expert_14_frac": 0.062331944704055786, "expert_15_frac": 0.06239987164735794, "expert_1_frac": 0.0627233237028122, "expert_2_frac": 0.06237931549549103, "expert_3_frac": 0.06261306256055832, "expert_4_frac": 0.06256913393735886, "expert_5_frac": 0.06224743649363518, "expert_6_frac": 0.0626690536737442, "expert_7_frac": 0.06252788007259369, "expert_8_frac": 0.0625937357544899, "expert_9_frac": 0.06243671476840973, "expert_frac_max": 0.06273722648620605, "expert_frac_min": 0.06224743649363518, "expert_frac_std": 0.0001532289170427248, "grad_norm": 0.6175411939620972, "learning_rate": 0.0002939412501637914, "loss": 4.2875, "router_aux_loss": 2.000367415100336, "router_entropy": 2.33591702952981, "router_entropy_norm": 0.8425039786076838, "step": 1100 }, { "epoch": 0.14250309789343246, "expert_0_frac": 0.06229696422815323, "expert_10_frac": 0.06262972950935364, "expert_11_frac": 0.062427107244729996, "expert_12_frac": 0.06219984218478203, "expert_13_frac": 0.06254806369543076, "expert_14_frac": 0.06229260936379433, "expert_15_frac": 0.062417853623628616, "expert_1_frac": 0.06263617426156998, "expert_2_frac": 0.06263332813978195, "expert_3_frac": 0.06255980581045151, "expert_4_frac": 0.06264110654592514, "expert_5_frac": 0.06253870576620102, "expert_6_frac": 0.06249291077256203, "expert_7_frac": 0.06253132969141006, "expert_8_frac": 0.06262201815843582, "expert_9_frac": 0.06253247708082199, "expert_frac_max": 0.06264110654592514, "expert_frac_min": 0.06219984218478203, "expert_frac_std": 0.00013755267718806863, "grad_norm": 0.5854677557945251, "learning_rate": 0.00029304650025150096, "loss": 4.2579, "router_aux_loss": 2.00031850785017, "router_entropy": 2.323672557796041, "router_entropy_norm": 0.8380877189455305, "step": 1150 }, { "epoch": 0.14869888475836432, "expert_0_frac": 0.06235099211335182, "expert_10_frac": 0.06260727345943451, "expert_11_frac": 0.06260436028242111, "expert_12_frac": 0.06245154142379761, "expert_13_frac": 0.06255651265382767, "expert_14_frac": 0.062305666506290436, "expert_15_frac": 0.06230290234088898, "expert_1_frac": 0.06253120303153992, "expert_2_frac": 0.06256424635648727, "expert_3_frac": 0.06265949457883835, "expert_4_frac": 0.0625964030623436, "expert_5_frac": 0.0623132660984993, "expert_6_frac": 0.06253556907176971, "expert_7_frac": 0.06253494322299957, "expert_8_frac": 0.06250181794166565, "expert_9_frac": 0.06258382648229599, "expert_frac_max": 0.06265949457883835, "expert_frac_min": 0.06230290234088898, "expert_frac_std": 0.00011859627556987107, "grad_norm": 0.46354418992996216, "learning_rate": 0.00029209170893050536, "loss": 4.2219, "router_aux_loss": 2.0003797383606434, "router_entropy": 2.3109171950817107, "router_entropy_norm": 0.8334871943123543, "step": 1200 }, { "epoch": 0.15489467162329615, "expert_0_frac": 0.062493301928043365, "expert_10_frac": 0.06252740323543549, "expert_11_frac": 0.06238264963030815, "expert_12_frac": 0.062336668372154236, "expert_13_frac": 0.062471043318510056, "expert_14_frac": 0.062253303825855255, "expert_15_frac": 0.06223702430725098, "expert_1_frac": 0.06259184330701828, "expert_2_frac": 0.06251081079244614, "expert_3_frac": 0.0626404732465744, "expert_4_frac": 0.06271036714315414, "expert_5_frac": 0.06241315230727196, "expert_6_frac": 0.06260694563388824, "expert_7_frac": 0.06258156895637512, "expert_8_frac": 0.06270106136798859, "expert_9_frac": 0.06254240870475769, "expert_frac_max": 0.06271036714315414, "expert_frac_min": 0.06223702430725098, "expert_frac_std": 0.00014433629985433072, "grad_norm": 0.44044944643974304, "learning_rate": 0.00029107727695871084, "loss": 4.1886, "router_aux_loss": 2.0003322799503804, "router_entropy": 2.300990598698457, "router_entropy_norm": 0.8299069314685977, "step": 1250 }, { "epoch": 0.161090458488228, "expert_0_frac": 0.062284037470817566, "expert_10_frac": 0.06267246603965759, "expert_11_frac": 0.06254246830940247, "expert_12_frac": 0.062176335602998734, "expert_13_frac": 0.06258271634578705, "expert_14_frac": 0.06226383522152901, "expert_15_frac": 0.062295541167259216, "expert_1_frac": 0.06266821175813675, "expert_2_frac": 0.06251491606235504, "expert_3_frac": 0.06266596168279648, "expert_4_frac": 0.06262225657701492, "expert_5_frac": 0.062298376113176346, "expert_6_frac": 0.06268490105867386, "expert_7_frac": 0.0625143051147461, "expert_8_frac": 0.06259044259786606, "expert_9_frac": 0.06262325495481491, "expert_frac_max": 0.06268490105867386, "expert_frac_min": 0.062176335602998734, "expert_frac_std": 0.00017450697487220168, "grad_norm": 0.4276027977466583, "learning_rate": 0.00029000363012720296, "loss": 4.1441, "router_aux_loss": 2.0003066021203995, "router_entropy": 2.289357506732146, "router_entropy_norm": 0.8257111804460971, "step": 1300 }, { "epoch": 0.16728624535315986, "expert_0_frac": 0.06231163814663887, "expert_10_frac": 0.06265905499458313, "expert_11_frac": 0.06236410140991211, "expert_12_frac": 0.062274329364299774, "expert_13_frac": 0.06251786649227142, "expert_14_frac": 0.06248300522565842, "expert_15_frac": 0.06248318403959274, "expert_1_frac": 0.06256091594696045, "expert_2_frac": 0.06251896172761917, "expert_3_frac": 0.06250979751348495, "expert_4_frac": 0.06253128498792648, "expert_5_frac": 0.06237177178263664, "expert_6_frac": 0.06275496631860733, "expert_7_frac": 0.06262163072824478, "expert_8_frac": 0.06238657608628273, "expert_9_frac": 0.06265093386173248, "expert_frac_max": 0.06275496631860733, "expert_frac_min": 0.062274329364299774, "expert_frac_std": 0.00013369132648222148, "grad_norm": 0.5024176836013794, "learning_rate": 0.0002888712190815277, "loss": 4.1224, "router_aux_loss": 2.000337022989988, "router_entropy": 2.2781822363535564, "router_entropy_norm": 0.8216805536571511, "step": 1350 }, { "epoch": 0.1734820322180917, "expert_0_frac": 0.06237514689564705, "expert_10_frac": 0.062498342245817184, "expert_11_frac": 0.06249630078673363, "expert_12_frac": 0.06231970712542534, "expert_13_frac": 0.06248846277594566, "expert_14_frac": 0.062270451337099075, "expert_15_frac": 0.06256610155105591, "expert_1_frac": 0.06258639693260193, "expert_2_frac": 0.0625196322798729, "expert_3_frac": 0.06262469291687012, "expert_4_frac": 0.06277842074632645, "expert_5_frac": 0.06232196465134621, "expert_6_frac": 0.06267911195755005, "expert_7_frac": 0.06257634609937668, "expert_8_frac": 0.06246921792626381, "expert_9_frac": 0.062429722398519516, "expert_frac_max": 0.06277842074632645, "expert_frac_min": 0.062270451337099075, "expert_frac_std": 0.0001370436802972108, "grad_norm": 0.42523810267448425, "learning_rate": 0.0002876805191325406, "loss": 4.087, "router_aux_loss": 2.0003104132413863, "router_entropy": 2.2699872746318577, "router_entropy_norm": 0.8187248459981087, "step": 1400 }, { "epoch": 0.17967781908302355, "expert_0_frac": 0.062234066426754, "expert_10_frac": 0.06256188452243805, "expert_11_frac": 0.06254622340202332, "expert_12_frac": 0.06240968033671379, "expert_13_frac": 0.06278626620769501, "expert_14_frac": 0.06218487024307251, "expert_15_frac": 0.06230517849326134, "expert_1_frac": 0.06253640353679657, "expert_2_frac": 0.0625469759106636, "expert_3_frac": 0.06253714859485626, "expert_4_frac": 0.06265972554683685, "expert_5_frac": 0.06244716793298721, "expert_6_frac": 0.06260564923286438, "expert_7_frac": 0.06254668533802032, "expert_8_frac": 0.06246053799986839, "expert_9_frac": 0.06263155490159988, "expert_frac_max": 0.06278626620769501, "expert_frac_min": 0.06218487024307251, "expert_frac_std": 0.00015694495232310146, "grad_norm": 0.39899736642837524, "learning_rate": 0.00028643203005690286, "loss": 4.0667, "router_aux_loss": 2.000341858714819, "router_entropy": 2.26084908430775, "router_entropy_norm": 0.8154289405322862, "step": 1450 }, { "epoch": 0.18587360594795538, "expert_0_frac": 0.062320563942193985, "expert_10_frac": 0.06271921843290329, "expert_11_frac": 0.06238391250371933, "expert_12_frac": 0.06240151822566986, "expert_13_frac": 0.06269849836826324, "expert_14_frac": 0.062356676906347275, "expert_15_frac": 0.06238163262605667, "expert_1_frac": 0.0625913217663765, "expert_2_frac": 0.06255612522363663, "expert_3_frac": 0.0625518411397934, "expert_4_frac": 0.06254532933235168, "expert_5_frac": 0.06224118173122406, "expert_6_frac": 0.06265821307897568, "expert_7_frac": 0.06254154443740845, "expert_8_frac": 0.06240622326731682, "expert_9_frac": 0.06264621019363403, "expert_frac_max": 0.06271921843290329, "expert_frac_min": 0.06224118173122406, "expert_frac_std": 0.00014549469051416963, "grad_norm": 0.41512852907180786, "learning_rate": 0.00028512627588730814, "loss": 4.0337, "router_aux_loss": 2.0003461696207525, "router_entropy": 2.2523704461505014, "router_entropy_norm": 0.8123709182265476, "step": 1500 }, { "epoch": 0.19206939281288724, "expert_0_frac": 0.062392860651016235, "expert_10_frac": 0.062398917973041534, "expert_11_frac": 0.06255253404378891, "expert_12_frac": 0.06235002726316452, "expert_13_frac": 0.06256391853094101, "expert_14_frac": 0.06229909881949425, "expert_15_frac": 0.062403980642557144, "expert_1_frac": 0.0625167191028595, "expert_2_frac": 0.06252924352884293, "expert_3_frac": 0.06263791769742966, "expert_4_frac": 0.06260311603546143, "expert_5_frac": 0.06242026016116142, "expert_6_frac": 0.0625079870223999, "expert_7_frac": 0.06265029311180115, "expert_8_frac": 0.06259024143218994, "expert_9_frac": 0.06258291751146317, "expert_frac_max": 0.06265029311180115, "expert_frac_min": 0.06229909881949425, "expert_frac_std": 0.00010814578126883134, "grad_norm": 0.5054246783256531, "learning_rate": 0.0002837638046925287, "loss": 4.0077, "router_aux_loss": 2.0003621953725816, "router_entropy": 2.2446445197115343, "router_entropy_norm": 0.809584379286605, "step": 1550 }, { "epoch": 0.1982651796778191, "expert_0_frac": 0.06226435676217079, "expert_10_frac": 0.06269991397857666, "expert_11_frac": 0.0623975433409214, "expert_12_frac": 0.06223442777991295, "expert_13_frac": 0.0626256987452507, "expert_14_frac": 0.06223084405064583, "expert_15_frac": 0.06239727884531021, "expert_1_frac": 0.06264633685350418, "expert_2_frac": 0.06251941621303558, "expert_3_frac": 0.06264705210924149, "expert_4_frac": 0.06256012618541718, "expert_5_frac": 0.062397200614213943, "expert_6_frac": 0.06272200495004654, "expert_7_frac": 0.06254942715167999, "expert_8_frac": 0.06245889142155647, "expert_9_frac": 0.06264950335025787, "expert_frac_max": 0.06272200495004654, "expert_frac_min": 0.06223084405064583, "expert_frac_std": 0.0001650626363698393, "grad_norm": 0.5019674301147461, "learning_rate": 0.0002823451883473724, "loss": 3.9899, "router_aux_loss": 2.000300102084875, "router_entropy": 2.23768438488245, "router_entropy_norm": 0.8070740412861453, "step": 1600 }, { "epoch": 0.2, "eval_loss": 3.348736524581909, "eval_runtime": 34.4759, "eval_samples_per_second": 50.122, "eval_steps_per_second": 6.265, "expert_0_frac": 0.062293633818626404, "expert_10_frac": 0.06241512671113014, "expert_11_frac": 0.06290776282548904, "expert_12_frac": 0.06183137372136116, "expert_13_frac": 0.06233253329992294, "expert_14_frac": 0.06204337999224663, "expert_15_frac": 0.06220891326665878, "expert_1_frac": 0.06302127987146378, "expert_2_frac": 0.062322936952114105, "expert_3_frac": 0.062286339700222015, "expert_4_frac": 0.06292441487312317, "expert_5_frac": 0.06253024935722351, "expert_6_frac": 0.06261050701141357, "expert_7_frac": 0.06317538768053055, "expert_8_frac": 0.06250172108411789, "expert_9_frac": 0.06259440630674362, "expert_frac_max": 0.06317538768053055, "expert_frac_min": 0.06183137372136116, "expert_frac_std": 0.0003643674135673791, "router_aux_loss": 2.000175356864929, "router_entropy": 2.2309101776191684, "router_entropy_norm": 0.8046307624799728, "step": 1614 }, { "epoch": 0.20446096654275092, "expert_0_frac": 0.06241779029369354, "expert_10_frac": 0.06254168599843979, "expert_11_frac": 0.06247587502002716, "expert_12_frac": 0.062394991517066956, "expert_13_frac": 0.06247551366686821, "expert_14_frac": 0.06237455829977989, "expert_15_frac": 0.06222521886229515, "expert_1_frac": 0.06245426833629608, "expert_2_frac": 0.06267520040273666, "expert_3_frac": 0.06251148879528046, "expert_4_frac": 0.06266897916793823, "expert_5_frac": 0.06242360174655914, "expert_6_frac": 0.06273163110017776, "expert_7_frac": 0.06260820478200912, "expert_8_frac": 0.06258939206600189, "expert_9_frac": 0.062431611120700836, "expert_frac_max": 0.06273163110017776, "expert_frac_min": 0.06222521886229515, "expert_frac_std": 0.0001309664803557098, "grad_norm": 0.44987165927886963, "learning_rate": 0.00028087102229264795, "loss": 3.9507, "router_aux_loss": 2.0002999247776136, "router_entropy": 2.228416964687683, "router_entropy_norm": 0.8037315259969391, "step": 1650 }, { "epoch": 0.21065675340768278, "expert_0_frac": 0.062353696674108505, "expert_10_frac": 0.0626240074634552, "expert_11_frac": 0.06262946128845215, "expert_12_frac": 0.062432609498500824, "expert_13_frac": 0.06261216104030609, "expert_14_frac": 0.06230517104268074, "expert_15_frac": 0.06234478950500488, "expert_1_frac": 0.06255390495061874, "expert_2_frac": 0.062430791556835175, "expert_3_frac": 0.06261441111564636, "expert_4_frac": 0.06257495284080505, "expert_5_frac": 0.06230694428086281, "expert_6_frac": 0.06264862418174744, "expert_7_frac": 0.0625498816370964, "expert_8_frac": 0.0624498575925827, "expert_9_frac": 0.0625687763094902, "expert_frac_max": 0.06264862418174744, "expert_frac_min": 0.06230517104268074, "expert_frac_std": 0.00012334466737229377, "grad_norm": 0.4263043701648712, "learning_rate": 0.00027934192528523803, "loss": 3.9304, "router_aux_loss": 2.0003829108178617, "router_entropy": 2.2245351150631905, "router_entropy_norm": 0.8023314446962562, "step": 1700 }, { "epoch": 0.21685254027261464, "expert_0_frac": 0.06229822337627411, "expert_10_frac": 0.06234228238463402, "expert_11_frac": 0.062404099851846695, "expert_12_frac": 0.06231901794672012, "expert_13_frac": 0.06277892738580704, "expert_14_frac": 0.06221300736069679, "expert_15_frac": 0.062457721680402756, "expert_1_frac": 0.06249331310391426, "expert_2_frac": 0.062426842749118805, "expert_3_frac": 0.06263561546802521, "expert_4_frac": 0.0626637190580368, "expert_5_frac": 0.062459539622068405, "expert_6_frac": 0.0625869482755661, "expert_7_frac": 0.06273391097784042, "expert_8_frac": 0.06258323788642883, "expert_9_frac": 0.06260362267494202, "expert_frac_max": 0.06277892738580704, "expert_frac_min": 0.06221300736069679, "expert_frac_std": 0.00016363832401111722, "grad_norm": 0.44618475437164307, "learning_rate": 0.0002777585391383867, "loss": 3.9089, "router_aux_loss": 2.0003176471590995, "router_entropy": 2.216816434636712, "router_entropy_norm": 0.7995475192028844, "step": 1750 }, { "epoch": 0.22304832713754646, "expert_0_frac": 0.06226443871855736, "expert_10_frac": 0.06262083351612091, "expert_11_frac": 0.06259326636791229, "expert_12_frac": 0.0622016042470932, "expert_13_frac": 0.06253740191459656, "expert_14_frac": 0.06229192763566971, "expert_15_frac": 0.06249629333615303, "expert_1_frac": 0.06274900585412979, "expert_2_frac": 0.06252726167440414, "expert_3_frac": 0.0625016838312149, "expert_4_frac": 0.06247502192854881, "expert_5_frac": 0.0623481310904026, "expert_6_frac": 0.06267069280147552, "expert_7_frac": 0.06268630176782608, "expert_8_frac": 0.06241276115179062, "expert_9_frac": 0.06262339651584625, "expert_frac_max": 0.06274900585412979, "expert_frac_min": 0.0622016042470932, "expert_frac_std": 0.000160344410687685, "grad_norm": 0.38775309920310974, "learning_rate": 0.00027612152845230865, "loss": 3.8949, "router_aux_loss": 2.000342970043421, "router_entropy": 2.211577178960045, "router_entropy_norm": 0.7976578571572152, "step": 1800 }, { "epoch": 0.22924411400247832, "expert_0_frac": 0.06244306638836861, "expert_10_frac": 0.0625232681632042, "expert_11_frac": 0.06246243789792061, "expert_12_frac": 0.06222465634346008, "expert_13_frac": 0.06267709285020828, "expert_14_frac": 0.06221655011177063, "expert_15_frac": 0.06233378127217293, "expert_1_frac": 0.0626348927617073, "expert_2_frac": 0.0624520517885685, "expert_3_frac": 0.06252284348011017, "expert_4_frac": 0.06256033480167389, "expert_5_frac": 0.06249850243330002, "expert_6_frac": 0.06259974837303162, "expert_7_frac": 0.06249947473406792, "expert_8_frac": 0.06257827579975128, "expert_9_frac": 0.06277304142713547, "expert_frac_max": 0.06277304142713547, "expert_frac_min": 0.06221655011177063, "expert_frac_std": 0.0001495839678682387, "grad_norm": 0.4576256275177002, "learning_rate": 0.00027443158033523435, "loss": 3.8763, "router_aux_loss": 2.0003334523737433, "router_entropy": 2.206269159093499, "router_entropy_norm": 0.7957433936726137, "step": 1850 }, { "epoch": 0.23543990086741015, "expert_0_frac": 0.06226146221160889, "expert_10_frac": 0.062378235161304474, "expert_11_frac": 0.06265708804130554, "expert_12_frac": 0.06239369139075279, "expert_13_frac": 0.06258013844490051, "expert_14_frac": 0.062292423099279404, "expert_15_frac": 0.0625077486038208, "expert_1_frac": 0.06268229335546494, "expert_2_frac": 0.06261877715587616, "expert_3_frac": 0.06257469952106476, "expert_4_frac": 0.06252875179052353, "expert_5_frac": 0.062346141785383224, "expert_6_frac": 0.06263620406389236, "expert_7_frac": 0.06256420910358429, "expert_8_frac": 0.06239767745137215, "expert_9_frac": 0.06258046627044678, "expert_frac_max": 0.06268229335546494, "expert_frac_min": 0.06226146221160889, "expert_frac_std": 0.0001352312829112634, "grad_norm": 0.44768062233924866, "learning_rate": 0.00027268940411500763, "loss": 3.864, "router_aux_loss": 2.0003784953057764, "router_entropy": 2.1981928451855977, "router_entropy_norm": 0.7928304791667157, "step": 1900 }, { "epoch": 0.241635687732342, "expert_0_frac": 0.062274038791656494, "expert_10_frac": 0.06261645257472992, "expert_11_frac": 0.0625358372926712, "expert_12_frac": 0.06225775554776192, "expert_13_frac": 0.0624920092523098, "expert_14_frac": 0.0622897669672966, "expert_15_frac": 0.06222781166434288, "expert_1_frac": 0.06258921325206757, "expert_2_frac": 0.062490224838256836, "expert_3_frac": 0.06263462454080582, "expert_4_frac": 0.06267813593149185, "expert_5_frac": 0.06240127235651016, "expert_6_frac": 0.06273884326219559, "expert_7_frac": 0.06260263919830322, "expert_8_frac": 0.06257489323616028, "expert_9_frac": 0.06259652227163315, "expert_frac_max": 0.06273884326219559, "expert_frac_min": 0.06222781166434288, "expert_frac_std": 0.00016188451263587922, "grad_norm": 0.44072648882865906, "learning_rate": 0.00027089573104135695, "loss": 3.8464, "router_aux_loss": 2.0002842231094835, "router_entropy": 2.1940508898099265, "router_entropy_norm": 0.7913365845466996, "step": 1950 }, { "epoch": 0.24783147459727387, "expert_0_frac": 0.06231727823615074, "expert_10_frac": 0.06263195723295212, "expert_11_frac": 0.06249087676405907, "expert_12_frac": 0.06225622817873955, "expert_13_frac": 0.06269091367721558, "expert_14_frac": 0.06215570494532585, "expert_15_frac": 0.062470827251672745, "expert_1_frac": 0.0627078041434288, "expert_2_frac": 0.06257681548595428, "expert_3_frac": 0.06265929341316223, "expert_4_frac": 0.06238482892513275, "expert_5_frac": 0.06231342628598213, "expert_6_frac": 0.06271243840456009, "expert_7_frac": 0.06250765919685364, "expert_8_frac": 0.06244330108165741, "expert_9_frac": 0.0626806691288948, "expert_frac_max": 0.06271243840456009, "expert_frac_min": 0.06215570494532585, "expert_frac_std": 0.00017695999122224748, "grad_norm": 0.45407283306121826, "learning_rate": 0.00026905131397896557, "loss": 3.8192, "router_aux_loss": 2.0003489625453947, "router_entropy": 2.189426625718673, "router_entropy_norm": 0.7896687338286466, "step": 2000 }, { "epoch": 0.2540272614622057, "expert_0_frac": 0.062279440462589264, "expert_10_frac": 0.06258825212717056, "expert_11_frac": 0.0625229924917221, "expert_12_frac": 0.06232941523194313, "expert_13_frac": 0.062425460666418076, "expert_14_frac": 0.06236887723207474, "expert_15_frac": 0.06254032999277115, "expert_1_frac": 0.0624471940100193, "expert_2_frac": 0.06263547390699387, "expert_3_frac": 0.06265461444854736, "expert_4_frac": 0.06267198920249939, "expert_5_frac": 0.062366899102926254, "expert_6_frac": 0.062466420233249664, "expert_7_frac": 0.06264328211545944, "expert_8_frac": 0.06244220957159996, "expert_9_frac": 0.06261716783046722, "expert_frac_max": 0.06267198920249939, "expert_frac_min": 0.062279440462589264, "expert_frac_std": 0.00012688770948443562, "grad_norm": 0.4619823694229126, "learning_rate": 0.0002671569270914689, "loss": 3.8077, "router_aux_loss": 2.000316036939621, "router_entropy": 2.1830097007751466, "router_entropy_norm": 0.787354317380201, "step": 2050 }, { "epoch": 0.26022304832713755, "expert_0_frac": 0.06231517344713211, "expert_10_frac": 0.06260181963443756, "expert_11_frac": 0.0624566413462162, "expert_12_frac": 0.06214236095547676, "expert_13_frac": 0.0626717060804367, "expert_14_frac": 0.062233492732048035, "expert_15_frac": 0.06237335875630379, "expert_1_frac": 0.0626189112663269, "expert_2_frac": 0.06243986263871193, "expert_3_frac": 0.0627051368355751, "expert_4_frac": 0.06272777915000916, "expert_5_frac": 0.06241276115179062, "expert_6_frac": 0.062677301466465, "expert_7_frac": 0.06261099129915237, "expert_8_frac": 0.06242264434695244, "expert_9_frac": 0.06259007751941681, "expert_frac_max": 0.06272777915000916, "expert_frac_min": 0.06214236095547676, "expert_frac_std": 0.00017664371989667416, "grad_norm": 0.5377533435821533, "learning_rate": 0.000265213365516512, "loss": 3.7954, "router_aux_loss": 2.0003011628985403, "router_entropy": 2.1784792892634868, "router_entropy_norm": 0.7857203168249365, "step": 2100 }, { "epoch": 0.2664188351920694, "expert_0_frac": 0.06247631832957268, "expert_10_frac": 0.06265552341938019, "expert_11_frac": 0.062469732016325, "expert_12_frac": 0.062348317354917526, "expert_13_frac": 0.062486276030540466, "expert_14_frac": 0.062405217438936234, "expert_15_frac": 0.062300167977809906, "expert_1_frac": 0.06241647154092789, "expert_2_frac": 0.0623706579208374, "expert_3_frac": 0.06280349940061569, "expert_4_frac": 0.06258194893598557, "expert_5_frac": 0.062391288578510284, "expert_6_frac": 0.06257753819227219, "expert_7_frac": 0.06254401057958603, "expert_8_frac": 0.06254838407039642, "expert_9_frac": 0.06262466311454773, "expert_frac_max": 0.06280349940061569, "expert_frac_min": 0.062300167977809906, "expert_frac_std": 0.00013113862951286137, "grad_norm": 0.4972023367881775, "learning_rate": 0.00026322144503200404, "loss": 3.7692, "router_aux_loss": 2.000367202758789, "router_entropy": 2.1743140166749555, "router_entropy_norm": 0.7842180122980803, "step": 2150 }, { "epoch": 0.27261462205700127, "expert_0_frac": 0.06236778944730759, "expert_10_frac": 0.06253974139690399, "expert_11_frac": 0.06243103742599487, "expert_12_frac": 0.06224328652024269, "expert_13_frac": 0.06255636364221573, "expert_14_frac": 0.062425754964351654, "expert_15_frac": 0.062366798520088196, "expert_1_frac": 0.06278834491968155, "expert_2_frac": 0.06251846253871918, "expert_3_frac": 0.06271340698003769, "expert_4_frac": 0.06249046325683594, "expert_5_frac": 0.062309712171554565, "expert_6_frac": 0.06265412271022797, "expert_7_frac": 0.06254371255636215, "expert_8_frac": 0.06236147880554199, "expert_9_frac": 0.06268955022096634, "expert_frac_max": 0.06278834491968155, "expert_frac_min": 0.06224328652024269, "expert_frac_std": 0.00015540422464255244, "grad_norm": 0.4365726709365845, "learning_rate": 0.00026118200171370843, "loss": 3.7575, "router_aux_loss": 2.000355373620987, "router_entropy": 2.167735231295228, "router_entropy_norm": 0.7818452170374789, "step": 2200 }, { "epoch": 0.2788104089219331, "expert_0_frac": 0.062455914914608, "expert_10_frac": 0.06251836568117142, "expert_11_frac": 0.06251351535320282, "expert_12_frac": 0.062291666865348816, "expert_13_frac": 0.06252916157245636, "expert_14_frac": 0.06226165220141411, "expert_15_frac": 0.06260184198617935, "expert_1_frac": 0.06268971413373947, "expert_2_frac": 0.06259042769670486, "expert_3_frac": 0.06244368478655815, "expert_4_frac": 0.062420859932899475, "expert_5_frac": 0.062372010201215744, "expert_6_frac": 0.0626329779624939, "expert_7_frac": 0.0627329871058464, "expert_8_frac": 0.062414951622486115, "expert_9_frac": 0.06253029406070709, "expert_frac_max": 0.0627329871058464, "expert_frac_min": 0.06226165220141411, "expert_frac_std": 0.00013266490714158863, "grad_norm": 0.5881329774856567, "learning_rate": 0.0002590958915843143, "loss": 3.7527, "router_aux_loss": 2.000336780399084, "router_entropy": 2.1648418973137935, "router_entropy_norm": 0.7808016673908161, "step": 2250 }, { "epoch": 0.2850061957868649, "expert_0_frac": 0.062241166830062866, "expert_10_frac": 0.06262774020433426, "expert_11_frac": 0.06260697543621063, "expert_12_frac": 0.06218230724334717, "expert_13_frac": 0.06255532056093216, "expert_14_frac": 0.06231023371219635, "expert_15_frac": 0.06251320242881775, "expert_1_frac": 0.06266742944717407, "expert_2_frac": 0.06255898624658585, "expert_3_frac": 0.0625995546579361, "expert_4_frac": 0.06255882233381271, "expert_5_frac": 0.06240781396627426, "expert_6_frac": 0.06258196383714676, "expert_7_frac": 0.06254728883504868, "expert_8_frac": 0.062440671026706696, "expert_9_frac": 0.06260056048631668, "expert_frac_max": 0.06266742944717407, "expert_frac_min": 0.06218230724334717, "expert_frac_std": 0.00014396726328413934, "grad_norm": 0.4369165897369385, "learning_rate": 0.0002569639902541347, "loss": 3.7403, "router_aux_loss": 2.000332098007202, "router_entropy": 2.160193564010163, "router_entropy_norm": 0.7791251355394294, "step": 2300 }, { "epoch": 0.29120198265179675, "expert_0_frac": 0.06230125576257706, "expert_10_frac": 0.06242264062166214, "expert_11_frac": 0.0626748725771904, "expert_12_frac": 0.06223186478018761, "expert_13_frac": 0.06251568347215652, "expert_14_frac": 0.06230219826102257, "expert_15_frac": 0.06251835078001022, "expert_1_frac": 0.06252612918615341, "expert_2_frac": 0.06243795529007912, "expert_3_frac": 0.06267717480659485, "expert_4_frac": 0.06260242313146591, "expert_5_frac": 0.06240590661764145, "expert_6_frac": 0.06257519125938416, "expert_7_frac": 0.06247491389513016, "expert_8_frac": 0.06258118152618408, "expert_9_frac": 0.06275228410959244, "expert_frac_max": 0.06275228410959244, "expert_frac_min": 0.06223186478018761, "expert_frac_std": 0.00014593535161111504, "grad_norm": 0.4153628349304199, "learning_rate": 0.00025478719255358405, "loss": 3.7196, "router_aux_loss": 2.00037807777524, "router_entropy": 2.156891989496847, "router_entropy_norm": 0.7779343442450578, "step": 2350 }, { "epoch": 0.29739776951672864, "expert_0_frac": 0.06231791898608208, "expert_10_frac": 0.06267321854829788, "expert_11_frac": 0.06255703419446945, "expert_12_frac": 0.062291208654642105, "expert_13_frac": 0.06265942007303238, "expert_14_frac": 0.06218227371573448, "expert_15_frac": 0.06219065561890602, "expert_1_frac": 0.06256262212991714, "expert_2_frac": 0.06257721036672592, "expert_3_frac": 0.0626627653837204, "expert_4_frac": 0.062482550740242004, "expert_5_frac": 0.0623604841530323, "expert_6_frac": 0.0626610815525055, "expert_7_frac": 0.06265909224748611, "expert_8_frac": 0.06247072294354439, "expert_9_frac": 0.06269176304340363, "expert_frac_max": 0.06269176304340363, "expert_frac_min": 0.06218227371573448, "expert_frac_std": 0.00017834350001066923, "grad_norm": 0.41617467999458313, "learning_rate": 0.00025256641215758743, "loss": 3.7003, "router_aux_loss": 2.0003033436834814, "router_entropy": 2.1526932566737136, "router_entropy_norm": 0.7764199714895698, "step": 2400 }, { "epoch": 0.3, "eval_loss": 3.081298589706421, "eval_runtime": 33.3012, "eval_samples_per_second": 51.89, "eval_steps_per_second": 6.486, "expert_0_frac": 0.06206047907471657, "expert_10_frac": 0.06263051182031631, "expert_11_frac": 0.06254921853542328, "expert_12_frac": 0.06210346519947052, "expert_13_frac": 0.06262776255607605, "expert_14_frac": 0.06231609359383583, "expert_15_frac": 0.06274214386940002, "expert_1_frac": 0.062285520136356354, "expert_2_frac": 0.06221441179513931, "expert_3_frac": 0.06259218603372574, "expert_4_frac": 0.06259699165821075, "expert_5_frac": 0.06273031234741211, "expert_6_frac": 0.06256095319986343, "expert_7_frac": 0.06282997876405716, "expert_8_frac": 0.06255850940942764, "expert_9_frac": 0.06260145455598831, "expert_frac_max": 0.06282997876405716, "expert_frac_min": 0.06206047907471657, "expert_frac_std": 0.00023144215811043978, "router_aux_loss": 2.000319122620251, "router_entropy": 2.150703725823458, "router_entropy_norm": 0.7757023999167298, "step": 2421 }, { "epoch": 0.30359355638166047, "expert_0_frac": 0.06226431205868721, "expert_10_frac": 0.06251043826341629, "expert_11_frac": 0.062482383102178574, "expert_12_frac": 0.06242340803146362, "expert_13_frac": 0.06245507672429085, "expert_14_frac": 0.062295958399772644, "expert_15_frac": 0.062416043132543564, "expert_1_frac": 0.06262195110321045, "expert_2_frac": 0.06257570534944534, "expert_3_frac": 0.0626882016658783, "expert_4_frac": 0.06279187649488449, "expert_5_frac": 0.06238429993391037, "expert_6_frac": 0.06260958313941956, "expert_7_frac": 0.0625062808394432, "expert_8_frac": 0.0622023344039917, "expert_9_frac": 0.06277214735746384, "expert_frac_max": 0.06279187649488449, "expert_frac_min": 0.0622023344039917, "expert_frac_std": 0.0001716299884719774, "grad_norm": 0.41244012117385864, "learning_rate": 0.00025030258120208086, "loss": 3.6924, "router_aux_loss": 2.0003207174354585, "router_entropy": 2.1494769665316262, "router_entropy_norm": 0.7752599400300574, "step": 2450 }, { "epoch": 0.3097893432465923, "expert_0_frac": 0.062158700078725815, "expert_10_frac": 0.06262535601854324, "expert_11_frac": 0.06255626678466797, "expert_12_frac": 0.0621267706155777, "expert_13_frac": 0.06274285167455673, "expert_14_frac": 0.062286049127578735, "expert_15_frac": 0.06258541345596313, "expert_1_frac": 0.06256241351366043, "expert_2_frac": 0.0624278262257576, "expert_3_frac": 0.062509685754776, "expert_4_frac": 0.0625128522515297, "expert_5_frac": 0.06246746703982353, "expert_6_frac": 0.0625898465514183, "expert_7_frac": 0.06269817799329758, "expert_8_frac": 0.06244150176644325, "expert_9_frac": 0.06270884722471237, "expert_frac_max": 0.06274285167455673, "expert_frac_min": 0.0621267706155777, "expert_frac_std": 0.00018101956811733544, "grad_norm": 0.4232514798641205, "learning_rate": 0.0002479966498927624, "loss": 3.6834, "router_aux_loss": 2.0003105348348615, "router_entropy": 2.1456227505579593, "router_entropy_norm": 0.7738698254621263, "step": 2500 }, { "epoch": 0.3159851301115242, "expert_0_frac": 0.06242232769727707, "expert_10_frac": 0.06251971423625946, "expert_11_frac": 0.06261476129293442, "expert_12_frac": 0.062365610152482986, "expert_13_frac": 0.06250380724668503, "expert_14_frac": 0.06229201331734657, "expert_15_frac": 0.06255565583705902, "expert_1_frac": 0.06260047852993011, "expert_2_frac": 0.06252625584602356, "expert_3_frac": 0.06255336105823517, "expert_4_frac": 0.06245260313153267, "expert_5_frac": 0.06221557781100273, "expert_6_frac": 0.06264305114746094, "expert_7_frac": 0.06258053332567215, "expert_8_frac": 0.0624939501285553, "expert_9_frac": 0.0626603215932846, "expert_frac_max": 0.0626603215932846, "expert_frac_min": 0.06221557781100273, "expert_frac_std": 0.00012476296979002655, "grad_norm": 0.4126286208629608, "learning_rate": 0.00024564958610625925, "loss": 3.6738, "router_aux_loss": 2.000373025536537, "router_entropy": 2.1407304053753613, "router_entropy_norm": 0.7721052849288136, "step": 2550 }, { "epoch": 0.322180916976456, "expert_0_frac": 0.06229957565665245, "expert_10_frac": 0.06251021474599838, "expert_11_frac": 0.06240880489349365, "expert_12_frac": 0.0621674545109272, "expert_13_frac": 0.062565378844738, "expert_14_frac": 0.06240019574761391, "expert_15_frac": 0.06233181059360504, "expert_1_frac": 0.06257034838199615, "expert_2_frac": 0.06248115003108978, "expert_3_frac": 0.06270527094602585, "expert_4_frac": 0.06253506988286972, "expert_5_frac": 0.06253746896982193, "expert_6_frac": 0.06264340132474899, "expert_7_frac": 0.06273256987333298, "expert_8_frac": 0.062466099858284, "expert_9_frac": 0.06264520436525345, "expert_frac_max": 0.06273256987333298, "expert_frac_min": 0.0621674545109272, "expert_frac_std": 0.00015245997929014266, "grad_norm": 0.42913854122161865, "learning_rate": 0.00024326237498387734, "loss": 3.6625, "router_aux_loss": 2.0002928037941454, "router_entropy": 2.1381007156769436, "router_entropy_norm": 0.7711568248570675, "step": 2600 }, { "epoch": 0.32837670384138784, "expert_0_frac": 0.062335118651390076, "expert_10_frac": 0.06263874471187592, "expert_11_frac": 0.06248386576771736, "expert_12_frac": 0.06223371624946594, "expert_13_frac": 0.0627041608095169, "expert_14_frac": 0.062123946845531464, "expert_15_frac": 0.06241362914443016, "expert_1_frac": 0.06246369332075119, "expert_2_frac": 0.0624663308262825, "expert_3_frac": 0.06285633146762848, "expert_4_frac": 0.0624915212392807, "expert_5_frac": 0.06226895749568939, "expert_6_frac": 0.06270433217287064, "expert_7_frac": 0.0626092478632927, "expert_8_frac": 0.06243276596069336, "expert_9_frac": 0.06277365982532501, "expert_frac_max": 0.06285633146762848, "expert_frac_min": 0.062123946845531464, "expert_frac_std": 0.00020398269407451153, "grad_norm": 0.4190995693206787, "learning_rate": 0.00024083601851810444, "loss": 3.6486, "router_aux_loss": 2.000305945575237, "router_entropy": 2.135682564576467, "router_entropy_norm": 0.7702846612068731, "step": 2650 }, { "epoch": 0.3345724907063197, "expert_0_frac": 0.062428977340459824, "expert_10_frac": 0.06245648115873337, "expert_11_frac": 0.06252357363700867, "expert_12_frac": 0.06209154799580574, "expert_13_frac": 0.0625762864947319, "expert_14_frac": 0.062406446784734726, "expert_15_frac": 0.06270066648721695, "expert_1_frac": 0.0626891627907753, "expert_2_frac": 0.06255494803190231, "expert_3_frac": 0.06254468858242035, "expert_4_frac": 0.062471628189086914, "expert_5_frac": 0.06238212808966637, "expert_6_frac": 0.062687948346138, "expert_7_frac": 0.06255471706390381, "expert_8_frac": 0.06231201812624931, "expert_9_frac": 0.06261879950761795, "expert_frac_max": 0.06270066648721695, "expert_frac_min": 0.06209154799580574, "expert_frac_std": 0.00015757165965624154, "grad_norm": 0.3790879249572754, "learning_rate": 0.0002383715351320406, "loss": 3.6488, "router_aux_loss": 2.000347435027361, "router_entropy": 2.131170397289097, "router_entropy_norm": 0.7686572408645856, "step": 2700 }, { "epoch": 0.34076827757125155, "expert_0_frac": 0.062156982719898224, "expert_10_frac": 0.062389545142650604, "expert_11_frac": 0.0625583827495575, "expert_12_frac": 0.062392160296440125, "expert_13_frac": 0.06247055158019066, "expert_14_frac": 0.06246820464730263, "expert_15_frac": 0.06243617460131645, "expert_1_frac": 0.06267351657152176, "expert_2_frac": 0.06233975291252136, "expert_3_frac": 0.06252095103263855, "expert_4_frac": 0.06271501630544662, "expert_5_frac": 0.062358953058719635, "expert_6_frac": 0.06264780461788177, "expert_7_frac": 0.06268636882305145, "expert_8_frac": 0.06252342462539673, "expert_9_frac": 0.06266223639249802, "expert_frac_max": 0.06271501630544662, "expert_frac_min": 0.062156982719898224, "expert_frac_std": 0.00015419730334542692, "grad_norm": 0.42185646295547485, "learning_rate": 0.0002358699592519316, "loss": 3.6241, "router_aux_loss": 2.0003399024903774, "router_entropy": 2.1298128960778318, "router_entropy_norm": 0.7681676257982123, "step": 2750 }, { "epoch": 0.3469640644361834, "expert_0_frac": 0.06238381192088127, "expert_10_frac": 0.06263288110494614, "expert_11_frac": 0.06245595961809158, "expert_12_frac": 0.06230363994836807, "expert_13_frac": 0.062405478209257126, "expert_14_frac": 0.06207668408751488, "expert_15_frac": 0.062493905425071716, "expert_1_frac": 0.06252549588680267, "expert_2_frac": 0.06255904585123062, "expert_3_frac": 0.0626806765794754, "expert_4_frac": 0.06250762194395065, "expert_5_frac": 0.062433961778879166, "expert_6_frac": 0.06261438131332397, "expert_7_frac": 0.0627550259232521, "expert_8_frac": 0.062472350895404816, "expert_9_frac": 0.062699094414711, "expert_frac_max": 0.0627550259232521, "expert_frac_min": 0.06207668408751488, "expert_frac_std": 0.00016724149463698268, "grad_norm": 0.3707304000854492, "learning_rate": 0.00023333234087298565, "loss": 3.626, "router_aux_loss": 2.000274879336357, "router_entropy": 2.123708927979072, "router_entropy_norm": 0.765966084671756, "step": 2800 }, { "epoch": 0.35315985130111527, "expert_0_frac": 0.06237242370843887, "expert_10_frac": 0.06265714764595032, "expert_11_frac": 0.06253128498792648, "expert_12_frac": 0.062280744314193726, "expert_13_frac": 0.06251207739114761, "expert_14_frac": 0.06227829307317734, "expert_15_frac": 0.06258461624383926, "expert_1_frac": 0.06235877051949501, "expert_2_frac": 0.062398917973041534, "expert_3_frac": 0.06260959059000015, "expert_4_frac": 0.0624103806912899, "expert_5_frac": 0.06252153217792511, "expert_6_frac": 0.0626278966665268, "expert_7_frac": 0.06266181170940399, "expert_8_frac": 0.06252672523260117, "expert_9_frac": 0.06266781687736511, "expert_frac_max": 0.06266781687736511, "expert_frac_min": 0.06227829307317734, "expert_frac_std": 0.0001338079891866073, "grad_norm": 0.41720008850097656, "learning_rate": 0.0002307597451186554, "loss": 3.6167, "router_aux_loss": 2.000353735238314, "router_entropy": 2.121400617274145, "router_entropy_norm": 0.7651335375700488, "step": 2850 }, { "epoch": 0.3593556381660471, "expert_0_frac": 0.06238774210214615, "expert_10_frac": 0.0625099465250969, "expert_11_frac": 0.06246226280927658, "expert_12_frac": 0.06219741329550743, "expert_13_frac": 0.06253630667924881, "expert_14_frac": 0.06228252500295639, "expert_15_frac": 0.06252117455005646, "expert_1_frac": 0.06270123273134232, "expert_2_frac": 0.06251953542232513, "expert_3_frac": 0.06263242661952972, "expert_4_frac": 0.06256052106618881, "expert_5_frac": 0.062256649136543274, "expert_6_frac": 0.06262441724538803, "expert_7_frac": 0.0625799149274826, "expert_8_frac": 0.06235365569591522, "expert_9_frac": 0.06287430226802826, "expert_frac_max": 0.06287430226802826, "expert_frac_min": 0.06219741329550743, "expert_frac_std": 0.00017500968533568084, "grad_norm": 0.40777263045310974, "learning_rate": 0.0002281532517935697, "loss": 3.6094, "router_aux_loss": 2.0003093630075455, "router_entropy": 2.118771378012995, "router_entropy_norm": 0.7641852399592057, "step": 2900 }, { "epoch": 0.3655514250309789, "expert_0_frac": 0.06230409815907478, "expert_10_frac": 0.062623530626297, "expert_11_frac": 0.06251265108585358, "expert_12_frac": 0.06222499907016754, "expert_13_frac": 0.06258230656385422, "expert_14_frac": 0.062339454889297485, "expert_15_frac": 0.06251765042543411, "expert_1_frac": 0.06265218555927277, "expert_2_frac": 0.06247617304325104, "expert_3_frac": 0.06254806369543076, "expert_4_frac": 0.06239999458193779, "expert_5_frac": 0.06245831772685051, "expert_6_frac": 0.06256891787052155, "expert_7_frac": 0.06273853778839111, "expert_8_frac": 0.06234954297542572, "expert_9_frac": 0.06270358711481094, "expert_frac_max": 0.06273853778839111, "expert_frac_min": 0.06222499907016754, "expert_frac_std": 0.00014737600577063859, "grad_norm": 0.42828845977783203, "learning_rate": 0.0002255139549303036, "loss": 3.586, "router_aux_loss": 2.0003345051407813, "router_entropy": 2.1164697188759845, "router_entropy_norm": 0.7633550919035104, "step": 2950 }, { "epoch": 0.37174721189591076, "expert_0_frac": 0.06223268806934357, "expert_10_frac": 0.06252967566251755, "expert_11_frac": 0.06271190196275711, "expert_12_frac": 0.062454599887132645, "expert_13_frac": 0.062481142580509186, "expert_14_frac": 0.06233172118663788, "expert_15_frac": 0.06249425560235977, "expert_1_frac": 0.06263557076454163, "expert_2_frac": 0.062410738319158554, "expert_3_frac": 0.06252335011959076, "expert_4_frac": 0.06250470131635666, "expert_5_frac": 0.06235555186867714, "expert_6_frac": 0.06261074542999268, "expert_7_frac": 0.062659852206707, "expert_8_frac": 0.062338899821043015, "expert_9_frac": 0.06272463500499725, "expert_frac_max": 0.06272463500499725, "expert_frac_min": 0.06223268806934357, "expert_frac_std": 0.00014376308536157012, "grad_norm": 0.3720088005065918, "learning_rate": 0.00022284296233017614, "loss": 3.5771, "router_aux_loss": 2.0003691291809083, "router_entropy": 2.1153419841329257, "router_entropy_norm": 0.7629483475731981, "step": 3000 }, { "epoch": 0.37794299876084264, "expert_0_frac": 0.06234049052000046, "expert_10_frac": 0.06246494501829147, "expert_11_frac": 0.06251261383295059, "expert_12_frac": 0.062142081558704376, "expert_13_frac": 0.06254368275403976, "expert_14_frac": 0.06233340874314308, "expert_15_frac": 0.06253020465373993, "expert_1_frac": 0.06254199147224426, "expert_2_frac": 0.062429092824459076, "expert_3_frac": 0.06272594630718231, "expert_4_frac": 0.06251217424869537, "expert_5_frac": 0.062371477484703064, "expert_6_frac": 0.06272263824939728, "expert_7_frac": 0.06261076778173447, "expert_8_frac": 0.0625164806842804, "expert_9_frac": 0.06270202994346619, "expert_frac_max": 0.06272594630718231, "expert_frac_min": 0.062142081558704376, "expert_frac_std": 0.00015555151912849396, "grad_norm": 0.38100388646125793, "learning_rate": 0.00022014139509826905, "loss": 3.5893, "router_aux_loss": 2.000323775559664, "router_entropy": 2.1117512386168045, "router_entropy_norm": 0.7616532598858975, "step": 3050 }, { "epoch": 0.38413878562577447, "expert_0_frac": 0.06226764991879463, "expert_10_frac": 0.06236816570162773, "expert_11_frac": 0.06274646520614624, "expert_12_frac": 0.06223607435822487, "expert_13_frac": 0.06262464821338654, "expert_14_frac": 0.06223903223872185, "expert_15_frac": 0.06242969632148743, "expert_1_frac": 0.06258812546730042, "expert_2_frac": 0.06244523450732231, "expert_3_frac": 0.06271171569824219, "expert_4_frac": 0.06243456155061722, "expert_5_frac": 0.06234009563922882, "expert_6_frac": 0.062552310526371, "expert_7_frac": 0.0627344399690628, "expert_8_frac": 0.0625125989317894, "expert_9_frac": 0.06276921182870865, "expert_frac_max": 0.06276921182870865, "expert_frac_min": 0.06223607435822487, "expert_frac_std": 0.00018368911696597934, "grad_norm": 0.39150938391685486, "learning_rate": 0.0002174103871728613, "loss": 3.5581, "router_aux_loss": 2.0003552556037905, "router_entropy": 2.108792674889167, "router_entropy_norm": 0.7605861835813934, "step": 3100 }, { "epoch": 0.3903345724907063, "expert_0_frac": 0.0623362734913826, "expert_10_frac": 0.06251577287912369, "expert_11_frac": 0.06251625716686249, "expert_12_frac": 0.062346331775188446, "expert_13_frac": 0.06256897002458572, "expert_14_frac": 0.0623774528503418, "expert_15_frac": 0.062373917549848557, "expert_1_frac": 0.062472108751535416, "expert_2_frac": 0.06244368851184845, "expert_3_frac": 0.06261617690324783, "expert_4_frac": 0.06247147545218468, "expert_5_frac": 0.06246494501829147, "expert_6_frac": 0.06274916231632233, "expert_7_frac": 0.06245803460478783, "expert_8_frac": 0.06257476657629013, "expert_9_frac": 0.06271468102931976, "expert_frac_max": 0.06274916231632233, "expert_frac_min": 0.0623362734913826, "expert_frac_std": 0.00012154950672993436, "grad_norm": 0.4003765881061554, "learning_rate": 0.00021465108484947738, "loss": 3.5526, "router_aux_loss": 2.000348298251629, "router_entropy": 2.1078100501249235, "router_entropy_norm": 0.7602317766127862, "step": 3150 }, { "epoch": 0.3965303593556382, "expert_0_frac": 0.062301527708768845, "expert_10_frac": 0.062433645129203796, "expert_11_frac": 0.06263574212789536, "expert_12_frac": 0.06211092323064804, "expert_13_frac": 0.062396157532930374, "expert_14_frac": 0.06230778619647026, "expert_15_frac": 0.06266193836927414, "expert_1_frac": 0.06248924881219864, "expert_2_frac": 0.062424615025520325, "expert_3_frac": 0.06274488568305969, "expert_4_frac": 0.06253775954246521, "expert_5_frac": 0.062442515045404434, "expert_6_frac": 0.0626094862818718, "expert_7_frac": 0.06269068270921707, "expert_8_frac": 0.062457650899887085, "expert_9_frac": 0.06275545060634613, "expert_frac_max": 0.06275545060634613, "expert_frac_min": 0.06211092323064804, "expert_frac_std": 0.0001776602875906974, "grad_norm": 0.37601521611213684, "learning_rate": 0.00021186464629974848, "loss": 3.5602, "router_aux_loss": 2.000331805050373, "router_entropy": 2.104159056780239, "router_entropy_norm": 0.7589149591146124, "step": 3200 }, { "epoch": 0.4, "eval_loss": 2.932450771331787, "eval_runtime": 33.668, "eval_samples_per_second": 51.325, "eval_steps_per_second": 6.416, "expert_0_frac": 0.06264324486255646, "expert_10_frac": 0.06252216547727585, "expert_11_frac": 0.06269601732492447, "expert_12_frac": 0.06214261054992676, "expert_13_frac": 0.0626559853553772, "expert_14_frac": 0.062081582844257355, "expert_15_frac": 0.06226176396012306, "expert_1_frac": 0.06273604184389114, "expert_2_frac": 0.06268617510795593, "expert_3_frac": 0.06241494044661522, "expert_4_frac": 0.062481921166181564, "expert_5_frac": 0.062450990080833435, "expert_6_frac": 0.06252512335777283, "expert_7_frac": 0.06260444968938828, "expert_8_frac": 0.06249772384762764, "expert_9_frac": 0.06259922683238983, "expert_frac_max": 0.06273604184389114, "expert_frac_min": 0.062081582844257355, "expert_frac_std": 0.00019408830848988146, "router_aux_loss": 2.000305411506848, "router_entropy": 2.1031951366897568, "router_entropy_norm": 0.7585672984310244, "step": 3228 }, { "epoch": 0.40272614622057, "expert_0_frac": 0.062352053821086884, "expert_10_frac": 0.0625324547290802, "expert_11_frac": 0.06247243285179138, "expert_12_frac": 0.06233026087284088, "expert_13_frac": 0.06254485249519348, "expert_14_frac": 0.06235339492559433, "expert_15_frac": 0.06269770115613937, "expert_1_frac": 0.062410831451416016, "expert_2_frac": 0.06258723139762878, "expert_3_frac": 0.062351156026124954, "expert_4_frac": 0.062363799661397934, "expert_5_frac": 0.06258995085954666, "expert_6_frac": 0.06269625574350357, "expert_7_frac": 0.06271589547395706, "expert_8_frac": 0.06235557794570923, "expert_9_frac": 0.06264617294073105, "expert_frac_max": 0.06271589547395706, "expert_frac_min": 0.06233026087284088, "expert_frac_std": 0.00014301776536740363, "grad_norm": 0.38073164224624634, "learning_rate": 0.00020905224108528923, "loss": 3.5458, "router_aux_loss": 2.000334331935102, "router_entropy": 2.10179190532389, "router_entropy_norm": 0.7580611896978355, "step": 3250 }, { "epoch": 0.40892193308550184, "expert_0_frac": 0.062395963817834854, "expert_10_frac": 0.06260203570127487, "expert_11_frac": 0.06264384835958481, "expert_12_frac": 0.06235386058688164, "expert_13_frac": 0.06258896738290787, "expert_14_frac": 0.0623982809484005, "expert_15_frac": 0.06242229789495468, "expert_1_frac": 0.06261791288852692, "expert_2_frac": 0.06222543120384216, "expert_3_frac": 0.06262610852718353, "expert_4_frac": 0.06244849041104317, "expert_5_frac": 0.062327079474925995, "expert_6_frac": 0.06249148026108742, "expert_7_frac": 0.0627094954252243, "expert_8_frac": 0.06250277161598206, "expert_9_frac": 0.06264599412679672, "expert_frac_max": 0.0627094954252243, "expert_frac_min": 0.06222543120384216, "expert_frac_std": 0.00013919193588662893, "grad_norm": 0.36956146359443665, "learning_rate": 0.00020621504966679337, "loss": 3.5394, "router_aux_loss": 2.000408248752356, "router_entropy": 2.1004575939973193, "router_entropy_norm": 0.7575799386143741, "step": 3300 }, { "epoch": 0.41511771995043373, "expert_0_frac": 0.06248762831091881, "expert_10_frac": 0.06243772804737091, "expert_11_frac": 0.06252548098564148, "expert_12_frac": 0.06232022866606712, "expert_13_frac": 0.06252936273813248, "expert_14_frac": 0.062245216220617294, "expert_15_frac": 0.06240775063633919, "expert_1_frac": 0.06255979090929031, "expert_2_frac": 0.06250588595867157, "expert_3_frac": 0.06262928992509842, "expert_4_frac": 0.062493931502103806, "expert_5_frac": 0.06225007027387619, "expert_6_frac": 0.06273949146270752, "expert_7_frac": 0.06255599111318588, "expert_8_frac": 0.06259478628635406, "expert_9_frac": 0.06271740049123764, "expert_frac_max": 0.06273949146270752, "expert_frac_min": 0.062245216220617294, "expert_frac_std": 0.00014395613106898963, "grad_norm": 0.4224052131175995, "learning_rate": 0.00020335426290855494, "loss": 3.5179, "router_aux_loss": 2.0003286020457747, "router_entropy": 2.0987680555631716, "router_entropy_norm": 0.7569705664342901, "step": 3350 }, { "epoch": 0.42131350681536556, "expert_0_frac": 0.06220756098628044, "expert_10_frac": 0.06257414072751999, "expert_11_frac": 0.06239807978272438, "expert_12_frac": 0.06225958094000816, "expert_13_frac": 0.062421929091215134, "expert_14_frac": 0.062199681997299194, "expert_15_frac": 0.062410350888967514, "expert_1_frac": 0.06256657093763351, "expert_2_frac": 0.062459468841552734, "expert_3_frac": 0.06267154961824417, "expert_4_frac": 0.06252565234899521, "expert_5_frac": 0.06251086294651031, "expert_6_frac": 0.06265921145677567, "expert_7_frac": 0.06275542080402374, "expert_8_frac": 0.0625537633895874, "expert_9_frac": 0.06282620877027512, "expert_frac_max": 0.06282620877027512, "expert_frac_min": 0.062199681997299194, "expert_frac_std": 0.00018251939036417753, "grad_norm": 0.38169124722480774, "learning_rate": 0.00020047108157862255, "loss": 3.5252, "router_aux_loss": 2.0002493160963057, "router_entropy": 2.0952287341778475, "router_entropy_norm": 0.7556940260816102, "step": 3400 }, { "epoch": 0.4275092936802974, "expert_0_frac": 0.06231572479009628, "expert_10_frac": 0.06259610503911972, "expert_11_frac": 0.06259749084711075, "expert_12_frac": 0.06229960918426514, "expert_13_frac": 0.06249750778079033, "expert_14_frac": 0.06224007532000542, "expert_15_frac": 0.062429435551166534, "expert_1_frac": 0.06246008723974228, "expert_2_frac": 0.06253749132156372, "expert_3_frac": 0.06280416995286942, "expert_4_frac": 0.06255106627941132, "expert_5_frac": 0.062294136732816696, "expert_6_frac": 0.06257584691047668, "expert_7_frac": 0.06268903613090515, "expert_8_frac": 0.06247768551111221, "expert_9_frac": 0.06263455748558044, "expert_frac_max": 0.06280416995286942, "expert_frac_min": 0.06224007532000542, "expert_frac_std": 0.00015620443446096033, "grad_norm": 0.37302330136299133, "learning_rate": 0.0001975667158447968, "loss": 3.5342, "router_aux_loss": 2.000293505489826, "router_entropy": 2.0930864307905237, "router_entropy_norm": 0.7549213534633673, "step": 3450 }, { "epoch": 0.43370508054522927, "expert_0_frac": 0.062250856310129166, "expert_10_frac": 0.06260839104652405, "expert_11_frac": 0.06263632327318192, "expert_12_frac": 0.06220966950058937, "expert_13_frac": 0.0625518411397934, "expert_14_frac": 0.06223663315176964, "expert_15_frac": 0.0625864714384079, "expert_1_frac": 0.06254573166370392, "expert_2_frac": 0.062443047761917114, "expert_3_frac": 0.06261158734560013, "expert_4_frac": 0.062505804002285, "expert_5_frac": 0.06240115687251091, "expert_6_frac": 0.06271927803754807, "expert_7_frac": 0.06251521408557892, "expert_8_frac": 0.0624253936111927, "expert_9_frac": 0.0627526193857193, "expert_frac_max": 0.0627526193857193, "expert_frac_min": 0.06220966950058937, "expert_frac_std": 0.00016360751760657877, "grad_norm": 0.39231953024864197, "learning_rate": 0.00019464238476668252, "loss": 3.5022, "router_aux_loss": 2.000298661738634, "router_entropy": 2.092628412457804, "router_entropy_norm": 0.7547561582690545, "step": 3500 }, { "epoch": 0.4399008674101611, "expert_0_frac": 0.062313877046108246, "expert_10_frac": 0.06249168515205383, "expert_11_frac": 0.06265368312597275, "expert_12_frac": 0.062291502952575684, "expert_13_frac": 0.06255083531141281, "expert_14_frac": 0.06223292276263237, "expert_15_frac": 0.06240782514214516, "expert_1_frac": 0.06274271756410599, "expert_2_frac": 0.0625762939453125, "expert_3_frac": 0.06271133571863174, "expert_4_frac": 0.06232191249728203, "expert_5_frac": 0.062295157462358475, "expert_6_frac": 0.06254404038190842, "expert_7_frac": 0.06275083869695663, "expert_8_frac": 0.062450066208839417, "expert_9_frac": 0.06266532838344574, "expert_frac_max": 0.06275083869695663, "expert_frac_min": 0.06223292276263237, "expert_frac_std": 0.00017549331823829561, "grad_norm": 0.37481606006622314, "learning_rate": 0.0001916993157840083, "loss": 3.503, "router_aux_loss": 2.0003165489435197, "router_entropy": 2.089450304582715, "router_entropy_norm": 0.7536098981513543, "step": 3550 }, { "epoch": 0.44609665427509293, "expert_0_frac": 0.06234418973326683, "expert_10_frac": 0.062443748116493225, "expert_11_frac": 0.06271863728761673, "expert_12_frac": 0.06224535033106804, "expert_13_frac": 0.062424443662166595, "expert_14_frac": 0.062366239726543427, "expert_15_frac": 0.06260008364915848, "expert_1_frac": 0.06257687509059906, "expert_2_frac": 0.0623343363404274, "expert_3_frac": 0.0625820904970169, "expert_4_frac": 0.06246402859687805, "expert_5_frac": 0.06257043033838272, "expert_6_frac": 0.0626128613948822, "expert_7_frac": 0.06262767314910889, "expert_8_frac": 0.06241843104362488, "expert_9_frac": 0.06267061084508896, "expert_frac_max": 0.06271863728761673, "expert_frac_min": 0.06224535033106804, "expert_frac_std": 0.00013780524022877216, "grad_norm": 0.39895886182785034, "learning_rate": 0.00018873874420142906, "loss": 3.4977, "router_aux_loss": 2.000358630567789, "router_entropy": 2.087255741332968, "router_entropy_norm": 0.7528183767720226, "step": 3600 }, { "epoch": 0.45229244114002476, "expert_0_frac": 0.0623842254281044, "expert_10_frac": 0.06250423938035965, "expert_11_frac": 0.06252159178256989, "expert_12_frac": 0.06225087121129036, "expert_13_frac": 0.06249937787652016, "expert_14_frac": 0.06226740777492523, "expert_15_frac": 0.06252508610486984, "expert_1_frac": 0.06258075684309006, "expert_2_frac": 0.06247315928339958, "expert_3_frac": 0.06254542618989944, "expert_4_frac": 0.06261557340621948, "expert_5_frac": 0.06244013085961342, "expert_6_frac": 0.06266670674085617, "expert_7_frac": 0.06256407499313354, "expert_8_frac": 0.062476083636283875, "expert_9_frac": 0.06268531084060669, "expert_frac_max": 0.06268531084060669, "expert_frac_min": 0.06225087121129036, "expert_frac_std": 0.00012213051377329975, "grad_norm": 0.37105000019073486, "learning_rate": 0.0001857619126700273, "loss": 3.4968, "router_aux_loss": 2.0002850928902625, "router_entropy": 2.085349079382916, "router_entropy_norm": 0.7521306938370246, "step": 3650 }, { "epoch": 0.45848822800495664, "expert_0_frac": 0.062216244637966156, "expert_10_frac": 0.06255313754081726, "expert_11_frac": 0.06254009902477264, "expert_12_frac": 0.06221632659435272, "expert_13_frac": 0.06249389052391052, "expert_14_frac": 0.06222785636782646, "expert_15_frac": 0.06247757747769356, "expert_1_frac": 0.06254442036151886, "expert_2_frac": 0.06252530217170715, "expert_3_frac": 0.06265315413475037, "expert_4_frac": 0.06257827579975128, "expert_5_frac": 0.062381528317928314, "expert_6_frac": 0.06273956596851349, "expert_7_frac": 0.06278820335865021, "expert_8_frac": 0.06248488277196884, "expert_9_frac": 0.06257954239845276, "expert_frac_max": 0.06278820335865021, "expert_frac_min": 0.062216244637966156, "expert_frac_std": 0.0001699533750070259, "grad_norm": 0.42140495777130127, "learning_rate": 0.00018277007066573098, "loss": 3.4851, "router_aux_loss": 2.0002259370684623, "router_entropy": 2.0829126729443668, "router_entropy_norm": 0.7512519459654033, "step": 3700 }, { "epoch": 0.4646840148698885, "expert_0_frac": 0.06227515637874603, "expert_10_frac": 0.06233804672956467, "expert_11_frac": 0.06253945827484131, "expert_12_frac": 0.06221473962068558, "expert_13_frac": 0.06275515258312225, "expert_14_frac": 0.06235675886273384, "expert_15_frac": 0.06251177936792374, "expert_1_frac": 0.06242473050951958, "expert_2_frac": 0.06246200203895569, "expert_3_frac": 0.06270095705986023, "expert_4_frac": 0.06234190613031387, "expert_5_frac": 0.06254827976226807, "expert_6_frac": 0.06270807236433029, "expert_7_frac": 0.06254412978887558, "expert_8_frac": 0.06251411885023117, "expert_9_frac": 0.0627647191286087, "expert_frac_max": 0.0627647191286087, "expert_frac_min": 0.06221473962068558, "expert_frac_std": 0.00017045202548615634, "grad_norm": 0.39511430263519287, "learning_rate": 0.0001797644739648662, "loss": 3.4754, "router_aux_loss": 2.0003580312430858, "router_entropy": 2.081217644525071, "router_entropy_norm": 0.7506405936917324, "step": 3750 }, { "epoch": 0.4708798017348203, "expert_0_frac": 0.062218230217695236, "expert_10_frac": 0.06246109679341316, "expert_11_frac": 0.06242981180548668, "expert_12_frac": 0.06220239773392677, "expert_13_frac": 0.06252938508987427, "expert_14_frac": 0.06232874095439911, "expert_15_frac": 0.06253427267074585, "expert_1_frac": 0.06251819431781769, "expert_2_frac": 0.06247918680310249, "expert_3_frac": 0.06275278329849243, "expert_4_frac": 0.062442347407341, "expert_5_frac": 0.06229308620095253, "expert_6_frac": 0.0625835508108139, "expert_7_frac": 0.06279103457927704, "expert_8_frac": 0.06255204975605011, "expert_9_frac": 0.06288384646177292, "expert_frac_max": 0.06288384646177292, "expert_frac_min": 0.06220239773392677, "expert_frac_std": 0.00019292465003672987, "grad_norm": 0.39034244418144226, "learning_rate": 0.00017674638411706612, "loss": 3.4804, "router_aux_loss": 2.0002825504541395, "router_entropy": 2.0786126318449774, "router_entropy_norm": 0.7497010339729764, "step": 3800 }, { "epoch": 0.4770755885997522, "expert_0_frac": 0.06214878335595131, "expert_10_frac": 0.06230198219418526, "expert_11_frac": 0.06274259835481644, "expert_12_frac": 0.06231401488184929, "expert_13_frac": 0.0625581219792366, "expert_14_frac": 0.062250979244709015, "expert_15_frac": 0.06247100979089737, "expert_1_frac": 0.06267211586236954, "expert_2_frac": 0.06251274049282074, "expert_3_frac": 0.06262460350990295, "expert_4_frac": 0.06242990493774414, "expert_5_frac": 0.06250366568565369, "expert_6_frac": 0.06266610324382782, "expert_7_frac": 0.06275343149900436, "expert_8_frac": 0.06240144744515419, "expert_9_frac": 0.06264853477478027, "expert_frac_max": 0.06275343149900436, "expert_frac_min": 0.06214878335595131, "expert_frac_std": 0.00018185534281656146, "grad_norm": 0.36116084456443787, "learning_rate": 0.00017371706791575632, "loss": 3.474, "router_aux_loss": 2.000293119698763, "router_entropy": 2.077186061926186, "router_entropy_norm": 0.749186507636146, "step": 3850 }, { "epoch": 0.483271375464684, "expert_0_frac": 0.06240856274962425, "expert_10_frac": 0.06241278350353241, "expert_11_frac": 0.06253360211849213, "expert_12_frac": 0.06221204996109009, "expert_13_frac": 0.06244863197207451, "expert_14_frac": 0.062473393976688385, "expert_15_frac": 0.06247289106249809, "expert_1_frac": 0.06267870217561722, "expert_2_frac": 0.06242802366614342, "expert_3_frac": 0.06274519115686417, "expert_4_frac": 0.062426675111055374, "expert_5_frac": 0.06239253655076027, "expert_6_frac": 0.0625360757112503, "expert_7_frac": 0.062477219849824905, "expert_8_frac": 0.06259272992610931, "expert_9_frac": 0.06276094913482666, "expert_frac_max": 0.06276094913482666, "expert_frac_min": 0.06221204996109009, "expert_frac_std": 0.00014065694995224476, "grad_norm": 0.39785218238830566, "learning_rate": 0.00017067779686643893, "loss": 3.4499, "router_aux_loss": 2.0003442023694515, "router_entropy": 2.0774265551939606, "router_entropy_norm": 0.7492732472473423, "step": 3900 }, { "epoch": 0.48946716232961585, "expert_0_frac": 0.06253805756568909, "expert_10_frac": 0.062361087650060654, "expert_11_frac": 0.06251271814107895, "expert_12_frac": 0.06228022277355194, "expert_13_frac": 0.062415316700935364, "expert_14_frac": 0.06231295317411423, "expert_15_frac": 0.062470030039548874, "expert_1_frac": 0.06256447732448578, "expert_2_frac": 0.06246081367135048, "expert_3_frac": 0.0626181960105896, "expert_4_frac": 0.06245271861553192, "expert_5_frac": 0.06253720074892044, "expert_6_frac": 0.06264904141426086, "expert_7_frac": 0.06271599978208542, "expert_8_frac": 0.06236222758889198, "expert_9_frac": 0.06274896115064621, "expert_frac_max": 0.06274896115064621, "expert_frac_min": 0.06228022277355194, "expert_frac_std": 0.00013793223479297012, "grad_norm": 0.36815178394317627, "learning_rate": 0.00016762984665299937, "loss": 3.4589, "router_aux_loss": 2.000360932648182, "router_entropy": 2.0749111812810104, "router_entropy_norm": 0.7483660178797937, "step": 3950 }, { "epoch": 0.49566294919454773, "expert_0_frac": 0.062341488897800446, "expert_10_frac": 0.0626181811094284, "expert_11_frac": 0.06261951476335526, "expert_12_frac": 0.06203712150454521, "expert_13_frac": 0.06248367950320244, "expert_14_frac": 0.06227276101708412, "expert_15_frac": 0.06273195147514343, "expert_1_frac": 0.062674380838871, "expert_2_frac": 0.06232789531350136, "expert_3_frac": 0.06270692497491837, "expert_4_frac": 0.06244327127933502, "expert_5_frac": 0.06220940500497818, "expert_6_frac": 0.06272995471954346, "expert_7_frac": 0.06274814158678055, "expert_8_frac": 0.06237934157252312, "expert_9_frac": 0.06267600506544113, "expert_frac_max": 0.06274814158678055, "expert_frac_min": 0.06203712150454521, "expert_frac_std": 0.00021967220527585596, "grad_norm": 0.36331707239151, "learning_rate": 0.00016457449660225917, "loss": 3.456, "router_aux_loss": 2.000295414030552, "router_entropy": 2.0727119484667975, "router_entropy_norm": 0.7475728123110874, "step": 4000 }, { "epoch": 0.5, "eval_loss": 2.835195779800415, "eval_runtime": 33.137, "eval_samples_per_second": 52.147, "eval_steps_per_second": 6.518, "expert_0_frac": 0.06223877891898155, "expert_10_frac": 0.06225793808698654, "expert_11_frac": 0.06256310641765594, "expert_12_frac": 0.06226780265569687, "expert_13_frac": 0.0626240149140358, "expert_14_frac": 0.06233023852109909, "expert_15_frac": 0.06236087530851364, "expert_1_frac": 0.06266454607248306, "expert_2_frac": 0.062462449073791504, "expert_3_frac": 0.06261497735977173, "expert_4_frac": 0.06259915977716446, "expert_5_frac": 0.06261510401964188, "expert_6_frac": 0.06256020814180374, "expert_7_frac": 0.06260185688734055, "expert_8_frac": 0.06254878640174866, "expert_9_frac": 0.06269016861915588, "expert_frac_max": 0.06269016861915588, "expert_frac_min": 0.06223877891898155, "expert_frac_std": 0.00015612253628205508, "router_aux_loss": 2.0002878901270247, "router_entropy": 2.0733980220131243, "router_entropy_norm": 0.7478202610368301, "step": 4035 }, { "epoch": 0.5018587360594795, "expert_0_frac": 0.06223723292350769, "expert_10_frac": 0.06237826868891716, "expert_11_frac": 0.06259053945541382, "expert_12_frac": 0.06238409876823425, "expert_13_frac": 0.06224346160888672, "expert_14_frac": 0.06210483983159065, "expert_15_frac": 0.06264093518257141, "expert_1_frac": 0.06250996142625809, "expert_2_frac": 0.06256218254566193, "expert_3_frac": 0.06271038204431534, "expert_4_frac": 0.06232079491019249, "expert_5_frac": 0.06244299188256264, "expert_6_frac": 0.06276946514844894, "expert_7_frac": 0.06290285289287567, "expert_8_frac": 0.06239217147231102, "expert_9_frac": 0.06280983984470367, "expert_frac_max": 0.06290285289287567, "expert_frac_min": 0.06210483983159065, "expert_frac_std": 0.00022707448806613684, "grad_norm": 0.38191738724708557, "learning_rate": 0.0001615130291469992, "loss": 3.4508, "router_aux_loss": 2.0002857352296513, "router_entropy": 2.071026981290844, "router_entropy_norm": 0.7469650888638851, "step": 4050 }, { "epoch": 0.5080545229244114, "expert_0_frac": 0.06229468062520027, "expert_10_frac": 0.06243317574262619, "expert_11_frac": 0.06253965944051743, "expert_12_frac": 0.06219393387436867, "expert_13_frac": 0.06261800229549408, "expert_14_frac": 0.06220797449350357, "expert_15_frac": 0.06253494322299957, "expert_1_frac": 0.06259635090827942, "expert_2_frac": 0.062486693263053894, "expert_3_frac": 0.062488868832588196, "expert_4_frac": 0.062300536781549454, "expert_5_frac": 0.06251173466444016, "expert_6_frac": 0.0627284049987793, "expert_7_frac": 0.06274477392435074, "expert_8_frac": 0.06251753866672516, "expert_9_frac": 0.0628027394413948, "expert_frac_max": 0.0628027394413948, "expert_frac_min": 0.06219393387436867, "expert_frac_std": 0.00018181400082539767, "grad_norm": 0.3982459008693695, "learning_rate": 0.00015844672928768015, "loss": 3.4419, "router_aux_loss": 2.0002834285795688, "router_entropy": 2.069342819737891, "router_entropy_norm": 0.7463576559837599, "step": 4100 }, { "epoch": 0.5142503097893433, "expert_0_frac": 0.06240158900618553, "expert_10_frac": 0.06245036423206329, "expert_11_frac": 0.06251361221075058, "expert_12_frac": 0.062208451330661774, "expert_13_frac": 0.0623130165040493, "expert_14_frac": 0.06229441985487938, "expert_15_frac": 0.06245220825076103, "expert_1_frac": 0.06258202344179153, "expert_2_frac": 0.06249164044857025, "expert_3_frac": 0.06279388070106506, "expert_4_frac": 0.06255616247653961, "expert_5_frac": 0.0623348131775856, "expert_6_frac": 0.06263235211372375, "expert_7_frac": 0.06268365681171417, "expert_8_frac": 0.06251730024814606, "expert_9_frac": 0.06277452409267426, "expert_frac_max": 0.06279388070106506, "expert_frac_min": 0.062208451330661774, "expert_frac_std": 0.0001687915064394474, "grad_norm": 0.42543739080429077, "learning_rate": 0.0001553768840530846, "loss": 3.4514, "router_aux_loss": 2.0002649518847466, "router_entropy": 2.0675219970072307, "router_entropy_norm": 0.7457009330027945, "step": 4150 }, { "epoch": 0.5204460966542751, "expert_0_frac": 0.062248531728982925, "expert_10_frac": 0.06262663751840591, "expert_11_frac": 0.06247673183679581, "expert_12_frac": 0.062475770711898804, "expert_13_frac": 0.06257357448339462, "expert_14_frac": 0.06223734840750694, "expert_15_frac": 0.06242147088050842, "expert_1_frac": 0.06263577193021774, "expert_2_frac": 0.06257237493991852, "expert_3_frac": 0.06253638863563538, "expert_4_frac": 0.06244934722781181, "expert_5_frac": 0.06250207126140594, "expert_6_frac": 0.06273172795772552, "expert_7_frac": 0.0626993179321289, "expert_8_frac": 0.062292035669088364, "expert_9_frac": 0.06252092123031616, "expert_frac_max": 0.06273172795772552, "expert_frac_min": 0.06223734840750694, "expert_frac_std": 0.0001473423035349697, "grad_norm": 0.3881041407585144, "learning_rate": 0.00015230478196010788, "loss": 3.4374, "router_aux_loss": 2.000279816389084, "router_entropy": 2.0661936601996422, "router_entropy_norm": 0.74522183677156, "step": 4200 }, { "epoch": 0.5266418835192069, "expert_0_frac": 0.062292248010635376, "expert_10_frac": 0.0625143051147461, "expert_11_frac": 0.06272810697555542, "expert_12_frac": 0.062146659940481186, "expert_13_frac": 0.06232764571905136, "expert_14_frac": 0.06244494393467903, "expert_15_frac": 0.06246032193303108, "expert_1_frac": 0.06257320195436478, "expert_2_frac": 0.06242893263697624, "expert_3_frac": 0.06263642758131027, "expert_4_frac": 0.06243368983268738, "expert_5_frac": 0.06238401681184769, "expert_6_frac": 0.06271476298570633, "expert_7_frac": 0.06273151189088821, "expert_8_frac": 0.06250733137130737, "expert_9_frac": 0.06267590820789337, "expert_frac_max": 0.06273151189088821, "expert_frac_min": 0.062146659940481186, "expert_frac_std": 0.00016973471792880446, "grad_norm": 0.3913940191268921, "learning_rate": 0.00014923171247292443, "loss": 3.4388, "router_aux_loss": 2.000279443114996, "router_entropy": 2.0648219058538477, "router_entropy_norm": 0.7447270809735611, "step": 4250 }, { "epoch": 0.5328376703841388, "expert_0_frac": 0.062424205243587494, "expert_10_frac": 0.06258242577314377, "expert_11_frac": 0.062486547976732254, "expert_12_frac": 0.06218972057104111, "expert_13_frac": 0.06255097687244415, "expert_14_frac": 0.06233542412519455, "expert_15_frac": 0.06249724701046944, "expert_1_frac": 0.062383346259593964, "expert_2_frac": 0.062382712960243225, "expert_3_frac": 0.0627749040722847, "expert_4_frac": 0.0625402182340622, "expert_5_frac": 0.06236400082707405, "expert_6_frac": 0.06257309019565582, "expert_7_frac": 0.06259342283010483, "expert_8_frac": 0.06245896965265274, "expert_9_frac": 0.0628628209233284, "expert_frac_max": 0.0628628209233284, "expert_frac_min": 0.06218972057104111, "expert_frac_std": 0.0001651518396101892, "grad_norm": 0.3753291070461273, "learning_rate": 0.0001461589654617563, "loss": 3.4265, "router_aux_loss": 2.000321349054575, "router_entropy": 2.0635299008215466, "router_entropy_norm": 0.744261088660335, "step": 4300 }, { "epoch": 0.5390334572490706, "expert_0_frac": 0.06218123808503151, "expert_10_frac": 0.0625486820936203, "expert_11_frac": 0.06267302483320236, "expert_12_frac": 0.06233431026339531, "expert_13_frac": 0.06256947666406631, "expert_14_frac": 0.062171194702386856, "expert_15_frac": 0.06244954094290733, "expert_1_frac": 0.0626150518655777, "expert_2_frac": 0.062484242022037506, "expert_3_frac": 0.06249242275953293, "expert_4_frac": 0.06238124892115593, "expert_5_frac": 0.06239190697669983, "expert_6_frac": 0.06276813894510269, "expert_7_frac": 0.06265375018119812, "expert_8_frac": 0.06257021427154541, "expert_9_frac": 0.06271558254957199, "expert_frac_max": 0.06276813894510269, "expert_frac_min": 0.062171194702386856, "expert_frac_std": 0.0001757133868522942, "grad_norm": 0.35662841796875, "learning_rate": 0.00014308783066147178, "loss": 3.4153, "router_aux_loss": 2.000260810405016, "router_entropy": 2.0623221289614837, "router_entropy_norm": 0.7438254770420755, "step": 4350 }, { "epoch": 0.5452292441140025, "expert_0_frac": 0.062258683145046234, "expert_10_frac": 0.062456462532281876, "expert_11_frac": 0.06260629743337631, "expert_12_frac": 0.0621049627661705, "expert_13_frac": 0.062417272478342056, "expert_14_frac": 0.06238047406077385, "expert_15_frac": 0.06264665722846985, "expert_1_frac": 0.0626235082745552, "expert_2_frac": 0.06254547834396362, "expert_3_frac": 0.0626249685883522, "expert_4_frac": 0.06244637444615364, "expert_5_frac": 0.06245977059006691, "expert_6_frac": 0.06254879385232925, "expert_7_frac": 0.06266503781080246, "expert_8_frac": 0.06235170364379883, "expert_9_frac": 0.06286357343196869, "expert_frac_max": 0.06286357343196869, "expert_frac_min": 0.0621049627661705, "expert_frac_std": 0.00018098823784384876, "grad_norm": 0.36747997999191284, "learning_rate": 0.00014001959713024002, "loss": 3.4252, "router_aux_loss": 2.000273561477661, "router_entropy": 2.0592385633910695, "router_entropy_norm": 0.7427133158529023, "step": 4400 }, { "epoch": 0.5514250309789344, "expert_0_frac": 0.06231154128909111, "expert_10_frac": 0.06250350177288055, "expert_11_frac": 0.06245879456400871, "expert_12_frac": 0.06217548996210098, "expert_13_frac": 0.062260840088129044, "expert_14_frac": 0.06228562816977501, "expert_15_frac": 0.06255662441253662, "expert_1_frac": 0.0626593753695488, "expert_2_frac": 0.062435850501060486, "expert_3_frac": 0.06265868246555328, "expert_4_frac": 0.0625995323061943, "expert_5_frac": 0.06244940683245659, "expert_6_frac": 0.06266835331916809, "expert_7_frac": 0.06271927058696747, "expert_8_frac": 0.06248893216252327, "expert_9_frac": 0.06276819854974747, "expert_frac_max": 0.06276819854974747, "expert_frac_min": 0.06217548996210098, "expert_frac_std": 0.00017597920668777078, "grad_norm": 0.41151490807533264, "learning_rate": 0.00013695555270847056, "loss": 3.427, "router_aux_loss": 2.0002112005651, "router_entropy": 2.0581923769166073, "router_entropy_norm": 0.7423359838432644, "step": 4450 }, { "epoch": 0.5576208178438662, "expert_0_frac": 0.062337107956409454, "expert_10_frac": 0.06262237578630447, "expert_11_frac": 0.06275221705436707, "expert_12_frac": 0.06221630424261093, "expert_13_frac": 0.062493275851011276, "expert_14_frac": 0.06234654784202576, "expert_15_frac": 0.06257208436727524, "expert_1_frac": 0.0625588446855545, "expert_2_frac": 0.06234808266162872, "expert_3_frac": 0.06260024011135101, "expert_4_frac": 0.06243022903800011, "expert_5_frac": 0.06240806728601456, "expert_6_frac": 0.06267375499010086, "expert_7_frac": 0.06255117803812027, "expert_8_frac": 0.06241467222571373, "expert_9_frac": 0.06267504394054413, "expert_frac_max": 0.06275221705436707, "expert_frac_min": 0.06221630424261093, "expert_frac_std": 0.00014945138536859304, "grad_norm": 0.376271516084671, "learning_rate": 0.0001338969834782632, "loss": 3.4129, "router_aux_loss": 2.000335542410612, "router_entropy": 2.0577339370921255, "router_entropy_norm": 0.7421706366279329, "step": 4500 }, { "epoch": 0.563816604708798, "expert_0_frac": 0.0622267946600914, "expert_10_frac": 0.06248943507671356, "expert_11_frac": 0.06273786723613739, "expert_12_frac": 0.062277115881443024, "expert_13_frac": 0.06260387599468231, "expert_14_frac": 0.06225940212607384, "expert_15_frac": 0.06244751811027527, "expert_1_frac": 0.06263836473226547, "expert_2_frac": 0.06259891390800476, "expert_3_frac": 0.0626433789730072, "expert_4_frac": 0.06251221895217896, "expert_5_frac": 0.062373142689466476, "expert_6_frac": 0.06243934854865074, "expert_7_frac": 0.06258486211299896, "expert_8_frac": 0.06255466490983963, "expert_9_frac": 0.0626131221652031, "expert_frac_max": 0.06273786723613739, "expert_frac_min": 0.0622267946600914, "expert_frac_std": 0.00015162151248659939, "grad_norm": 0.39026713371276855, "learning_rate": 0.00013084517322359637, "loss": 3.4059, "router_aux_loss": 2.000299784094095, "router_entropy": 2.056280114464462, "router_entropy_norm": 0.7416462809541173, "step": 4550 }, { "epoch": 0.5700123915737298, "expert_0_frac": 0.06235774978995323, "expert_10_frac": 0.06246906518936157, "expert_11_frac": 0.06250017136335373, "expert_12_frac": 0.062340375036001205, "expert_13_frac": 0.06265813112258911, "expert_14_frac": 0.062344953417778015, "expert_15_frac": 0.06246919929981232, "expert_1_frac": 0.0626433864235878, "expert_2_frac": 0.0623575858771801, "expert_3_frac": 0.06266095489263535, "expert_4_frac": 0.06238768249750137, "expert_5_frac": 0.06245439127087593, "expert_6_frac": 0.06249968335032463, "expert_7_frac": 0.0627267062664032, "expert_8_frac": 0.06241398677229881, "expert_9_frac": 0.06271599978208542, "expert_frac_max": 0.0627267062664032, "expert_frac_min": 0.062340375036001205, "expert_frac_std": 0.00013749160279985517, "grad_norm": 0.3813641369342804, "learning_rate": 0.0001278014028914802, "loss": 3.4044, "router_aux_loss": 2.000373487025499, "router_entropy": 2.0545436954001586, "router_entropy_norm": 0.7410200001608734, "step": 4600 }, { "epoch": 0.5762081784386617, "expert_0_frac": 0.06234966963529587, "expert_10_frac": 0.062438640743494034, "expert_11_frac": 0.06256365031003952, "expert_12_frac": 0.062073301523923874, "expert_13_frac": 0.06242337077856064, "expert_14_frac": 0.062409333884716034, "expert_15_frac": 0.06248563155531883, "expert_1_frac": 0.06252411752939224, "expert_2_frac": 0.06244285777211189, "expert_3_frac": 0.06261327862739563, "expert_4_frac": 0.06238767132163048, "expert_5_frac": 0.06257081776857376, "expert_6_frac": 0.06280014663934708, "expert_7_frac": 0.06257054209709167, "expert_8_frac": 0.06250441074371338, "expert_9_frac": 0.06284257769584656, "expert_frac_max": 0.06284257769584656, "expert_frac_min": 0.062073301523923874, "expert_frac_std": 0.0001776777789928019, "grad_norm": 0.37938594818115234, "learning_rate": 0.00012476695005429975, "loss": 3.3886, "router_aux_loss": 2.0002758315205575, "router_entropy": 2.0531820828095078, "router_entropy_norm": 0.7405289022278375, "step": 4650 }, { "epoch": 0.5824039653035935, "expert_0_frac": 0.06221843138337135, "expert_10_frac": 0.06256091594696045, "expert_11_frac": 0.0626656711101532, "expert_12_frac": 0.06217745691537857, "expert_13_frac": 0.06265641748905182, "expert_14_frac": 0.062169160693883896, "expert_15_frac": 0.06261997669935226, "expert_1_frac": 0.06251893937587738, "expert_2_frac": 0.06247605010867119, "expert_3_frac": 0.0625799298286438, "expert_4_frac": 0.06254834681749344, "expert_5_frac": 0.06241624429821968, "expert_6_frac": 0.06244943290948868, "expert_7_frac": 0.06266841292381287, "expert_8_frac": 0.06259527057409286, "expert_9_frac": 0.06267937272787094, "expert_frac_max": 0.06267937272787094, "expert_frac_min": 0.062169160693883896, "expert_frac_std": 0.00017351898713968694, "grad_norm": 0.36018115282058716, "learning_rate": 0.0001217430883735755, "loss": 3.3863, "router_aux_loss": 2.00024845123291, "router_entropy": 2.0513762359196943, "router_entropy_norm": 0.7398775806397029, "step": 4700 }, { "epoch": 0.5885997521685254, "expert_0_frac": 0.06233708932995796, "expert_10_frac": 0.06242017820477486, "expert_11_frac": 0.06262047588825226, "expert_12_frac": 0.06229627877473831, "expert_13_frac": 0.06242483854293823, "expert_14_frac": 0.062415316700935364, "expert_15_frac": 0.06253886222839355, "expert_1_frac": 0.06252031028270721, "expert_2_frac": 0.06237517297267914, "expert_3_frac": 0.06263025850057602, "expert_4_frac": 0.06249498575925827, "expert_5_frac": 0.06240921840071678, "expert_6_frac": 0.06260804086923599, "expert_7_frac": 0.06275253742933273, "expert_8_frac": 0.06233108788728714, "expert_9_frac": 0.06282538175582886, "expert_frac_max": 0.06282538175582886, "expert_frac_min": 0.06229627877473831, "expert_frac_std": 0.00015369657194241881, "grad_norm": 0.3805563449859619, "learning_rate": 0.00011873108706536448, "loss": 3.3834, "router_aux_loss": 2.0003303153812886, "router_entropy": 2.049379334114492, "router_entropy_norm": 0.739157350556826, "step": 4750 }, { "epoch": 0.5947955390334573, "expert_0_frac": 0.0621938481926918, "expert_10_frac": 0.06241048872470856, "expert_11_frac": 0.06265752017498016, "expert_12_frac": 0.062160950154066086, "expert_13_frac": 0.06252051889896393, "expert_14_frac": 0.06239565089344978, "expert_15_frac": 0.06259883940219879, "expert_1_frac": 0.06257259845733643, "expert_2_frac": 0.06241338327527046, "expert_3_frac": 0.06267257779836655, "expert_4_frac": 0.06234891340136528, "expert_5_frac": 0.06232818588614464, "expert_6_frac": 0.06263818591833115, "expert_7_frac": 0.06266766041517258, "expert_8_frac": 0.06272201240062714, "expert_9_frac": 0.06269869208335876, "expert_frac_max": 0.06272201240062714, "expert_frac_min": 0.062160950154066086, "expert_frac_std": 0.00018137702136300504, "grad_norm": 0.37568414211273193, "learning_rate": 0.00011573221036752828, "loss": 3.397, "router_aux_loss": 2.000270184874535, "router_entropy": 2.046731663371126, "router_entropy_norm": 0.7382024051939856, "step": 4800 }, { "epoch": 0.6, "eval_loss": 2.7643399238586426, "eval_runtime": 34.8236, "eval_samples_per_second": 49.622, "eval_steps_per_second": 6.203, "expert_0_frac": 0.06227129325270653, "expert_10_frac": 0.06248059123754501, "expert_11_frac": 0.06245836243033409, "expert_12_frac": 0.062425948679447174, "expert_13_frac": 0.062481872737407684, "expert_14_frac": 0.06234771013259888, "expert_15_frac": 0.062448468059301376, "expert_1_frac": 0.06265126913785934, "expert_2_frac": 0.062485091388225555, "expert_3_frac": 0.06254459172487259, "expert_4_frac": 0.06255730241537094, "expert_5_frac": 0.06245243921875954, "expert_6_frac": 0.06256099790334702, "expert_7_frac": 0.06259247660636902, "expert_8_frac": 0.06239942088723183, "expert_9_frac": 0.06284219026565552, "expert_frac_max": 0.06284219026565552, "expert_frac_min": 0.06227129325270653, "expert_frac_std": 0.00013069015403743833, "router_aux_loss": 2.0002922967479035, "router_entropy": 2.0467925748757057, "router_entropy_norm": 0.7382243743753832, "step": 4842 } ], "logging_steps": 50, "max_steps": 8070, "num_input_tokens_seen": 0, "num_train_epochs": 9223372036854775807, "save_steps": 1614, "stateful_callbacks": { "EarlyStoppingCallback": { "args": { "early_stopping_patience": 3, "early_stopping_threshold": 0.0 }, "attributes": { "early_stopping_patience_counter": 0 } }, "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 7.141687525260657e+17, "train_batch_size": 8, "trial_name": null, "trial_params": null }