6047 lines
173 KiB
JSON
6047 lines
173 KiB
JSON
{
|
|
"best_global_step": 11500,
|
|
"best_metric": 0.8230946660041809,
|
|
"best_model_checkpoint": "/workspace/tanglish/checkpoints/qwen_lora_v1/lora/checkpoint-11500",
|
|
"epoch": 2.383419689119171,
|
|
"eval_steps": 500,
|
|
"global_step": 11500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 1.4381825268268584,
|
|
"epoch": 0.004145077720207254,
|
|
"grad_norm": 2.1717000007629395,
|
|
"learning_rate": 5.248618784530387e-06,
|
|
"loss": 4.0762,
|
|
"mean_token_accuracy": 0.4441531464457512,
|
|
"num_tokens": 51480.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 1.5037598952651023,
|
|
"epoch": 0.008290155440414507,
|
|
"grad_norm": 2.724255323410034,
|
|
"learning_rate": 1.0773480662983426e-05,
|
|
"loss": 3.8127,
|
|
"mean_token_accuracy": 0.4525611069053411,
|
|
"num_tokens": 103460.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 1.901321442425251,
|
|
"epoch": 0.012435233160621761,
|
|
"grad_norm": 1.1939035654067993,
|
|
"learning_rate": 1.6298342541436465e-05,
|
|
"loss": 3.1292,
|
|
"mean_token_accuracy": 0.48335910513997077,
|
|
"num_tokens": 154920.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 2.358680176734924,
|
|
"epoch": 0.016580310880829015,
|
|
"grad_norm": 0.7893019318580627,
|
|
"learning_rate": 2.1823204419889505e-05,
|
|
"loss": 2.3084,
|
|
"mean_token_accuracy": 0.554056815057993,
|
|
"num_tokens": 207786.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 1.830135741829872,
|
|
"epoch": 0.02072538860103627,
|
|
"grad_norm": 0.5736128091812134,
|
|
"learning_rate": 2.7348066298342545e-05,
|
|
"loss": 1.708,
|
|
"mean_token_accuracy": 0.6866068348288537,
|
|
"num_tokens": 256847.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 1.4562795095145702,
|
|
"epoch": 0.024870466321243522,
|
|
"grad_norm": 0.9956451058387756,
|
|
"learning_rate": 3.287292817679558e-05,
|
|
"loss": 1.4132,
|
|
"mean_token_accuracy": 0.7327493987977505,
|
|
"num_tokens": 309233.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 1.4073016680777073,
|
|
"epoch": 0.029015544041450778,
|
|
"grad_norm": 0.8126519918441772,
|
|
"learning_rate": 3.8397790055248625e-05,
|
|
"loss": 1.3879,
|
|
"mean_token_accuracy": 0.7353364005684853,
|
|
"num_tokens": 361206.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 1.28325657248497,
|
|
"epoch": 0.03316062176165803,
|
|
"grad_norm": 0.6408252120018005,
|
|
"learning_rate": 4.392265193370166e-05,
|
|
"loss": 1.2517,
|
|
"mean_token_accuracy": 0.7543429806828499,
|
|
"num_tokens": 412093.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 1.219002117216587,
|
|
"epoch": 0.03730569948186528,
|
|
"grad_norm": 0.6377286911010742,
|
|
"learning_rate": 4.94475138121547e-05,
|
|
"loss": 1.2238,
|
|
"mean_token_accuracy": 0.7544910028576851,
|
|
"num_tokens": 463753.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 1.2931843183934688,
|
|
"epoch": 0.04145077720207254,
|
|
"grad_norm": 0.7240749001502991,
|
|
"learning_rate": 5.4972375690607735e-05,
|
|
"loss": 1.3026,
|
|
"mean_token_accuracy": 0.7400020457804203,
|
|
"num_tokens": 515356.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 1.2196356683969498,
|
|
"epoch": 0.04559585492227979,
|
|
"grad_norm": 0.7647127509117126,
|
|
"learning_rate": 6.049723756906077e-05,
|
|
"loss": 1.2411,
|
|
"mean_token_accuracy": 0.7544264450669289,
|
|
"num_tokens": 565236.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 1.171409723162651,
|
|
"epoch": 0.049740932642487044,
|
|
"grad_norm": 0.7827390432357788,
|
|
"learning_rate": 6.602209944751382e-05,
|
|
"loss": 1.1776,
|
|
"mean_token_accuracy": 0.7626889519393444,
|
|
"num_tokens": 615515.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 1.1406699948012828,
|
|
"epoch": 0.0538860103626943,
|
|
"grad_norm": 1.0029950141906738,
|
|
"learning_rate": 7.154696132596685e-05,
|
|
"loss": 1.1546,
|
|
"mean_token_accuracy": 0.766391609609127,
|
|
"num_tokens": 666062.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 1.1219736263155937,
|
|
"epoch": 0.058031088082901555,
|
|
"grad_norm": 1.0503102540969849,
|
|
"learning_rate": 7.70718232044199e-05,
|
|
"loss": 1.1352,
|
|
"mean_token_accuracy": 0.7641930885612964,
|
|
"num_tokens": 719494.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 1.1847783662378788,
|
|
"epoch": 0.06217616580310881,
|
|
"grad_norm": 0.8290141820907593,
|
|
"learning_rate": 8.259668508287292e-05,
|
|
"loss": 1.202,
|
|
"mean_token_accuracy": 0.7585213594138622,
|
|
"num_tokens": 772839.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 1.1723797246813774,
|
|
"epoch": 0.06632124352331606,
|
|
"grad_norm": 0.877815842628479,
|
|
"learning_rate": 8.812154696132597e-05,
|
|
"loss": 1.1825,
|
|
"mean_token_accuracy": 0.7599225588142872,
|
|
"num_tokens": 825189.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 1.1410918802022934,
|
|
"epoch": 0.07046632124352331,
|
|
"grad_norm": 0.7032957673072815,
|
|
"learning_rate": 9.364640883977901e-05,
|
|
"loss": 1.1333,
|
|
"mean_token_accuracy": 0.7682646617293358,
|
|
"num_tokens": 877294.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 1.1239484950900078,
|
|
"epoch": 0.07461139896373056,
|
|
"grad_norm": 0.8480610847473145,
|
|
"learning_rate": 9.917127071823204e-05,
|
|
"loss": 1.1536,
|
|
"mean_token_accuracy": 0.7669501408934594,
|
|
"num_tokens": 927234.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 1.110478015244007,
|
|
"epoch": 0.07875647668393783,
|
|
"grad_norm": 0.7844213843345642,
|
|
"learning_rate": 0.0001046961325966851,
|
|
"loss": 1.1255,
|
|
"mean_token_accuracy": 0.7683746472001076,
|
|
"num_tokens": 980398.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 1.1618236370384694,
|
|
"epoch": 0.08290155440414508,
|
|
"grad_norm": 0.7246690988540649,
|
|
"learning_rate": 0.00011022099447513811,
|
|
"loss": 1.1729,
|
|
"mean_token_accuracy": 0.7629252031445504,
|
|
"num_tokens": 1030335.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 1.1364129185676575,
|
|
"epoch": 0.08704663212435233,
|
|
"grad_norm": 0.7912014722824097,
|
|
"learning_rate": 0.00011574585635359116,
|
|
"loss": 1.152,
|
|
"mean_token_accuracy": 0.7651066802442074,
|
|
"num_tokens": 1083694.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 1.0954272992908956,
|
|
"epoch": 0.09119170984455958,
|
|
"grad_norm": 0.7310007214546204,
|
|
"learning_rate": 0.00012127071823204422,
|
|
"loss": 1.1039,
|
|
"mean_token_accuracy": 0.7700782015919685,
|
|
"num_tokens": 1134346.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 1.1179637335240842,
|
|
"epoch": 0.09533678756476684,
|
|
"grad_norm": 0.7415347099304199,
|
|
"learning_rate": 0.00012679558011049723,
|
|
"loss": 1.1267,
|
|
"mean_token_accuracy": 0.7677439145743847,
|
|
"num_tokens": 1188485.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 1.0841590076684953,
|
|
"epoch": 0.09948186528497409,
|
|
"grad_norm": 0.8192847371101379,
|
|
"learning_rate": 0.00013232044198895027,
|
|
"loss": 1.095,
|
|
"mean_token_accuracy": 0.7726523540914059,
|
|
"num_tokens": 1239371.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 1.0643972344696522,
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 0.8887308239936829,
|
|
"learning_rate": 0.00013784530386740332,
|
|
"loss": 1.0762,
|
|
"mean_token_accuracy": 0.7769059598445892,
|
|
"num_tokens": 1290491.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"eval_entropy": 1.095322513439524,
|
|
"eval_loss": 1.1184215545654297,
|
|
"eval_mean_token_accuracy": 0.7680243507144958,
|
|
"eval_num_tokens": 1290491.0,
|
|
"eval_runtime": 198.0973,
|
|
"eval_samples_per_second": 20.51,
|
|
"eval_steps_per_second": 2.564,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 1.0820845410227775,
|
|
"epoch": 0.1077720207253886,
|
|
"grad_norm": 0.7347564697265625,
|
|
"learning_rate": 0.00014337016574585636,
|
|
"loss": 1.0979,
|
|
"mean_token_accuracy": 0.7743508994579316,
|
|
"num_tokens": 1341205.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"entropy": 1.1001912213861942,
|
|
"epoch": 0.11191709844559586,
|
|
"grad_norm": 0.690479040145874,
|
|
"learning_rate": 0.0001488950276243094,
|
|
"loss": 1.1105,
|
|
"mean_token_accuracy": 0.7694470949470997,
|
|
"num_tokens": 1394461.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"entropy": 1.0999469496309757,
|
|
"epoch": 0.11606217616580311,
|
|
"grad_norm": 0.6917790174484253,
|
|
"learning_rate": 0.00015441988950276245,
|
|
"loss": 1.117,
|
|
"mean_token_accuracy": 0.7682196438312531,
|
|
"num_tokens": 1447828.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"entropy": 1.0735741600394249,
|
|
"epoch": 0.12020725388601036,
|
|
"grad_norm": 0.833886444568634,
|
|
"learning_rate": 0.00015994475138121547,
|
|
"loss": 1.086,
|
|
"mean_token_accuracy": 0.7741858348250389,
|
|
"num_tokens": 1498479.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"entropy": 1.0962664999067784,
|
|
"epoch": 0.12435233160621761,
|
|
"grad_norm": 0.6647179126739502,
|
|
"learning_rate": 0.0001654696132596685,
|
|
"loss": 1.0958,
|
|
"mean_token_accuracy": 0.7741696208715438,
|
|
"num_tokens": 1548137.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"entropy": 1.0694029107689857,
|
|
"epoch": 0.12849740932642487,
|
|
"grad_norm": 0.6861886382102966,
|
|
"learning_rate": 0.00017099447513812155,
|
|
"loss": 1.0829,
|
|
"mean_token_accuracy": 0.7720654770731926,
|
|
"num_tokens": 1599638.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"entropy": 1.075825183838606,
|
|
"epoch": 0.13264248704663212,
|
|
"grad_norm": 0.7000752687454224,
|
|
"learning_rate": 0.0001765193370165746,
|
|
"loss": 1.0825,
|
|
"mean_token_accuracy": 0.7762658178806305,
|
|
"num_tokens": 1650323.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"entropy": 1.083068310469389,
|
|
"epoch": 0.13678756476683937,
|
|
"grad_norm": 0.7568516731262207,
|
|
"learning_rate": 0.00018204419889502764,
|
|
"loss": 1.0918,
|
|
"mean_token_accuracy": 0.7720131345093251,
|
|
"num_tokens": 1701215.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"entropy": 1.0660202227532865,
|
|
"epoch": 0.14093264248704662,
|
|
"grad_norm": 0.8106701374053955,
|
|
"learning_rate": 0.00018756906077348066,
|
|
"loss": 1.073,
|
|
"mean_token_accuracy": 0.7741113729774952,
|
|
"num_tokens": 1751275.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"entropy": 1.00438876375556,
|
|
"epoch": 0.14507772020725387,
|
|
"grad_norm": 0.6197234988212585,
|
|
"learning_rate": 0.0001930939226519337,
|
|
"loss": 1.0234,
|
|
"mean_token_accuracy": 0.7818360298871994,
|
|
"num_tokens": 1803256.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"entropy": 1.1083124563097955,
|
|
"epoch": 0.14922279792746113,
|
|
"grad_norm": 0.6046066284179688,
|
|
"learning_rate": 0.00019861878453038677,
|
|
"loss": 1.122,
|
|
"mean_token_accuracy": 0.7704763405025006,
|
|
"num_tokens": 1855372.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"entropy": 1.097743584215641,
|
|
"epoch": 0.1533678756476684,
|
|
"grad_norm": 0.6518492102622986,
|
|
"learning_rate": 0.00019999941280374287,
|
|
"loss": 1.1116,
|
|
"mean_token_accuracy": 0.7682732604444027,
|
|
"num_tokens": 1906015.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"entropy": 1.1098480015993117,
|
|
"epoch": 0.15751295336787566,
|
|
"grad_norm": 0.6173190474510193,
|
|
"learning_rate": 0.00019999680305650554,
|
|
"loss": 1.1173,
|
|
"mean_token_accuracy": 0.7627138867974281,
|
|
"num_tokens": 1959638.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"entropy": 1.0928526185452938,
|
|
"epoch": 0.1616580310880829,
|
|
"grad_norm": 0.6099253296852112,
|
|
"learning_rate": 0.00019999210556868917,
|
|
"loss": 1.1156,
|
|
"mean_token_accuracy": 0.7711292572319508,
|
|
"num_tokens": 2010476.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"entropy": 1.0808037273585795,
|
|
"epoch": 0.16580310880829016,
|
|
"grad_norm": 0.6099389791488647,
|
|
"learning_rate": 0.0001999853204383683,
|
|
"loss": 1.087,
|
|
"mean_token_accuracy": 0.7749647930264473,
|
|
"num_tokens": 2060494.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"entropy": 1.048035841435194,
|
|
"epoch": 0.1699481865284974,
|
|
"grad_norm": 0.5170192122459412,
|
|
"learning_rate": 0.0001999764478072033,
|
|
"loss": 1.0642,
|
|
"mean_token_accuracy": 0.7727734588086606,
|
|
"num_tokens": 2113260.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"entropy": 1.0665065109729768,
|
|
"epoch": 0.17409326424870467,
|
|
"grad_norm": 0.6461181640625,
|
|
"learning_rate": 0.00019996548786043768,
|
|
"loss": 1.0795,
|
|
"mean_token_accuracy": 0.777741952240467,
|
|
"num_tokens": 2164157.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"entropy": 1.0095063984394073,
|
|
"epoch": 0.17823834196891192,
|
|
"grad_norm": 0.5376838445663452,
|
|
"learning_rate": 0.00019995244082689396,
|
|
"loss": 1.0238,
|
|
"mean_token_accuracy": 0.7842500291764736,
|
|
"num_tokens": 2215501.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"entropy": 1.0482122428715228,
|
|
"epoch": 0.18238341968911917,
|
|
"grad_norm": 0.5362586379051208,
|
|
"learning_rate": 0.00019993730697896912,
|
|
"loss": 1.0567,
|
|
"mean_token_accuracy": 0.7757732585072518,
|
|
"num_tokens": 2265908.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"entropy": 1.0375815689563752,
|
|
"epoch": 0.18652849740932642,
|
|
"grad_norm": 0.6826100945472717,
|
|
"learning_rate": 0.0001999200866326287,
|
|
"loss": 1.055,
|
|
"mean_token_accuracy": 0.777643445879221,
|
|
"num_tokens": 2316578.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"entropy": 1.0592326551675797,
|
|
"epoch": 0.19067357512953367,
|
|
"grad_norm": 0.4966115355491638,
|
|
"learning_rate": 0.00019990078014740036,
|
|
"loss": 1.0682,
|
|
"mean_token_accuracy": 0.7757245123386383,
|
|
"num_tokens": 2367491.0,
|
|
"step": 920
|
|
},
|
|
{
|
|
"entropy": 1.0522546865046025,
|
|
"epoch": 0.19481865284974093,
|
|
"grad_norm": 0.5403271317481995,
|
|
"learning_rate": 0.00019987938792636632,
|
|
"loss": 1.0597,
|
|
"mean_token_accuracy": 0.7824762858450413,
|
|
"num_tokens": 2418531.0,
|
|
"step": 940
|
|
},
|
|
{
|
|
"entropy": 1.0112147249281407,
|
|
"epoch": 0.19896373056994818,
|
|
"grad_norm": 0.591804027557373,
|
|
"learning_rate": 0.00019985591041615496,
|
|
"loss": 1.0179,
|
|
"mean_token_accuracy": 0.7824166387319564,
|
|
"num_tokens": 2468312.0,
|
|
"step": 960
|
|
},
|
|
{
|
|
"entropy": 1.0457121133804321,
|
|
"epoch": 0.20310880829015543,
|
|
"grad_norm": 0.4792396128177643,
|
|
"learning_rate": 0.00019983034810693137,
|
|
"loss": 1.0567,
|
|
"mean_token_accuracy": 0.7786681324243545,
|
|
"num_tokens": 2518124.0,
|
|
"step": 980
|
|
},
|
|
{
|
|
"entropy": 1.0062039978802204,
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 0.6134971976280212,
|
|
"learning_rate": 0.00019980270153238742,
|
|
"loss": 1.0284,
|
|
"mean_token_accuracy": 0.7829495541751385,
|
|
"num_tokens": 2569706.0,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"eval_entropy": 1.0445965941496722,
|
|
"eval_loss": 1.059064507484436,
|
|
"eval_mean_token_accuracy": 0.7763740899055962,
|
|
"eval_num_tokens": 2569706.0,
|
|
"eval_runtime": 197.7332,
|
|
"eval_samples_per_second": 20.548,
|
|
"eval_steps_per_second": 2.569,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"entropy": 1.0440333500504493,
|
|
"epoch": 0.21139896373056996,
|
|
"grad_norm": 0.6377447843551636,
|
|
"learning_rate": 0.0001997729712697303,
|
|
"loss": 1.0571,
|
|
"mean_token_accuracy": 0.7803341060876846,
|
|
"num_tokens": 2618883.0,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"entropy": 1.021172496676445,
|
|
"epoch": 0.2155440414507772,
|
|
"grad_norm": 0.5175086855888367,
|
|
"learning_rate": 0.0001997411579396706,
|
|
"loss": 1.0292,
|
|
"mean_token_accuracy": 0.7807957075536252,
|
|
"num_tokens": 2670732.0,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"entropy": 1.0275346338748932,
|
|
"epoch": 0.21968911917098446,
|
|
"grad_norm": 0.5990492701530457,
|
|
"learning_rate": 0.00019970726220640929,
|
|
"loss": 1.0447,
|
|
"mean_token_accuracy": 0.77860803976655,
|
|
"num_tokens": 2722930.0,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"entropy": 1.0109625704586507,
|
|
"epoch": 0.22383419689119172,
|
|
"grad_norm": 0.4519039988517761,
|
|
"learning_rate": 0.00019967128477762404,
|
|
"loss": 1.0286,
|
|
"mean_token_accuracy": 0.7781259723007679,
|
|
"num_tokens": 2775645.0,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"entropy": 1.024503505975008,
|
|
"epoch": 0.22797927461139897,
|
|
"grad_norm": 0.5599638819694519,
|
|
"learning_rate": 0.00019963322640445424,
|
|
"loss": 1.0358,
|
|
"mean_token_accuracy": 0.7802064262330533,
|
|
"num_tokens": 2829031.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"entropy": 1.000456291437149,
|
|
"epoch": 0.23212435233160622,
|
|
"grad_norm": 0.42756351828575134,
|
|
"learning_rate": 0.0001995930878814854,
|
|
"loss": 1.0071,
|
|
"mean_token_accuracy": 0.7876958273351192,
|
|
"num_tokens": 2882439.0,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"entropy": 1.0316657554358244,
|
|
"epoch": 0.23626943005181347,
|
|
"grad_norm": 0.4877125322818756,
|
|
"learning_rate": 0.0001995508700467325,
|
|
"loss": 1.0422,
|
|
"mean_token_accuracy": 0.7804410368204117,
|
|
"num_tokens": 2935155.0,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"entropy": 1.0400867745280267,
|
|
"epoch": 0.24041450777202072,
|
|
"grad_norm": 0.501750111579895,
|
|
"learning_rate": 0.00019950657378162262,
|
|
"loss": 1.06,
|
|
"mean_token_accuracy": 0.7792659178376198,
|
|
"num_tokens": 2986224.0,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"entropy": 1.0030486956238747,
|
|
"epoch": 0.24455958549222798,
|
|
"grad_norm": 0.4440039396286011,
|
|
"learning_rate": 0.00019946020001097644,
|
|
"loss": 1.0081,
|
|
"mean_token_accuracy": 0.7861276485025883,
|
|
"num_tokens": 3037391.0,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"entropy": 1.0251952454447746,
|
|
"epoch": 0.24870466321243523,
|
|
"grad_norm": 0.6198555827140808,
|
|
"learning_rate": 0.00019941174970298888,
|
|
"loss": 1.036,
|
|
"mean_token_accuracy": 0.7791207812726497,
|
|
"num_tokens": 3087069.0,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"entropy": 1.0355947606265545,
|
|
"epoch": 0.2528497409326425,
|
|
"grad_norm": 0.43304187059402466,
|
|
"learning_rate": 0.0001993612238692091,
|
|
"loss": 1.0457,
|
|
"mean_token_accuracy": 0.7767570614814758,
|
|
"num_tokens": 3140164.0,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"entropy": 1.0067585781216621,
|
|
"epoch": 0.25699481865284973,
|
|
"grad_norm": 0.5553335547447205,
|
|
"learning_rate": 0.0001993086235645191,
|
|
"loss": 1.0238,
|
|
"mean_token_accuracy": 0.7811847984790802,
|
|
"num_tokens": 3192879.0,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"entropy": 1.0447653152048588,
|
|
"epoch": 0.261139896373057,
|
|
"grad_norm": 0.5924249291419983,
|
|
"learning_rate": 0.0001992539498871119,
|
|
"loss": 1.049,
|
|
"mean_token_accuracy": 0.7775058671832085,
|
|
"num_tokens": 3244981.0,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"entropy": 1.0076527014374732,
|
|
"epoch": 0.26528497409326424,
|
|
"grad_norm": 0.5087696313858032,
|
|
"learning_rate": 0.00019919720397846855,
|
|
"loss": 1.018,
|
|
"mean_token_accuracy": 0.784369270503521,
|
|
"num_tokens": 3293733.0,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"entropy": 1.0454526104032993,
|
|
"epoch": 0.2694300518134715,
|
|
"grad_norm": 0.5536513924598694,
|
|
"learning_rate": 0.00019913838702333432,
|
|
"loss": 1.0648,
|
|
"mean_token_accuracy": 0.7793882772326469,
|
|
"num_tokens": 3343268.0,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"entropy": 1.0259532511234284,
|
|
"epoch": 0.27357512953367874,
|
|
"grad_norm": 0.5604416728019714,
|
|
"learning_rate": 0.00019907750024969386,
|
|
"loss": 1.0366,
|
|
"mean_token_accuracy": 0.779184664785862,
|
|
"num_tokens": 3395934.0,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"entropy": 1.0157927133142948,
|
|
"epoch": 0.277720207253886,
|
|
"grad_norm": 0.45751333236694336,
|
|
"learning_rate": 0.00019901454492874572,
|
|
"loss": 1.0379,
|
|
"mean_token_accuracy": 0.7799046277999878,
|
|
"num_tokens": 3448448.0,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"entropy": 1.013536487519741,
|
|
"epoch": 0.28186528497409324,
|
|
"grad_norm": 0.4128531813621521,
|
|
"learning_rate": 0.00019894952237487566,
|
|
"loss": 1.0201,
|
|
"mean_token_accuracy": 0.7843268230557442,
|
|
"num_tokens": 3499346.0,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"entropy": 1.0443965464830398,
|
|
"epoch": 0.2860103626943005,
|
|
"grad_norm": 0.49815455079078674,
|
|
"learning_rate": 0.0001988824339456294,
|
|
"loss": 1.0593,
|
|
"mean_token_accuracy": 0.7785378187894821,
|
|
"num_tokens": 3551298.0,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"entropy": 1.011622816324234,
|
|
"epoch": 0.29015544041450775,
|
|
"grad_norm": 0.46742376685142517,
|
|
"learning_rate": 0.000198813281041684,
|
|
"loss": 1.0193,
|
|
"mean_token_accuracy": 0.7825383462011815,
|
|
"num_tokens": 3602656.0,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"entropy": 0.9844559028744697,
|
|
"epoch": 0.29430051813471503,
|
|
"grad_norm": 0.40691128373146057,
|
|
"learning_rate": 0.0001987420651068189,
|
|
"loss": 1.0042,
|
|
"mean_token_accuracy": 0.7844808354973793,
|
|
"num_tokens": 3654678.0,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"entropy": 1.0059320218861103,
|
|
"epoch": 0.29844559585492225,
|
|
"grad_norm": 0.42339223623275757,
|
|
"learning_rate": 0.00019866878762788562,
|
|
"loss": 1.0142,
|
|
"mean_token_accuracy": 0.7823195405304432,
|
|
"num_tokens": 3705949.0,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"entropy": 0.9958507776260376,
|
|
"epoch": 0.30259067357512953,
|
|
"grad_norm": 0.48249927163124084,
|
|
"learning_rate": 0.00019859345013477665,
|
|
"loss": 1.0121,
|
|
"mean_token_accuracy": 0.7889884993433952,
|
|
"num_tokens": 3756251.0,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"entropy": 0.9966832913458348,
|
|
"epoch": 0.3067357512953368,
|
|
"grad_norm": 0.3815440237522125,
|
|
"learning_rate": 0.00019851605420039377,
|
|
"loss": 1.0226,
|
|
"mean_token_accuracy": 0.7830308295786381,
|
|
"num_tokens": 3808938.0,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"entropy": 0.9885330185294151,
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 0.4722138047218323,
|
|
"learning_rate": 0.00019843660144061497,
|
|
"loss": 0.9955,
|
|
"mean_token_accuracy": 0.7874660678207874,
|
|
"num_tokens": 3858248.0,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"eval_entropy": 1.032557885595194,
|
|
"eval_loss": 1.0242446660995483,
|
|
"eval_mean_token_accuracy": 0.7811775283785317,
|
|
"eval_num_tokens": 3858248.0,
|
|
"eval_runtime": 198.2363,
|
|
"eval_samples_per_second": 20.496,
|
|
"eval_steps_per_second": 2.563,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"entropy": 1.0101246651262046,
|
|
"epoch": 0.3150259067357513,
|
|
"grad_norm": 0.42561495304107666,
|
|
"learning_rate": 0.00019835509351426073,
|
|
"loss": 1.0177,
|
|
"mean_token_accuracy": 0.7840446695685387,
|
|
"num_tokens": 3909740.0,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"entropy": 0.9913592956960201,
|
|
"epoch": 0.31917098445595854,
|
|
"grad_norm": 0.4773913025856018,
|
|
"learning_rate": 0.00019827153212305954,
|
|
"loss": 1.0043,
|
|
"mean_token_accuracy": 0.7856849782168865,
|
|
"num_tokens": 3958454.0,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"entropy": 1.0170710057020187,
|
|
"epoch": 0.3233160621761658,
|
|
"grad_norm": 0.4388096034526825,
|
|
"learning_rate": 0.00019818591901161225,
|
|
"loss": 1.0284,
|
|
"mean_token_accuracy": 0.7777703814208508,
|
|
"num_tokens": 4010172.0,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"entropy": 1.0317365549504758,
|
|
"epoch": 0.32746113989637304,
|
|
"grad_norm": 0.4321315884590149,
|
|
"learning_rate": 0.00019809825596735567,
|
|
"loss": 1.0507,
|
|
"mean_token_accuracy": 0.782105403393507,
|
|
"num_tokens": 4061342.0,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"entropy": 1.0285033784806727,
|
|
"epoch": 0.3316062176165803,
|
|
"grad_norm": 0.4316249489784241,
|
|
"learning_rate": 0.00019800854482052526,
|
|
"loss": 1.053,
|
|
"mean_token_accuracy": 0.7812598653137683,
|
|
"num_tokens": 4112308.0,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"entropy": 0.959014667570591,
|
|
"epoch": 0.33575129533678755,
|
|
"grad_norm": 0.4981176555156708,
|
|
"learning_rate": 0.0001979167874441169,
|
|
"loss": 0.9713,
|
|
"mean_token_accuracy": 0.7900767177343369,
|
|
"num_tokens": 4163721.0,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"entropy": 0.9863104440271855,
|
|
"epoch": 0.3398963730569948,
|
|
"grad_norm": 0.5106677412986755,
|
|
"learning_rate": 0.00019782298575384783,
|
|
"loss": 0.9888,
|
|
"mean_token_accuracy": 0.786780058592558,
|
|
"num_tokens": 4214428.0,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"entropy": 0.9868629761040211,
|
|
"epoch": 0.34404145077720205,
|
|
"grad_norm": 0.4558473527431488,
|
|
"learning_rate": 0.00019772714170811662,
|
|
"loss": 1.0034,
|
|
"mean_token_accuracy": 0.7846312940120697,
|
|
"num_tokens": 4267755.0,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"entropy": 1.0307603180408478,
|
|
"epoch": 0.34818652849740933,
|
|
"grad_norm": 0.4738333225250244,
|
|
"learning_rate": 0.00019762925730796224,
|
|
"loss": 1.0414,
|
|
"mean_token_accuracy": 0.7802501529455185,
|
|
"num_tokens": 4317809.0,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"entropy": 1.0127912051975727,
|
|
"epoch": 0.35233160621761656,
|
|
"grad_norm": 0.48111966252326965,
|
|
"learning_rate": 0.0001975293345970224,
|
|
"loss": 1.0147,
|
|
"mean_token_accuracy": 0.7798892751336097,
|
|
"num_tokens": 4370912.0,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"entropy": 0.9599951460957528,
|
|
"epoch": 0.35647668393782384,
|
|
"grad_norm": 0.4724443256855011,
|
|
"learning_rate": 0.00019742737566149078,
|
|
"loss": 0.9585,
|
|
"mean_token_accuracy": 0.7909531377255916,
|
|
"num_tokens": 4423603.0,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"entropy": 1.0278802808374166,
|
|
"epoch": 0.3606217616580311,
|
|
"grad_norm": 0.5354254245758057,
|
|
"learning_rate": 0.0001973233826300735,
|
|
"loss": 1.035,
|
|
"mean_token_accuracy": 0.7811824880540371,
|
|
"num_tokens": 4476943.0,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"entropy": 0.9914717882871628,
|
|
"epoch": 0.36476683937823834,
|
|
"grad_norm": 0.47376251220703125,
|
|
"learning_rate": 0.00019721735767394468,
|
|
"loss": 1.0099,
|
|
"mean_token_accuracy": 0.781830509006977,
|
|
"num_tokens": 4527823.0,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"entropy": 1.020705473423004,
|
|
"epoch": 0.3689119170984456,
|
|
"grad_norm": 0.4668932259082794,
|
|
"learning_rate": 0.00019710930300670108,
|
|
"loss": 1.0274,
|
|
"mean_token_accuracy": 0.7800539046525955,
|
|
"num_tokens": 4581082.0,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"entropy": 1.0137963436543942,
|
|
"epoch": 0.37305699481865284,
|
|
"grad_norm": 0.4402451515197754,
|
|
"learning_rate": 0.00019699922088431605,
|
|
"loss": 1.0194,
|
|
"mean_token_accuracy": 0.7841047510504723,
|
|
"num_tokens": 4630834.0,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"entropy": 0.9691961273550987,
|
|
"epoch": 0.3772020725388601,
|
|
"grad_norm": 0.4267026484012604,
|
|
"learning_rate": 0.00019688711360509213,
|
|
"loss": 0.9716,
|
|
"mean_token_accuracy": 0.7890592306852341,
|
|
"num_tokens": 4682795.0,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"entropy": 0.9769839994609356,
|
|
"epoch": 0.38134715025906735,
|
|
"grad_norm": 0.3717665374279022,
|
|
"learning_rate": 0.00019677298350961333,
|
|
"loss": 0.9986,
|
|
"mean_token_accuracy": 0.7845027431845665,
|
|
"num_tokens": 4734983.0,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"entropy": 1.0078977264463902,
|
|
"epoch": 0.3854922279792746,
|
|
"grad_norm": 0.5216906070709229,
|
|
"learning_rate": 0.00019665683298069616,
|
|
"loss": 1.0144,
|
|
"mean_token_accuracy": 0.7834078282117843,
|
|
"num_tokens": 4786331.0,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"entropy": 0.9985935389995575,
|
|
"epoch": 0.38963730569948185,
|
|
"grad_norm": 0.408572793006897,
|
|
"learning_rate": 0.00019653866444333985,
|
|
"loss": 1.0096,
|
|
"mean_token_accuracy": 0.7860454954206944,
|
|
"num_tokens": 4836661.0,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"entropy": 0.9866756744682789,
|
|
"epoch": 0.39378238341968913,
|
|
"grad_norm": 0.43202802538871765,
|
|
"learning_rate": 0.00019641848036467576,
|
|
"loss": 1.0044,
|
|
"mean_token_accuracy": 0.7888299062848091,
|
|
"num_tokens": 4887993.0,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"entropy": 0.9607140734791756,
|
|
"epoch": 0.39792746113989635,
|
|
"grad_norm": 0.4784650206565857,
|
|
"learning_rate": 0.00019629628325391595,
|
|
"loss": 0.9777,
|
|
"mean_token_accuracy": 0.7884210042655468,
|
|
"num_tokens": 4937865.0,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"entropy": 0.947428410500288,
|
|
"epoch": 0.40207253886010363,
|
|
"grad_norm": 0.5031050443649292,
|
|
"learning_rate": 0.0001961720756623006,
|
|
"loss": 0.9529,
|
|
"mean_token_accuracy": 0.7923657827079296,
|
|
"num_tokens": 4988659.0,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"entropy": 0.9914572931826114,
|
|
"epoch": 0.40621761658031086,
|
|
"grad_norm": 0.46874579787254333,
|
|
"learning_rate": 0.0001960458601830449,
|
|
"loss": 1.0075,
|
|
"mean_token_accuracy": 0.7866203740239144,
|
|
"num_tokens": 5038807.0,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"entropy": 1.0107652366161346,
|
|
"epoch": 0.41036269430051814,
|
|
"grad_norm": 0.5741726160049438,
|
|
"learning_rate": 0.00019591763945128493,
|
|
"loss": 1.0242,
|
|
"mean_token_accuracy": 0.7811738647520542,
|
|
"num_tokens": 5090621.0,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"entropy": 1.022918675094843,
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 0.5132948756217957,
|
|
"learning_rate": 0.00019578741614402245,
|
|
"loss": 1.0264,
|
|
"mean_token_accuracy": 0.7782419688999653,
|
|
"num_tokens": 5143415.0,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"eval_entropy": 0.9987384857624535,
|
|
"eval_loss": 1.0029442310333252,
|
|
"eval_mean_token_accuracy": 0.7838117130628721,
|
|
"eval_num_tokens": 5143415.0,
|
|
"eval_runtime": 197.9171,
|
|
"eval_samples_per_second": 20.529,
|
|
"eval_steps_per_second": 2.567,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"entropy": 0.961260611563921,
|
|
"epoch": 0.41865284974093264,
|
|
"grad_norm": 0.5051520466804504,
|
|
"learning_rate": 0.00019565519298006924,
|
|
"loss": 0.9695,
|
|
"mean_token_accuracy": 0.7914115615189076,
|
|
"num_tokens": 5194400.0,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"entropy": 0.9982391525059938,
|
|
"epoch": 0.4227979274611399,
|
|
"grad_norm": 0.4538373351097107,
|
|
"learning_rate": 0.00019552097271999022,
|
|
"loss": 1.0146,
|
|
"mean_token_accuracy": 0.7841442748904228,
|
|
"num_tokens": 5245625.0,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"entropy": 0.9815789975225926,
|
|
"epoch": 0.42694300518134715,
|
|
"grad_norm": 0.4690403342247009,
|
|
"learning_rate": 0.00019538475816604578,
|
|
"loss": 0.9959,
|
|
"mean_token_accuracy": 0.7872560724616051,
|
|
"num_tokens": 5295237.0,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"entropy": 0.9912715546786786,
|
|
"epoch": 0.4310880829015544,
|
|
"grad_norm": 0.42687225341796875,
|
|
"learning_rate": 0.00019524655216213334,
|
|
"loss": 0.9866,
|
|
"mean_token_accuracy": 0.7859402887523175,
|
|
"num_tokens": 5348418.0,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"entropy": 1.0102518431842327,
|
|
"epoch": 0.43523316062176165,
|
|
"grad_norm": 0.44384920597076416,
|
|
"learning_rate": 0.00019510635759372805,
|
|
"loss": 1.0238,
|
|
"mean_token_accuracy": 0.7797439776360988,
|
|
"num_tokens": 5399166.0,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"entropy": 1.0160183981060982,
|
|
"epoch": 0.43937823834196893,
|
|
"grad_norm": 0.44171077013015747,
|
|
"learning_rate": 0.00019496417738782228,
|
|
"loss": 1.0228,
|
|
"mean_token_accuracy": 0.7803056098520755,
|
|
"num_tokens": 5451958.0,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"entropy": 0.9693700931966305,
|
|
"epoch": 0.44352331606217615,
|
|
"grad_norm": 0.4557332694530487,
|
|
"learning_rate": 0.00019482001451286488,
|
|
"loss": 0.9848,
|
|
"mean_token_accuracy": 0.7881055019795895,
|
|
"num_tokens": 5504481.0,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"entropy": 0.9666793867945671,
|
|
"epoch": 0.44766839378238343,
|
|
"grad_norm": 0.45267391204833984,
|
|
"learning_rate": 0.00019467387197869883,
|
|
"loss": 0.9834,
|
|
"mean_token_accuracy": 0.7889953970909118,
|
|
"num_tokens": 5551843.0,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"entropy": 0.9949551515281201,
|
|
"epoch": 0.45181347150259066,
|
|
"grad_norm": 0.438851535320282,
|
|
"learning_rate": 0.00019452575283649873,
|
|
"loss": 1.0144,
|
|
"mean_token_accuracy": 0.782796959578991,
|
|
"num_tokens": 5604244.0,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"entropy": 0.9963484168052673,
|
|
"epoch": 0.45595854922279794,
|
|
"grad_norm": 0.47502249479293823,
|
|
"learning_rate": 0.00019437566017870678,
|
|
"loss": 1.0137,
|
|
"mean_token_accuracy": 0.7818049520254136,
|
|
"num_tokens": 5655400.0,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"entropy": 0.962771862745285,
|
|
"epoch": 0.46010362694300516,
|
|
"grad_norm": 0.42144647240638733,
|
|
"learning_rate": 0.00019422359713896863,
|
|
"loss": 0.9667,
|
|
"mean_token_accuracy": 0.7884697504341602,
|
|
"num_tokens": 5708666.0,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"entropy": 0.9682818073779345,
|
|
"epoch": 0.46424870466321244,
|
|
"grad_norm": 0.3935494124889374,
|
|
"learning_rate": 0.00019406956689206744,
|
|
"loss": 0.9675,
|
|
"mean_token_accuracy": 0.7886212930083275,
|
|
"num_tokens": 5761629.0,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"entropy": 1.037352129071951,
|
|
"epoch": 0.46839378238341967,
|
|
"grad_norm": 0.4130975902080536,
|
|
"learning_rate": 0.00019391357265385807,
|
|
"loss": 1.0403,
|
|
"mean_token_accuracy": 0.7768575675785542,
|
|
"num_tokens": 5812646.0,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"entropy": 0.9832554623484612,
|
|
"epoch": 0.47253886010362695,
|
|
"grad_norm": 0.3829593360424042,
|
|
"learning_rate": 0.00019375561768119958,
|
|
"loss": 0.9976,
|
|
"mean_token_accuracy": 0.7897411204874516,
|
|
"num_tokens": 5862817.0,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"entropy": 0.9487610541284084,
|
|
"epoch": 0.47668393782383417,
|
|
"grad_norm": 0.4701037108898163,
|
|
"learning_rate": 0.00019359570527188754,
|
|
"loss": 0.9638,
|
|
"mean_token_accuracy": 0.792264959961176,
|
|
"num_tokens": 5912716.0,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"entropy": 0.954827756434679,
|
|
"epoch": 0.48082901554404145,
|
|
"grad_norm": 0.4769448935985565,
|
|
"learning_rate": 0.00019343383876458485,
|
|
"loss": 0.9595,
|
|
"mean_token_accuracy": 0.7927994206547737,
|
|
"num_tokens": 5963750.0,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"entropy": 0.9979561999440193,
|
|
"epoch": 0.48497409326424873,
|
|
"grad_norm": 0.46122485399246216,
|
|
"learning_rate": 0.0001932700215387524,
|
|
"loss": 1.0035,
|
|
"mean_token_accuracy": 0.7901160277426242,
|
|
"num_tokens": 6013594.0,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"entropy": 0.9591921351850032,
|
|
"epoch": 0.48911917098445595,
|
|
"grad_norm": 0.4695828855037689,
|
|
"learning_rate": 0.00019310425701457814,
|
|
"loss": 0.9747,
|
|
"mean_token_accuracy": 0.7882664181292057,
|
|
"num_tokens": 6066506.0,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"entropy": 0.9904223918914795,
|
|
"epoch": 0.49326424870466323,
|
|
"grad_norm": 0.43157315254211426,
|
|
"learning_rate": 0.00019293654865290604,
|
|
"loss": 1.007,
|
|
"mean_token_accuracy": 0.7831035263836383,
|
|
"num_tokens": 6120258.0,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"entropy": 0.9716513626277447,
|
|
"epoch": 0.49740932642487046,
|
|
"grad_norm": 0.4715091586112976,
|
|
"learning_rate": 0.00019276689995516346,
|
|
"loss": 0.9743,
|
|
"mean_token_accuracy": 0.7874019861221313,
|
|
"num_tokens": 6171154.0,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"entropy": 0.9984053768217563,
|
|
"epoch": 0.5015544041450777,
|
|
"grad_norm": 0.4164447486400604,
|
|
"learning_rate": 0.0001925953144632884,
|
|
"loss": 1.0145,
|
|
"mean_token_accuracy": 0.781635707616806,
|
|
"num_tokens": 6222515.0,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"entropy": 0.9804062567651272,
|
|
"epoch": 0.505699481865285,
|
|
"grad_norm": 0.45674899220466614,
|
|
"learning_rate": 0.00019242179575965537,
|
|
"loss": 0.9808,
|
|
"mean_token_accuracy": 0.7865826293826104,
|
|
"num_tokens": 6275939.0,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"entropy": 0.9783611670136452,
|
|
"epoch": 0.5098445595854922,
|
|
"grad_norm": 0.4376664161682129,
|
|
"learning_rate": 0.00019224634746700051,
|
|
"loss": 0.992,
|
|
"mean_token_accuracy": 0.7872013546526432,
|
|
"num_tokens": 6325162.0,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"entropy": 0.9830627031624317,
|
|
"epoch": 0.5139896373056995,
|
|
"grad_norm": 0.3772311508655548,
|
|
"learning_rate": 0.00019206897324834614,
|
|
"loss": 0.9948,
|
|
"mean_token_accuracy": 0.7873731434345246,
|
|
"num_tokens": 6376061.0,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"entropy": 0.9824939467012882,
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 0.4799821376800537,
|
|
"learning_rate": 0.0001918896768069242,
|
|
"loss": 0.988,
|
|
"mean_token_accuracy": 0.7887446887791156,
|
|
"num_tokens": 6428028.0,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"eval_entropy": 0.9617302230023962,
|
|
"eval_loss": 0.9882201552391052,
|
|
"eval_mean_token_accuracy": 0.7864582590230806,
|
|
"eval_num_tokens": 6428028.0,
|
|
"eval_runtime": 197.0794,
|
|
"eval_samples_per_second": 20.616,
|
|
"eval_steps_per_second": 2.578,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"entropy": 0.9796643897891044,
|
|
"epoch": 0.522279792746114,
|
|
"grad_norm": 0.45946526527404785,
|
|
"learning_rate": 0.0001917084618860989,
|
|
"loss": 0.9933,
|
|
"mean_token_accuracy": 0.7855292394757271,
|
|
"num_tokens": 6479048.0,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"entropy": 0.9833916153758764,
|
|
"epoch": 0.5264248704663212,
|
|
"grad_norm": 0.3963780999183655,
|
|
"learning_rate": 0.00019152533226928867,
|
|
"loss": 0.9962,
|
|
"mean_token_accuracy": 0.7817724473774433,
|
|
"num_tokens": 6529826.0,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"entropy": 0.9843226552009583,
|
|
"epoch": 0.5305699481865285,
|
|
"grad_norm": 0.4751962125301361,
|
|
"learning_rate": 0.000191340291779887,
|
|
"loss": 0.9994,
|
|
"mean_token_accuracy": 0.7830713532865048,
|
|
"num_tokens": 6580699.0,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"entropy": 0.9555308140814305,
|
|
"epoch": 0.5347150259067357,
|
|
"grad_norm": 0.40511226654052734,
|
|
"learning_rate": 0.00019115334428118286,
|
|
"loss": 0.9761,
|
|
"mean_token_accuracy": 0.7889509312808514,
|
|
"num_tokens": 6633814.0,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"entropy": 0.9578822962939739,
|
|
"epoch": 0.538860103626943,
|
|
"grad_norm": 0.4532525837421417,
|
|
"learning_rate": 0.00019096449367627977,
|
|
"loss": 0.9656,
|
|
"mean_token_accuracy": 0.7934875927865506,
|
|
"num_tokens": 6684053.0,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"entropy": 0.9569033116102219,
|
|
"epoch": 0.5430051813471503,
|
|
"grad_norm": 0.5332207679748535,
|
|
"learning_rate": 0.00019077374390801455,
|
|
"loss": 0.9788,
|
|
"mean_token_accuracy": 0.7917719252407551,
|
|
"num_tokens": 6733968.0,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"entropy": 0.9321629591286182,
|
|
"epoch": 0.5471502590673575,
|
|
"grad_norm": 0.4724487364292145,
|
|
"learning_rate": 0.00019058109895887478,
|
|
"loss": 0.9322,
|
|
"mean_token_accuracy": 0.7971892558038235,
|
|
"num_tokens": 6785674.0,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"entropy": 0.9211927853524685,
|
|
"epoch": 0.5512953367875648,
|
|
"grad_norm": 0.4107973277568817,
|
|
"learning_rate": 0.0001903865628509158,
|
|
"loss": 0.9379,
|
|
"mean_token_accuracy": 0.7973543092608452,
|
|
"num_tokens": 6837186.0,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"entropy": 0.9783548146486283,
|
|
"epoch": 0.555440414507772,
|
|
"grad_norm": 0.4260256588459015,
|
|
"learning_rate": 0.0001901901396456768,
|
|
"loss": 0.986,
|
|
"mean_token_accuracy": 0.7868110507726669,
|
|
"num_tokens": 6889132.0,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"entropy": 0.9350843414664268,
|
|
"epoch": 0.5595854922279793,
|
|
"grad_norm": 0.43470704555511475,
|
|
"learning_rate": 0.00018999183344409578,
|
|
"loss": 0.9496,
|
|
"mean_token_accuracy": 0.796025338023901,
|
|
"num_tokens": 6940037.0,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"entropy": 0.9442426189780235,
|
|
"epoch": 0.5637305699481865,
|
|
"grad_norm": 0.5161582827568054,
|
|
"learning_rate": 0.00018979164838642417,
|
|
"loss": 0.9485,
|
|
"mean_token_accuracy": 0.7975613526999951,
|
|
"num_tokens": 6990041.0,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"entropy": 0.9514821983873845,
|
|
"epoch": 0.5678756476683938,
|
|
"grad_norm": 0.35760676860809326,
|
|
"learning_rate": 0.00018958958865214026,
|
|
"loss": 0.9622,
|
|
"mean_token_accuracy": 0.7867028988897801,
|
|
"num_tokens": 7044442.0,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"entropy": 0.9858744382858277,
|
|
"epoch": 0.572020725388601,
|
|
"grad_norm": 0.4453827738761902,
|
|
"learning_rate": 0.00018938565845986197,
|
|
"loss": 1.0003,
|
|
"mean_token_accuracy": 0.7861887782812118,
|
|
"num_tokens": 7093249.0,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"entropy": 0.9741125054657459,
|
|
"epoch": 0.5761658031088083,
|
|
"grad_norm": 0.433883935213089,
|
|
"learning_rate": 0.00018917986206725887,
|
|
"loss": 0.9804,
|
|
"mean_token_accuracy": 0.7858184657990932,
|
|
"num_tokens": 7144603.0,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"entropy": 0.9409362945705653,
|
|
"epoch": 0.5803108808290155,
|
|
"grad_norm": 0.41870972514152527,
|
|
"learning_rate": 0.000188972203770963,
|
|
"loss": 0.9447,
|
|
"mean_token_accuracy": 0.7908757150173187,
|
|
"num_tokens": 7196254.0,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"entropy": 0.9704828377813101,
|
|
"epoch": 0.5844559585492228,
|
|
"grad_norm": 0.37623921036720276,
|
|
"learning_rate": 0.00018876268790647956,
|
|
"loss": 0.9889,
|
|
"mean_token_accuracy": 0.7871991626918315,
|
|
"num_tokens": 7248841.0,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"entropy": 0.9590335190296173,
|
|
"epoch": 0.5886010362694301,
|
|
"grad_norm": 0.42611730098724365,
|
|
"learning_rate": 0.00018855131884809608,
|
|
"loss": 0.9564,
|
|
"mean_token_accuracy": 0.7912212461233139,
|
|
"num_tokens": 7300448.0,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"entropy": 0.9557400576770305,
|
|
"epoch": 0.5927461139896373,
|
|
"grad_norm": 0.41997987031936646,
|
|
"learning_rate": 0.00018833810100879128,
|
|
"loss": 0.9629,
|
|
"mean_token_accuracy": 0.7843785919249058,
|
|
"num_tokens": 7353316.0,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"entropy": 0.962028032168746,
|
|
"epoch": 0.5968911917098445,
|
|
"grad_norm": 0.41813725233078003,
|
|
"learning_rate": 0.00018812303884014283,
|
|
"loss": 0.975,
|
|
"mean_token_accuracy": 0.792021531611681,
|
|
"num_tokens": 7402745.0,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"entropy": 0.9566993176937103,
|
|
"epoch": 0.6010362694300518,
|
|
"grad_norm": 0.40095973014831543,
|
|
"learning_rate": 0.00018790613683223437,
|
|
"loss": 0.9677,
|
|
"mean_token_accuracy": 0.7910853162407875,
|
|
"num_tokens": 7453651.0,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"entropy": 0.9973684482276439,
|
|
"epoch": 0.6051813471502591,
|
|
"grad_norm": 0.4190211296081543,
|
|
"learning_rate": 0.00018768739951356198,
|
|
"loss": 1.0137,
|
|
"mean_token_accuracy": 0.7809802234172821,
|
|
"num_tokens": 7507052.0,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"entropy": 0.934837969392538,
|
|
"epoch": 0.6093264248704663,
|
|
"grad_norm": 0.4074060916900635,
|
|
"learning_rate": 0.00018746683145093939,
|
|
"loss": 0.9397,
|
|
"mean_token_accuracy": 0.7906834915280342,
|
|
"num_tokens": 7561383.0,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"entropy": 0.950137410312891,
|
|
"epoch": 0.6134715025906736,
|
|
"grad_norm": 0.397320032119751,
|
|
"learning_rate": 0.0001872444372494028,
|
|
"loss": 0.9578,
|
|
"mean_token_accuracy": 0.7921940423548222,
|
|
"num_tokens": 7611992.0,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"entropy": 0.9245664164423942,
|
|
"epoch": 0.6176165803108808,
|
|
"grad_norm": 0.4212367534637451,
|
|
"learning_rate": 0.00018702022155211458,
|
|
"loss": 0.9547,
|
|
"mean_token_accuracy": 0.7904628373682498,
|
|
"num_tokens": 7664532.0,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"entropy": 0.9380590286105871,
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 0.3710251748561859,
|
|
"learning_rate": 0.00018679418904026654,
|
|
"loss": 0.9341,
|
|
"mean_token_accuracy": 0.7926829554140568,
|
|
"num_tokens": 7715917.0,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"eval_entropy": 0.9522759174033413,
|
|
"eval_loss": 0.9706581830978394,
|
|
"eval_mean_token_accuracy": 0.7886052683113128,
|
|
"eval_num_tokens": 7715917.0,
|
|
"eval_runtime": 197.1854,
|
|
"eval_samples_per_second": 20.605,
|
|
"eval_steps_per_second": 2.576,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"entropy": 0.9760079063475132,
|
|
"epoch": 0.6259067357512953,
|
|
"grad_norm": 0.38734835386276245,
|
|
"learning_rate": 0.00018656634443298202,
|
|
"loss": 0.9954,
|
|
"mean_token_accuracy": 0.7867716372013092,
|
|
"num_tokens": 7769962.0,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"entropy": 0.9858164794743061,
|
|
"epoch": 0.6300518134715026,
|
|
"grad_norm": 0.4148380160331726,
|
|
"learning_rate": 0.00018633669248721736,
|
|
"loss": 0.9893,
|
|
"mean_token_accuracy": 0.785686731338501,
|
|
"num_tokens": 7824173.0,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"entropy": 0.9614122346043587,
|
|
"epoch": 0.6341968911917099,
|
|
"grad_norm": 0.4662570357322693,
|
|
"learning_rate": 0.00018610523799766275,
|
|
"loss": 0.9708,
|
|
"mean_token_accuracy": 0.7884422443807125,
|
|
"num_tokens": 7874061.0,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"entropy": 0.9372418120503425,
|
|
"epoch": 0.6383419689119171,
|
|
"grad_norm": 0.43578842282295227,
|
|
"learning_rate": 0.000185871985796642,
|
|
"loss": 0.9501,
|
|
"mean_token_accuracy": 0.7922198139131069,
|
|
"num_tokens": 7925220.0,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"entropy": 0.9477806560695171,
|
|
"epoch": 0.6424870466321243,
|
|
"grad_norm": 0.43452900648117065,
|
|
"learning_rate": 0.0001856369407540116,
|
|
"loss": 0.9672,
|
|
"mean_token_accuracy": 0.7929503574967385,
|
|
"num_tokens": 7977422.0,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"entropy": 0.9531604364514351,
|
|
"epoch": 0.6466321243523316,
|
|
"grad_norm": 0.46345794200897217,
|
|
"learning_rate": 0.0001854001077770592,
|
|
"loss": 0.9618,
|
|
"mean_token_accuracy": 0.7890139617025852,
|
|
"num_tokens": 8030901.0,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"entropy": 0.9154306463897228,
|
|
"epoch": 0.6507772020725389,
|
|
"grad_norm": 0.44431430101394653,
|
|
"learning_rate": 0.000185161491810401,
|
|
"loss": 0.9283,
|
|
"mean_token_accuracy": 0.8002380356192589,
|
|
"num_tokens": 8082737.0,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"entropy": 0.9435916144400835,
|
|
"epoch": 0.6549222797927461,
|
|
"grad_norm": 0.3655269145965576,
|
|
"learning_rate": 0.00018492109783587867,
|
|
"loss": 0.9571,
|
|
"mean_token_accuracy": 0.7891912691295146,
|
|
"num_tokens": 8133737.0,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"entropy": 0.9871173314750195,
|
|
"epoch": 0.6590673575129534,
|
|
"grad_norm": 0.45032334327697754,
|
|
"learning_rate": 0.00018467893087245526,
|
|
"loss": 0.9938,
|
|
"mean_token_accuracy": 0.7833716459572315,
|
|
"num_tokens": 8183945.0,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"entropy": 0.9330227866768837,
|
|
"epoch": 0.6632124352331606,
|
|
"grad_norm": 0.4046294093132019,
|
|
"learning_rate": 0.00018443499597611024,
|
|
"loss": 0.9434,
|
|
"mean_token_accuracy": 0.7933455288410187,
|
|
"num_tokens": 8235327.0,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"entropy": 0.9379861585795879,
|
|
"epoch": 0.6673575129533679,
|
|
"grad_norm": 0.4296433627605438,
|
|
"learning_rate": 0.0001841892982397344,
|
|
"loss": 0.9527,
|
|
"mean_token_accuracy": 0.7907838642597198,
|
|
"num_tokens": 8286392.0,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"entropy": 0.9368982739746571,
|
|
"epoch": 0.6715025906735751,
|
|
"grad_norm": 0.5243311524391174,
|
|
"learning_rate": 0.00018394184279302294,
|
|
"loss": 0.9464,
|
|
"mean_token_accuracy": 0.7926255285739898,
|
|
"num_tokens": 8335527.0,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"entropy": 0.9379900746047497,
|
|
"epoch": 0.6756476683937824,
|
|
"grad_norm": 0.397651344537735,
|
|
"learning_rate": 0.0001836926348023689,
|
|
"loss": 0.9504,
|
|
"mean_token_accuracy": 0.791745250672102,
|
|
"num_tokens": 8388161.0,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"entropy": 0.9145612478256225,
|
|
"epoch": 0.6797927461139897,
|
|
"grad_norm": 0.43967580795288086,
|
|
"learning_rate": 0.0001834416794707549,
|
|
"loss": 0.9165,
|
|
"mean_token_accuracy": 0.7961649537086487,
|
|
"num_tokens": 8441889.0,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"entropy": 0.926224023476243,
|
|
"epoch": 0.6839378238341969,
|
|
"grad_norm": 0.45205602049827576,
|
|
"learning_rate": 0.0001831889820376448,
|
|
"loss": 0.939,
|
|
"mean_token_accuracy": 0.7933929577469826,
|
|
"num_tokens": 8490987.0,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"entropy": 0.9750065371394158,
|
|
"epoch": 0.6880829015544041,
|
|
"grad_norm": 0.43906891345977783,
|
|
"learning_rate": 0.00018293454777887404,
|
|
"loss": 0.9809,
|
|
"mean_token_accuracy": 0.7898602753877639,
|
|
"num_tokens": 8540947.0,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"entropy": 0.9698948271572589,
|
|
"epoch": 0.6922279792746114,
|
|
"grad_norm": 0.4355306327342987,
|
|
"learning_rate": 0.00018267838200653977,
|
|
"loss": 0.9721,
|
|
"mean_token_accuracy": 0.785855708271265,
|
|
"num_tokens": 8592141.0,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"entropy": 0.9159326329827309,
|
|
"epoch": 0.6963730569948187,
|
|
"grad_norm": 0.37019234895706177,
|
|
"learning_rate": 0.00018242049006888974,
|
|
"loss": 0.9234,
|
|
"mean_token_accuracy": 0.8000622071325779,
|
|
"num_tokens": 8642193.0,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"entropy": 0.8978213213384152,
|
|
"epoch": 0.7005181347150259,
|
|
"grad_norm": 0.3895004391670227,
|
|
"learning_rate": 0.00018216087735021072,
|
|
"loss": 0.9101,
|
|
"mean_token_accuracy": 0.7981852144002914,
|
|
"num_tokens": 8693653.0,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"entropy": 0.9253171287477017,
|
|
"epoch": 0.7046632124352331,
|
|
"grad_norm": 0.40894782543182373,
|
|
"learning_rate": 0.00018189954927071603,
|
|
"loss": 0.9329,
|
|
"mean_token_accuracy": 0.7947318039834499,
|
|
"num_tokens": 8744455.0,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"entropy": 0.8997811481356621,
|
|
"epoch": 0.7088082901554404,
|
|
"grad_norm": 0.39331623911857605,
|
|
"learning_rate": 0.00018163651128643248,
|
|
"loss": 0.9036,
|
|
"mean_token_accuracy": 0.8017535306513309,
|
|
"num_tokens": 8796115.0,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"entropy": 0.9271700248122215,
|
|
"epoch": 0.7129533678756477,
|
|
"grad_norm": 0.42100563645362854,
|
|
"learning_rate": 0.0001813717688890864,
|
|
"loss": 0.9429,
|
|
"mean_token_accuracy": 0.7925452619791031,
|
|
"num_tokens": 8846934.0,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"entropy": 0.9542303301393986,
|
|
"epoch": 0.7170984455958549,
|
|
"grad_norm": 0.4123167395591736,
|
|
"learning_rate": 0.00018110532760598895,
|
|
"loss": 0.9613,
|
|
"mean_token_accuracy": 0.7912637561559677,
|
|
"num_tokens": 8895911.0,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"entropy": 0.9155400857329369,
|
|
"epoch": 0.7212435233160622,
|
|
"grad_norm": 0.3995860517024994,
|
|
"learning_rate": 0.00018083719299992067,
|
|
"loss": 0.9283,
|
|
"mean_token_accuracy": 0.796960674971342,
|
|
"num_tokens": 8945784.0,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"entropy": 0.934383400529623,
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 0.4636269509792328,
|
|
"learning_rate": 0.00018056737066901563,
|
|
"loss": 0.9472,
|
|
"mean_token_accuracy": 0.7902762576937675,
|
|
"num_tokens": 8996927.0,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"eval_entropy": 0.9803924484281089,
|
|
"eval_loss": 0.9615499973297119,
|
|
"eval_mean_token_accuracy": 0.7901124554121588,
|
|
"eval_num_tokens": 8996927.0,
|
|
"eval_runtime": 197.2805,
|
|
"eval_samples_per_second": 20.595,
|
|
"eval_steps_per_second": 2.575,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"entropy": 0.9384146019816398,
|
|
"epoch": 0.7295336787564767,
|
|
"grad_norm": 0.39725178480148315,
|
|
"learning_rate": 0.00018029586624664415,
|
|
"loss": 0.9441,
|
|
"mean_token_accuracy": 0.791342943161726,
|
|
"num_tokens": 9047604.0,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"entropy": 0.9334720097482204,
|
|
"epoch": 0.7336787564766839,
|
|
"grad_norm": 0.3954644501209259,
|
|
"learning_rate": 0.00018002268540129543,
|
|
"loss": 0.9417,
|
|
"mean_token_accuracy": 0.795846226811409,
|
|
"num_tokens": 9099017.0,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"entropy": 0.907962579652667,
|
|
"epoch": 0.7378238341968912,
|
|
"grad_norm": 0.39940011501312256,
|
|
"learning_rate": 0.0001797478338364592,
|
|
"loss": 0.9206,
|
|
"mean_token_accuracy": 0.7963109843432903,
|
|
"num_tokens": 9149326.0,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"entropy": 0.9557456441223622,
|
|
"epoch": 0.7419689119170985,
|
|
"grad_norm": 0.4359956383705139,
|
|
"learning_rate": 0.0001794713172905066,
|
|
"loss": 0.9658,
|
|
"mean_token_accuracy": 0.7891202479600906,
|
|
"num_tokens": 9200715.0,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"entropy": 0.9499155536293984,
|
|
"epoch": 0.7461139896373057,
|
|
"grad_norm": 0.507350504398346,
|
|
"learning_rate": 0.00017919314153657027,
|
|
"loss": 0.9634,
|
|
"mean_token_accuracy": 0.7931549407541751,
|
|
"num_tokens": 9251534.0,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"entropy": 0.9438272595405579,
|
|
"epoch": 0.7502590673575129,
|
|
"grad_norm": 0.4581449329853058,
|
|
"learning_rate": 0.000178913312382424,
|
|
"loss": 0.9584,
|
|
"mean_token_accuracy": 0.7942813694477081,
|
|
"num_tokens": 9302614.0,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"entropy": 0.9297045323997736,
|
|
"epoch": 0.7544041450777202,
|
|
"grad_norm": 0.35071489214897156,
|
|
"learning_rate": 0.00017863183567036137,
|
|
"loss": 0.9414,
|
|
"mean_token_accuracy": 0.7904184371232986,
|
|
"num_tokens": 9353455.0,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"entropy": 0.9570980660617352,
|
|
"epoch": 0.7585492227979275,
|
|
"grad_norm": 0.4017302989959717,
|
|
"learning_rate": 0.0001783487172770738,
|
|
"loss": 0.9741,
|
|
"mean_token_accuracy": 0.7883284002542496,
|
|
"num_tokens": 9405502.0,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"entropy": 0.9436403628438711,
|
|
"epoch": 0.7626943005181347,
|
|
"grad_norm": 0.5486421585083008,
|
|
"learning_rate": 0.00017806396311352782,
|
|
"loss": 0.9596,
|
|
"mean_token_accuracy": 0.7928663790225983,
|
|
"num_tokens": 9455621.0,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"entropy": 0.9594994522631168,
|
|
"epoch": 0.7668393782383419,
|
|
"grad_norm": 0.4096115231513977,
|
|
"learning_rate": 0.00017777757912484175,
|
|
"loss": 0.973,
|
|
"mean_token_accuracy": 0.7888169556856155,
|
|
"num_tokens": 9507088.0,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"entropy": 0.940360489487648,
|
|
"epoch": 0.7709844559585493,
|
|
"grad_norm": 0.3976588845252991,
|
|
"learning_rate": 0.00017748957129016147,
|
|
"loss": 0.9386,
|
|
"mean_token_accuracy": 0.7945226788520813,
|
|
"num_tokens": 9558157.0,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"entropy": 0.946375384926796,
|
|
"epoch": 0.7751295336787565,
|
|
"grad_norm": 0.4322788715362549,
|
|
"learning_rate": 0.00017719994562253558,
|
|
"loss": 0.951,
|
|
"mean_token_accuracy": 0.7903898723423481,
|
|
"num_tokens": 9611510.0,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"entropy": 0.9477811448276043,
|
|
"epoch": 0.7792746113989637,
|
|
"grad_norm": 0.43729427456855774,
|
|
"learning_rate": 0.00017690870816879014,
|
|
"loss": 0.9661,
|
|
"mean_token_accuracy": 0.7892401278018951,
|
|
"num_tokens": 9662287.0,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"entropy": 0.949357783794403,
|
|
"epoch": 0.783419689119171,
|
|
"grad_norm": 0.3893526792526245,
|
|
"learning_rate": 0.0001766158650094019,
|
|
"loss": 0.9518,
|
|
"mean_token_accuracy": 0.793975007534027,
|
|
"num_tokens": 9715516.0,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"entropy": 0.9205279342830182,
|
|
"epoch": 0.7875647668393783,
|
|
"grad_norm": 0.46602073311805725,
|
|
"learning_rate": 0.00017632142225837183,
|
|
"loss": 0.9304,
|
|
"mean_token_accuracy": 0.7955152101814746,
|
|
"num_tokens": 9766133.0,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"entropy": 0.9258571691811085,
|
|
"epoch": 0.7917098445595855,
|
|
"grad_norm": 0.39727622270584106,
|
|
"learning_rate": 0.0001760253860630973,
|
|
"loss": 0.9364,
|
|
"mean_token_accuracy": 0.7936219781637192,
|
|
"num_tokens": 9817516.0,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"entropy": 0.9301609247922897,
|
|
"epoch": 0.7958549222797927,
|
|
"grad_norm": 0.43491441011428833,
|
|
"learning_rate": 0.0001757277626042436,
|
|
"loss": 0.9432,
|
|
"mean_token_accuracy": 0.7936487175524235,
|
|
"num_tokens": 9871371.0,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"entropy": 0.9301216453313828,
|
|
"epoch": 0.8,
|
|
"grad_norm": 0.37254294753074646,
|
|
"learning_rate": 0.0001754285580956152,
|
|
"loss": 0.9328,
|
|
"mean_token_accuracy": 0.7928548909723758,
|
|
"num_tokens": 9923261.0,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"entropy": 0.9340247794985771,
|
|
"epoch": 0.8041450777202073,
|
|
"grad_norm": 0.4496139585971832,
|
|
"learning_rate": 0.00017512777878402568,
|
|
"loss": 0.9544,
|
|
"mean_token_accuracy": 0.7901494115591049,
|
|
"num_tokens": 9973379.0,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"entropy": 0.9295564122498036,
|
|
"epoch": 0.8082901554404145,
|
|
"grad_norm": 0.470420777797699,
|
|
"learning_rate": 0.00017482543094916758,
|
|
"loss": 0.9383,
|
|
"mean_token_accuracy": 0.7970516562461853,
|
|
"num_tokens": 10024029.0,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"entropy": 0.936601135879755,
|
|
"epoch": 0.8124352331606217,
|
|
"grad_norm": 0.38742539286613464,
|
|
"learning_rate": 0.0001745215209034811,
|
|
"loss": 0.9611,
|
|
"mean_token_accuracy": 0.7904755167663098,
|
|
"num_tokens": 10077153.0,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"entropy": 0.9289965465664863,
|
|
"epoch": 0.816580310880829,
|
|
"grad_norm": 0.4919315278530121,
|
|
"learning_rate": 0.00017421605499202242,
|
|
"loss": 0.9364,
|
|
"mean_token_accuracy": 0.7947124816477299,
|
|
"num_tokens": 10128837.0,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"entropy": 0.9195003911852837,
|
|
"epoch": 0.8207253886010363,
|
|
"grad_norm": 0.43714791536331177,
|
|
"learning_rate": 0.0001739090395923313,
|
|
"loss": 0.9194,
|
|
"mean_token_accuracy": 0.796821691840887,
|
|
"num_tokens": 10180279.0,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"entropy": 0.9256344065070152,
|
|
"epoch": 0.8248704663212435,
|
|
"grad_norm": 0.3864556550979614,
|
|
"learning_rate": 0.00017360048111429767,
|
|
"loss": 0.9483,
|
|
"mean_token_accuracy": 0.7930616028606892,
|
|
"num_tokens": 10230973.0,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"entropy": 0.9640789791941643,
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 0.4234633147716522,
|
|
"learning_rate": 0.00017329038600002807,
|
|
"loss": 0.9681,
|
|
"mean_token_accuracy": 0.7887310594320297,
|
|
"num_tokens": 10283849.0,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"eval_entropy": 0.9277598650202038,
|
|
"eval_loss": 0.9486145973205566,
|
|
"eval_mean_token_accuracy": 0.7921127796642423,
|
|
"eval_num_tokens": 10283849.0,
|
|
"eval_runtime": 196.6351,
|
|
"eval_samples_per_second": 20.663,
|
|
"eval_steps_per_second": 2.583,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"entropy": 0.939198125898838,
|
|
"epoch": 0.8331606217616581,
|
|
"grad_norm": 0.40732425451278687,
|
|
"learning_rate": 0.00017297876072371103,
|
|
"loss": 0.9512,
|
|
"mean_token_accuracy": 0.7914685815572738,
|
|
"num_tokens": 10336653.0,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"entropy": 0.918938459455967,
|
|
"epoch": 0.8373056994818653,
|
|
"grad_norm": 0.5398784875869751,
|
|
"learning_rate": 0.00017266561179148183,
|
|
"loss": 0.9186,
|
|
"mean_token_accuracy": 0.7927700787782669,
|
|
"num_tokens": 10387330.0,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"entropy": 0.9128594268113375,
|
|
"epoch": 0.8414507772020725,
|
|
"grad_norm": 0.43407782912254333,
|
|
"learning_rate": 0.0001723509457412869,
|
|
"loss": 0.931,
|
|
"mean_token_accuracy": 0.7971027828752995,
|
|
"num_tokens": 10438933.0,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"entropy": 0.9039206847548484,
|
|
"epoch": 0.8455958549222798,
|
|
"grad_norm": 0.46065929532051086,
|
|
"learning_rate": 0.000172034769142747,
|
|
"loss": 0.9192,
|
|
"mean_token_accuracy": 0.7961257986724377,
|
|
"num_tokens": 10491722.0,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"entropy": 0.9538728334009647,
|
|
"epoch": 0.8497409326424871,
|
|
"grad_norm": 0.45542430877685547,
|
|
"learning_rate": 0.00017171708859702037,
|
|
"loss": 0.9573,
|
|
"mean_token_accuracy": 0.7893645465373993,
|
|
"num_tokens": 10546885.0,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"entropy": 0.9198624655604363,
|
|
"epoch": 0.8538860103626943,
|
|
"grad_norm": 0.3631950318813324,
|
|
"learning_rate": 0.00017139791073666468,
|
|
"loss": 0.9313,
|
|
"mean_token_accuracy": 0.7931442521512508,
|
|
"num_tokens": 10600746.0,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"entropy": 0.9213518410921097,
|
|
"epoch": 0.8580310880829015,
|
|
"grad_norm": 0.4949886202812195,
|
|
"learning_rate": 0.00017107724222549872,
|
|
"loss": 0.9335,
|
|
"mean_token_accuracy": 0.7955467194318772,
|
|
"num_tokens": 10653477.0,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"entropy": 0.9536297917366028,
|
|
"epoch": 0.8621761658031089,
|
|
"grad_norm": 0.4028345048427582,
|
|
"learning_rate": 0.00017075508975846311,
|
|
"loss": 0.9605,
|
|
"mean_token_accuracy": 0.7890767879784107,
|
|
"num_tokens": 10706433.0,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"entropy": 0.9051979005336761,
|
|
"epoch": 0.8663212435233161,
|
|
"grad_norm": 0.4099532663822174,
|
|
"learning_rate": 0.0001704314600614807,
|
|
"loss": 0.9178,
|
|
"mean_token_accuracy": 0.7976575642824173,
|
|
"num_tokens": 10758950.0,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"entropy": 0.9456002049148082,
|
|
"epoch": 0.8704663212435233,
|
|
"grad_norm": 0.44490471482276917,
|
|
"learning_rate": 0.00017010635989131597,
|
|
"loss": 0.9531,
|
|
"mean_token_accuracy": 0.791679710894823,
|
|
"num_tokens": 10810783.0,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"entropy": 0.9459167554974556,
|
|
"epoch": 0.8746113989637305,
|
|
"grad_norm": 0.4577106535434723,
|
|
"learning_rate": 0.0001697797960354341,
|
|
"loss": 0.9609,
|
|
"mean_token_accuracy": 0.7873589277267456,
|
|
"num_tokens": 10863258.0,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"entropy": 0.8723687015473842,
|
|
"epoch": 0.8787564766839379,
|
|
"grad_norm": 0.40351179242134094,
|
|
"learning_rate": 0.00016945177531185922,
|
|
"loss": 0.8735,
|
|
"mean_token_accuracy": 0.8079392656683921,
|
|
"num_tokens": 10914577.0,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"entropy": 0.9348406352102756,
|
|
"epoch": 0.8829015544041451,
|
|
"grad_norm": 0.42083558440208435,
|
|
"learning_rate": 0.00016912230456903198,
|
|
"loss": 0.9457,
|
|
"mean_token_accuracy": 0.7947359785437584,
|
|
"num_tokens": 10966052.0,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"entropy": 0.892266447097063,
|
|
"epoch": 0.8870466321243523,
|
|
"grad_norm": 0.413835346698761,
|
|
"learning_rate": 0.0001687913906856667,
|
|
"loss": 0.8917,
|
|
"mean_token_accuracy": 0.7988453857600689,
|
|
"num_tokens": 11017837.0,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"entropy": 0.9510011091828346,
|
|
"epoch": 0.8911917098445595,
|
|
"grad_norm": 0.41305822134017944,
|
|
"learning_rate": 0.00016845904057060767,
|
|
"loss": 0.9565,
|
|
"mean_token_accuracy": 0.7917753420770168,
|
|
"num_tokens": 11068955.0,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"entropy": 0.9034442327916622,
|
|
"epoch": 0.8953367875647669,
|
|
"grad_norm": 0.42375388741493225,
|
|
"learning_rate": 0.000168125261162685,
|
|
"loss": 0.9089,
|
|
"mean_token_accuracy": 0.7968145549297333,
|
|
"num_tokens": 11119250.0,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"entropy": 0.9643872417509556,
|
|
"epoch": 0.8994818652849741,
|
|
"grad_norm": 0.4074117839336395,
|
|
"learning_rate": 0.0001677900594305695,
|
|
"loss": 0.9788,
|
|
"mean_token_accuracy": 0.7875596329569816,
|
|
"num_tokens": 11172260.0,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"entropy": 0.92549347281456,
|
|
"epoch": 0.9036269430051813,
|
|
"grad_norm": 0.456617534160614,
|
|
"learning_rate": 0.0001674534423726276,
|
|
"loss": 0.9426,
|
|
"mean_token_accuracy": 0.7943698972463608,
|
|
"num_tokens": 11225236.0,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"entropy": 0.9161198817193508,
|
|
"epoch": 0.9077720207253887,
|
|
"grad_norm": 0.5100305080413818,
|
|
"learning_rate": 0.0001671154170167748,
|
|
"loss": 0.9266,
|
|
"mean_token_accuracy": 0.7931325800716877,
|
|
"num_tokens": 11277166.0,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"entropy": 0.8795958921313286,
|
|
"epoch": 0.9119170984455959,
|
|
"grad_norm": 0.4124648869037628,
|
|
"learning_rate": 0.00016677599042032933,
|
|
"loss": 0.8845,
|
|
"mean_token_accuracy": 0.8016303740441799,
|
|
"num_tokens": 11328093.0,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"entropy": 0.9241185076534748,
|
|
"epoch": 0.9160621761658031,
|
|
"grad_norm": 0.47688350081443787,
|
|
"learning_rate": 0.00016643516966986446,
|
|
"loss": 0.9312,
|
|
"mean_token_accuracy": 0.7907878324389458,
|
|
"num_tokens": 11380215.0,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"entropy": 0.8871541582047939,
|
|
"epoch": 0.9202072538860103,
|
|
"grad_norm": 0.44556674361228943,
|
|
"learning_rate": 0.00016609296188106084,
|
|
"loss": 0.9016,
|
|
"mean_token_accuracy": 0.7992243818938732,
|
|
"num_tokens": 11432104.0,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"entropy": 0.9216676332056523,
|
|
"epoch": 0.9243523316062177,
|
|
"grad_norm": 0.43731555342674255,
|
|
"learning_rate": 0.00016574937419855772,
|
|
"loss": 0.9234,
|
|
"mean_token_accuracy": 0.795772710442543,
|
|
"num_tokens": 11482908.0,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"entropy": 0.9152931831777096,
|
|
"epoch": 0.9284974093264249,
|
|
"grad_norm": 0.40907105803489685,
|
|
"learning_rate": 0.00016540441379580387,
|
|
"loss": 0.9271,
|
|
"mean_token_accuracy": 0.7936010919511318,
|
|
"num_tokens": 11535803.0,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"entropy": 0.941212996840477,
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 0.469002902507782,
|
|
"learning_rate": 0.00016505808787490792,
|
|
"loss": 0.9491,
|
|
"mean_token_accuracy": 0.793215012550354,
|
|
"num_tokens": 11586430.0,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"eval_entropy": 0.8924233510972946,
|
|
"eval_loss": 0.9386545419692993,
|
|
"eval_mean_token_accuracy": 0.7933616472745505,
|
|
"eval_num_tokens": 11586430.0,
|
|
"eval_runtime": 197.7861,
|
|
"eval_samples_per_second": 20.542,
|
|
"eval_steps_per_second": 2.568,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"entropy": 0.8890458445996046,
|
|
"epoch": 0.9367875647668393,
|
|
"grad_norm": 0.3855128884315491,
|
|
"learning_rate": 0.00016471040366648776,
|
|
"loss": 0.8964,
|
|
"mean_token_accuracy": 0.8021313309669494,
|
|
"num_tokens": 11639117.0,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"entropy": 0.9057373393326997,
|
|
"epoch": 0.9409326424870467,
|
|
"grad_norm": 0.46859732270240784,
|
|
"learning_rate": 0.00016436136842951977,
|
|
"loss": 0.9213,
|
|
"mean_token_accuracy": 0.7989621929824352,
|
|
"num_tokens": 11689267.0,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"entropy": 0.9198335148394108,
|
|
"epoch": 0.9450777202072539,
|
|
"grad_norm": 0.40903231501579285,
|
|
"learning_rate": 0.0001640109894511872,
|
|
"loss": 0.9284,
|
|
"mean_token_accuracy": 0.7965403601527214,
|
|
"num_tokens": 11740945.0,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"entropy": 0.9119356460869312,
|
|
"epoch": 0.9492227979274611,
|
|
"grad_norm": 0.42043864727020264,
|
|
"learning_rate": 0.00016365927404672807,
|
|
"loss": 0.9249,
|
|
"mean_token_accuracy": 0.796663673222065,
|
|
"num_tokens": 11794600.0,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"entropy": 0.9416319027543067,
|
|
"epoch": 0.9533678756476683,
|
|
"grad_norm": 0.41366055607795715,
|
|
"learning_rate": 0.0001633062295592823,
|
|
"loss": 0.9583,
|
|
"mean_token_accuracy": 0.7909570150077343,
|
|
"num_tokens": 11844693.0,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"entropy": 0.9526241350919008,
|
|
"epoch": 0.9575129533678757,
|
|
"grad_norm": 0.41388946771621704,
|
|
"learning_rate": 0.00016295186335973857,
|
|
"loss": 0.9674,
|
|
"mean_token_accuracy": 0.790430235862732,
|
|
"num_tokens": 11893941.0,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"entropy": 0.9134444534778595,
|
|
"epoch": 0.9616580310880829,
|
|
"grad_norm": 0.3884899914264679,
|
|
"learning_rate": 0.0001625961828465804,
|
|
"loss": 0.929,
|
|
"mean_token_accuracy": 0.7953568585216999,
|
|
"num_tokens": 11945935.0,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"entropy": 0.9262224324047565,
|
|
"epoch": 0.9658031088082901,
|
|
"grad_norm": 0.4220770299434662,
|
|
"learning_rate": 0.0001622391954457315,
|
|
"loss": 0.9259,
|
|
"mean_token_accuracy": 0.7925183959305286,
|
|
"num_tokens": 11999075.0,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"entropy": 0.931026178970933,
|
|
"epoch": 0.9699481865284975,
|
|
"grad_norm": 0.4672662615776062,
|
|
"learning_rate": 0.00016188090861040107,
|
|
"loss": 0.9433,
|
|
"mean_token_accuracy": 0.7952560156583786,
|
|
"num_tokens": 12047725.0,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"entropy": 0.9058261185884475,
|
|
"epoch": 0.9740932642487047,
|
|
"grad_norm": 0.48223602771759033,
|
|
"learning_rate": 0.00016152132982092784,
|
|
"loss": 0.9122,
|
|
"mean_token_accuracy": 0.7959005422890186,
|
|
"num_tokens": 12099181.0,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"entropy": 0.9087510220706463,
|
|
"epoch": 0.9782383419689119,
|
|
"grad_norm": 0.423324316740036,
|
|
"learning_rate": 0.00016116046658462414,
|
|
"loss": 0.9114,
|
|
"mean_token_accuracy": 0.7989039994776249,
|
|
"num_tokens": 12152610.0,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"entropy": 0.9042955547571182,
|
|
"epoch": 0.9823834196891191,
|
|
"grad_norm": 0.46391963958740234,
|
|
"learning_rate": 0.00016079832643561904,
|
|
"loss": 0.9199,
|
|
"mean_token_accuracy": 0.7983533076941967,
|
|
"num_tokens": 12202583.0,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"entropy": 0.9194691389799118,
|
|
"epoch": 0.9865284974093265,
|
|
"grad_norm": 0.4457080364227295,
|
|
"learning_rate": 0.00016043491693470114,
|
|
"loss": 0.9331,
|
|
"mean_token_accuracy": 0.7974948555231094,
|
|
"num_tokens": 12253776.0,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"entropy": 0.8977299593389034,
|
|
"epoch": 0.9906735751295337,
|
|
"grad_norm": 0.47106996178627014,
|
|
"learning_rate": 0.00016007024566916058,
|
|
"loss": 0.9083,
|
|
"mean_token_accuracy": 0.795095720142126,
|
|
"num_tokens": 12305175.0,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"entropy": 0.9298750199377537,
|
|
"epoch": 0.9948186528497409,
|
|
"grad_norm": 0.5093875527381897,
|
|
"learning_rate": 0.00015970432025263074,
|
|
"loss": 0.9329,
|
|
"mean_token_accuracy": 0.793617632240057,
|
|
"num_tokens": 12357750.0,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"entropy": 0.9333759970963001,
|
|
"epoch": 0.9989637305699481,
|
|
"grad_norm": 0.41947904229164124,
|
|
"learning_rate": 0.0001593371483249293,
|
|
"loss": 0.9529,
|
|
"mean_token_accuracy": 0.793743172287941,
|
|
"num_tokens": 12409215.0,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"entropy": 0.8986701011657715,
|
|
"epoch": 1.0031088082901554,
|
|
"grad_norm": 0.5347554683685303,
|
|
"learning_rate": 0.00015896873755189868,
|
|
"loss": 0.9002,
|
|
"mean_token_accuracy": 0.8045198410749436,
|
|
"num_tokens": 12460502.0,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"entropy": 0.8791909657418728,
|
|
"epoch": 1.0072538860103626,
|
|
"grad_norm": 0.5265800952911377,
|
|
"learning_rate": 0.00015859909562524594,
|
|
"loss": 0.8929,
|
|
"mean_token_accuracy": 0.8018290206789971,
|
|
"num_tokens": 12510710.0,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"entropy": 0.8866066768765449,
|
|
"epoch": 1.01139896373057,
|
|
"grad_norm": 0.5650938749313354,
|
|
"learning_rate": 0.00015822823026238235,
|
|
"loss": 0.8927,
|
|
"mean_token_accuracy": 0.8024179935455322,
|
|
"num_tokens": 12561257.0,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"entropy": 0.8647308357059955,
|
|
"epoch": 1.0155440414507773,
|
|
"grad_norm": 0.4968569576740265,
|
|
"learning_rate": 0.0001578561492062621,
|
|
"loss": 0.8744,
|
|
"mean_token_accuracy": 0.8047979861497879,
|
|
"num_tokens": 12610438.0,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"entropy": 0.8654443889856338,
|
|
"epoch": 1.0196891191709845,
|
|
"grad_norm": 0.48992252349853516,
|
|
"learning_rate": 0.0001574828602252207,
|
|
"loss": 0.87,
|
|
"mean_token_accuracy": 0.8046348355710506,
|
|
"num_tokens": 12661271.0,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"entropy": 0.8522643312811852,
|
|
"epoch": 1.0238341968911917,
|
|
"grad_norm": 0.4607287049293518,
|
|
"learning_rate": 0.00015710837111281286,
|
|
"loss": 0.8611,
|
|
"mean_token_accuracy": 0.8077139921486378,
|
|
"num_tokens": 12709628.0,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"entropy": 0.8478192627429962,
|
|
"epoch": 1.027979274611399,
|
|
"grad_norm": 0.537442684173584,
|
|
"learning_rate": 0.0001567326896876497,
|
|
"loss": 0.8664,
|
|
"mean_token_accuracy": 0.8070318132638932,
|
|
"num_tokens": 12760895.0,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"entropy": 0.890300364792347,
|
|
"epoch": 1.0321243523316062,
|
|
"grad_norm": 0.4382570683956146,
|
|
"learning_rate": 0.00015635582379323556,
|
|
"loss": 0.8902,
|
|
"mean_token_accuracy": 0.7996452160179615,
|
|
"num_tokens": 12812211.0,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"entropy": 0.8272694803774356,
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 0.41449499130249023,
|
|
"learning_rate": 0.0001559777812978042,
|
|
"loss": 0.8292,
|
|
"mean_token_accuracy": 0.811527369171381,
|
|
"num_tokens": 12861861.0,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"eval_entropy": 0.8690557431752287,
|
|
"eval_loss": 0.9333181977272034,
|
|
"eval_mean_token_accuracy": 0.7949006537518163,
|
|
"eval_num_tokens": 12861861.0,
|
|
"eval_runtime": 198.9365,
|
|
"eval_samples_per_second": 20.424,
|
|
"eval_steps_per_second": 2.554,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"entropy": 0.8840624932199717,
|
|
"epoch": 1.0404145077720208,
|
|
"grad_norm": 0.45501822233200073,
|
|
"learning_rate": 0.00015559857009415446,
|
|
"loss": 0.8945,
|
|
"mean_token_accuracy": 0.7989841207861901,
|
|
"num_tokens": 12914151.0,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"entropy": 0.8572823069989681,
|
|
"epoch": 1.044559585492228,
|
|
"grad_norm": 0.5166523456573486,
|
|
"learning_rate": 0.0001552181980994857,
|
|
"loss": 0.8709,
|
|
"mean_token_accuracy": 0.804636436700821,
|
|
"num_tokens": 12965541.0,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"entropy": 0.8395044583827257,
|
|
"epoch": 1.0487046632124353,
|
|
"grad_norm": 0.4897792935371399,
|
|
"learning_rate": 0.00015483667325523226,
|
|
"loss": 0.8372,
|
|
"mean_token_accuracy": 0.8102173201739788,
|
|
"num_tokens": 13015739.0,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"entropy": 0.8277278915047646,
|
|
"epoch": 1.0528497409326425,
|
|
"grad_norm": 0.49756333231925964,
|
|
"learning_rate": 0.00015445400352689776,
|
|
"loss": 0.8385,
|
|
"mean_token_accuracy": 0.810855308175087,
|
|
"num_tokens": 13067799.0,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"entropy": 0.8553471677005291,
|
|
"epoch": 1.0569948186528497,
|
|
"grad_norm": 0.45558294653892517,
|
|
"learning_rate": 0.00015407019690388883,
|
|
"loss": 0.8575,
|
|
"mean_token_accuracy": 0.8095838151872158,
|
|
"num_tokens": 13119789.0,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"entropy": 0.8509638644754887,
|
|
"epoch": 1.061139896373057,
|
|
"grad_norm": 0.4646300971508026,
|
|
"learning_rate": 0.0001536852613993483,
|
|
"loss": 0.8584,
|
|
"mean_token_accuracy": 0.8090752191841603,
|
|
"num_tokens": 13170846.0,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"entropy": 0.8805008858442307,
|
|
"epoch": 1.0652849740932642,
|
|
"grad_norm": 0.5446432828903198,
|
|
"learning_rate": 0.0001532992050499878,
|
|
"loss": 0.9017,
|
|
"mean_token_accuracy": 0.7998991191387177,
|
|
"num_tokens": 13220584.0,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"entropy": 0.842747250199318,
|
|
"epoch": 1.0694300518134714,
|
|
"grad_norm": 0.5087260007858276,
|
|
"learning_rate": 0.00015291203591592006,
|
|
"loss": 0.8422,
|
|
"mean_token_accuracy": 0.8114993080496788,
|
|
"num_tokens": 13272164.0,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"entropy": 0.8400920636951923,
|
|
"epoch": 1.0735751295336788,
|
|
"grad_norm": 0.4728451669216156,
|
|
"learning_rate": 0.00015252376208049065,
|
|
"loss": 0.8444,
|
|
"mean_token_accuracy": 0.8054023750126362,
|
|
"num_tokens": 13322247.0,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"entropy": 0.8870699379593134,
|
|
"epoch": 1.077720207253886,
|
|
"grad_norm": 0.40374264121055603,
|
|
"learning_rate": 0.00015213439165010913,
|
|
"loss": 0.9051,
|
|
"mean_token_accuracy": 0.8014436505734921,
|
|
"num_tokens": 13373989.0,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"entropy": 0.8589731320738793,
|
|
"epoch": 1.0818652849740933,
|
|
"grad_norm": 0.5243843197822571,
|
|
"learning_rate": 0.00015174393275407984,
|
|
"loss": 0.8628,
|
|
"mean_token_accuracy": 0.8063308753073215,
|
|
"num_tokens": 13425462.0,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"entropy": 0.8770733550190926,
|
|
"epoch": 1.0860103626943005,
|
|
"grad_norm": 0.5799617171287537,
|
|
"learning_rate": 0.00015135239354443228,
|
|
"loss": 0.8892,
|
|
"mean_token_accuracy": 0.802444314211607,
|
|
"num_tokens": 13475590.0,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"entropy": 0.8747978635132313,
|
|
"epoch": 1.0901554404145077,
|
|
"grad_norm": 0.43597203493118286,
|
|
"learning_rate": 0.00015095978219575065,
|
|
"loss": 0.8794,
|
|
"mean_token_accuracy": 0.8046314589679241,
|
|
"num_tokens": 13526393.0,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"entropy": 0.8770341359078884,
|
|
"epoch": 1.094300518134715,
|
|
"grad_norm": 0.5226919651031494,
|
|
"learning_rate": 0.00015056610690500355,
|
|
"loss": 0.8933,
|
|
"mean_token_accuracy": 0.8010272890329361,
|
|
"num_tokens": 13576698.0,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"entropy": 0.8940253898501396,
|
|
"epoch": 1.0984455958549222,
|
|
"grad_norm": 0.5339784622192383,
|
|
"learning_rate": 0.0001501713758913725,
|
|
"loss": 0.9002,
|
|
"mean_token_accuracy": 0.7996869809925556,
|
|
"num_tokens": 13626951.0,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"entropy": 0.8270330794155598,
|
|
"epoch": 1.1025906735751296,
|
|
"grad_norm": 0.5476909875869751,
|
|
"learning_rate": 0.0001497755973960805,
|
|
"loss": 0.8318,
|
|
"mean_token_accuracy": 0.8103690385818482,
|
|
"num_tokens": 13676675.0,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"entropy": 0.8499515745788813,
|
|
"epoch": 1.1067357512953369,
|
|
"grad_norm": 0.45196080207824707,
|
|
"learning_rate": 0.00014937877968222,
|
|
"loss": 0.8501,
|
|
"mean_token_accuracy": 0.808135537803173,
|
|
"num_tokens": 13728080.0,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"entropy": 0.859894473478198,
|
|
"epoch": 1.110880829015544,
|
|
"grad_norm": 0.4680575430393219,
|
|
"learning_rate": 0.00014898093103458024,
|
|
"loss": 0.868,
|
|
"mean_token_accuracy": 0.8064894266426563,
|
|
"num_tokens": 13780037.0,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"entropy": 0.87540787383914,
|
|
"epoch": 1.1150259067357513,
|
|
"grad_norm": 0.5199806094169617,
|
|
"learning_rate": 0.0001485820597594745,
|
|
"loss": 0.8895,
|
|
"mean_token_accuracy": 0.8030873388051987,
|
|
"num_tokens": 13833519.0,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"entropy": 0.8485930152237415,
|
|
"epoch": 1.1191709844559585,
|
|
"grad_norm": 0.49646317958831787,
|
|
"learning_rate": 0.00014818217418456636,
|
|
"loss": 0.8513,
|
|
"mean_token_accuracy": 0.807842418551445,
|
|
"num_tokens": 13886015.0,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"entropy": 0.8749716937541961,
|
|
"epoch": 1.1233160621761658,
|
|
"grad_norm": 0.5455443859100342,
|
|
"learning_rate": 0.0001477812826586963,
|
|
"loss": 0.8841,
|
|
"mean_token_accuracy": 0.8027867391705513,
|
|
"num_tokens": 13937925.0,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"entropy": 0.8707845766097307,
|
|
"epoch": 1.127461139896373,
|
|
"grad_norm": 0.4450802206993103,
|
|
"learning_rate": 0.00014737939355170687,
|
|
"loss": 0.8801,
|
|
"mean_token_accuracy": 0.8022250264883042,
|
|
"num_tokens": 13988764.0,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"entropy": 0.8741528563201427,
|
|
"epoch": 1.1316062176165804,
|
|
"grad_norm": 0.5555823445320129,
|
|
"learning_rate": 0.00014697651525426829,
|
|
"loss": 0.879,
|
|
"mean_token_accuracy": 0.8004407949745656,
|
|
"num_tokens": 14040186.0,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"entropy": 0.8610983267426491,
|
|
"epoch": 1.1357512953367876,
|
|
"grad_norm": 0.5157086849212646,
|
|
"learning_rate": 0.0001465726561777032,
|
|
"loss": 0.8726,
|
|
"mean_token_accuracy": 0.8097968354821206,
|
|
"num_tokens": 14091303.0,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"entropy": 0.823892167955637,
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 0.49958255887031555,
|
|
"learning_rate": 0.00014616782475381093,
|
|
"loss": 0.829,
|
|
"mean_token_accuracy": 0.8124792411923408,
|
|
"num_tokens": 14141648.0,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"eval_entropy": 0.8880770857643894,
|
|
"eval_loss": 0.923703134059906,
|
|
"eval_mean_token_accuracy": 0.7956895906859496,
|
|
"eval_num_tokens": 14141648.0,
|
|
"eval_runtime": 198.4957,
|
|
"eval_samples_per_second": 20.469,
|
|
"eval_steps_per_second": 2.559,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"entropy": 0.8655072554945946,
|
|
"epoch": 1.144041450777202,
|
|
"grad_norm": 0.5644556879997253,
|
|
"learning_rate": 0.00014576202943469165,
|
|
"loss": 0.875,
|
|
"mean_token_accuracy": 0.8008928537368775,
|
|
"num_tokens": 14194700.0,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"entropy": 0.8973465126007796,
|
|
"epoch": 1.1481865284974093,
|
|
"grad_norm": 0.49897336959838867,
|
|
"learning_rate": 0.00014535527869256976,
|
|
"loss": 0.9007,
|
|
"mean_token_accuracy": 0.8015423193573952,
|
|
"num_tokens": 14245950.0,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"entropy": 0.8233009725809097,
|
|
"epoch": 1.1523316062176165,
|
|
"grad_norm": 0.5420376658439636,
|
|
"learning_rate": 0.00014494758101961697,
|
|
"loss": 0.8345,
|
|
"mean_token_accuracy": 0.807911217212677,
|
|
"num_tokens": 14298656.0,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"entropy": 0.8658364303410053,
|
|
"epoch": 1.1564766839378238,
|
|
"grad_norm": 0.540746808052063,
|
|
"learning_rate": 0.00014453894492777529,
|
|
"loss": 0.8705,
|
|
"mean_token_accuracy": 0.8011497743427753,
|
|
"num_tokens": 14351015.0,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"entropy": 0.8474766366183758,
|
|
"epoch": 1.160621761658031,
|
|
"grad_norm": 0.5403730869293213,
|
|
"learning_rate": 0.00014412937894857887,
|
|
"loss": 0.8594,
|
|
"mean_token_accuracy": 0.8074184566736221,
|
|
"num_tokens": 14402619.0,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"entropy": 0.8391683183610439,
|
|
"epoch": 1.1647668393782384,
|
|
"grad_norm": 0.5576226115226746,
|
|
"learning_rate": 0.00014371889163297628,
|
|
"loss": 0.8424,
|
|
"mean_token_accuracy": 0.8085787981748581,
|
|
"num_tokens": 14454711.0,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"entropy": 0.8149294909089804,
|
|
"epoch": 1.1689119170984457,
|
|
"grad_norm": 0.5232577919960022,
|
|
"learning_rate": 0.00014330749155115174,
|
|
"loss": 0.8388,
|
|
"mean_token_accuracy": 0.8110665172338486,
|
|
"num_tokens": 14503711.0,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"entropy": 0.8229341760277749,
|
|
"epoch": 1.1730569948186529,
|
|
"grad_norm": 0.5092782974243164,
|
|
"learning_rate": 0.00014289518729234624,
|
|
"loss": 0.8213,
|
|
"mean_token_accuracy": 0.812018783390522,
|
|
"num_tokens": 14554288.0,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"entropy": 0.8426572777330875,
|
|
"epoch": 1.1772020725388601,
|
|
"grad_norm": 0.5316895246505737,
|
|
"learning_rate": 0.00014248198746467843,
|
|
"loss": 0.8471,
|
|
"mean_token_accuracy": 0.8073183879256248,
|
|
"num_tokens": 14606122.0,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"entropy": 0.8306729834526777,
|
|
"epoch": 1.1813471502590673,
|
|
"grad_norm": 0.5324336886405945,
|
|
"learning_rate": 0.00014206790069496447,
|
|
"loss": 0.8514,
|
|
"mean_token_accuracy": 0.8104600816965103,
|
|
"num_tokens": 14656128.0,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"entropy": 0.8300536021590232,
|
|
"epoch": 1.1854922279792746,
|
|
"grad_norm": 0.471822589635849,
|
|
"learning_rate": 0.00014165293562853833,
|
|
"loss": 0.8314,
|
|
"mean_token_accuracy": 0.8092638865113259,
|
|
"num_tokens": 14708944.0,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"entropy": 0.8340942483395338,
|
|
"epoch": 1.1896373056994818,
|
|
"grad_norm": 0.42829155921936035,
|
|
"learning_rate": 0.00014123710092907107,
|
|
"loss": 0.8519,
|
|
"mean_token_accuracy": 0.8097477406263351,
|
|
"num_tokens": 14761403.0,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"entropy": 0.834505184367299,
|
|
"epoch": 1.193782383419689,
|
|
"grad_norm": 0.41529011726379395,
|
|
"learning_rate": 0.0001408204052783899,
|
|
"loss": 0.8459,
|
|
"mean_token_accuracy": 0.8096166871488094,
|
|
"num_tokens": 14811565.0,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"entropy": 0.8581496514379978,
|
|
"epoch": 1.1979274611398965,
|
|
"grad_norm": 0.5302514433860779,
|
|
"learning_rate": 0.00014040285737629727,
|
|
"loss": 0.8743,
|
|
"mean_token_accuracy": 0.8023737244307995,
|
|
"num_tokens": 14863298.0,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"entropy": 0.8606280945241451,
|
|
"epoch": 1.2020725388601037,
|
|
"grad_norm": 0.5317311882972717,
|
|
"learning_rate": 0.00013998446594038876,
|
|
"loss": 0.8807,
|
|
"mean_token_accuracy": 0.8070294849574566,
|
|
"num_tokens": 14912856.0,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"entropy": 0.856556510925293,
|
|
"epoch": 1.206217616580311,
|
|
"grad_norm": 0.4416259527206421,
|
|
"learning_rate": 0.0001395652397058715,
|
|
"loss": 0.8749,
|
|
"mean_token_accuracy": 0.8048085369169712,
|
|
"num_tokens": 14965818.0,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"entropy": 0.9081879466772079,
|
|
"epoch": 1.2103626943005181,
|
|
"grad_norm": 0.5628664493560791,
|
|
"learning_rate": 0.00013914518742538145,
|
|
"loss": 0.9154,
|
|
"mean_token_accuracy": 0.8001430742442608,
|
|
"num_tokens": 15017461.0,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"entropy": 0.8471661895513535,
|
|
"epoch": 1.2145077720207254,
|
|
"grad_norm": 0.4217177927494049,
|
|
"learning_rate": 0.00013872431786880095,
|
|
"loss": 0.8606,
|
|
"mean_token_accuracy": 0.802645842730999,
|
|
"num_tokens": 15069830.0,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"entropy": 0.832932249456644,
|
|
"epoch": 1.2186528497409326,
|
|
"grad_norm": 0.473371684551239,
|
|
"learning_rate": 0.0001383026398230754,
|
|
"loss": 0.8356,
|
|
"mean_token_accuracy": 0.8073890879750252,
|
|
"num_tokens": 15121475.0,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"entropy": 0.8435809016227722,
|
|
"epoch": 1.2227979274611398,
|
|
"grad_norm": 0.5021840929985046,
|
|
"learning_rate": 0.00013788016209202996,
|
|
"loss": 0.862,
|
|
"mean_token_accuracy": 0.8057342693209648,
|
|
"num_tokens": 15173323.0,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"entropy": 0.858016487956047,
|
|
"epoch": 1.2269430051813472,
|
|
"grad_norm": 0.49231767654418945,
|
|
"learning_rate": 0.00013745689349618561,
|
|
"loss": 0.8561,
|
|
"mean_token_accuracy": 0.8049639880657196,
|
|
"num_tokens": 15225327.0,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"entropy": 0.8462564747780561,
|
|
"epoch": 1.2310880829015545,
|
|
"grad_norm": 0.4842222332954407,
|
|
"learning_rate": 0.0001370328428725751,
|
|
"loss": 0.8604,
|
|
"mean_token_accuracy": 0.8090966388583183,
|
|
"num_tokens": 15274814.0,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"entropy": 0.8369969923049212,
|
|
"epoch": 1.2352331606217617,
|
|
"grad_norm": 0.4423791766166687,
|
|
"learning_rate": 0.00013660801907455842,
|
|
"loss": 0.8384,
|
|
"mean_token_accuracy": 0.8092852041125298,
|
|
"num_tokens": 15327291.0,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"entropy": 0.8335775695741177,
|
|
"epoch": 1.239378238341969,
|
|
"grad_norm": 0.5259456038475037,
|
|
"learning_rate": 0.00013618243097163795,
|
|
"loss": 0.8393,
|
|
"mean_token_accuracy": 0.8095761790871621,
|
|
"num_tokens": 15381193.0,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"entropy": 0.8415486201643944,
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 0.5251457691192627,
|
|
"learning_rate": 0.00013575608744927328,
|
|
"loss": 0.8474,
|
|
"mean_token_accuracy": 0.8060796700417996,
|
|
"num_tokens": 15433001.0,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"eval_entropy": 0.8585143367371221,
|
|
"eval_loss": 0.9117017984390259,
|
|
"eval_mean_token_accuracy": 0.7978430621501967,
|
|
"eval_num_tokens": 15433001.0,
|
|
"eval_runtime": 198.6993,
|
|
"eval_samples_per_second": 20.448,
|
|
"eval_steps_per_second": 2.557,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"entropy": 0.8643816657364368,
|
|
"epoch": 1.2476683937823834,
|
|
"grad_norm": 0.5006239414215088,
|
|
"learning_rate": 0.0001353289974086957,
|
|
"loss": 0.8698,
|
|
"mean_token_accuracy": 0.8013977088034153,
|
|
"num_tokens": 15485038.0,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"entropy": 0.843774139136076,
|
|
"epoch": 1.2518134715025906,
|
|
"grad_norm": 0.5511196255683899,
|
|
"learning_rate": 0.00013490116976672232,
|
|
"loss": 0.8469,
|
|
"mean_token_accuracy": 0.8050686173141003,
|
|
"num_tokens": 15538963.0,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"entropy": 0.8365302201360464,
|
|
"epoch": 1.255958549222798,
|
|
"grad_norm": 0.49812430143356323,
|
|
"learning_rate": 0.00013447261345557004,
|
|
"loss": 0.8521,
|
|
"mean_token_accuracy": 0.8066954366862774,
|
|
"num_tokens": 15588940.0,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"entropy": 0.8695347078144551,
|
|
"epoch": 1.2601036269430053,
|
|
"grad_norm": 0.48924511671066284,
|
|
"learning_rate": 0.0001340433374226689,
|
|
"loss": 0.8838,
|
|
"mean_token_accuracy": 0.79907456189394,
|
|
"num_tokens": 15641733.0,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"entropy": 0.8577761381864548,
|
|
"epoch": 1.2642487046632125,
|
|
"grad_norm": 0.5156821608543396,
|
|
"learning_rate": 0.00013361335063047535,
|
|
"loss": 0.86,
|
|
"mean_token_accuracy": 0.805779180675745,
|
|
"num_tokens": 15691140.0,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"entropy": 0.8075544081628323,
|
|
"epoch": 1.2683937823834197,
|
|
"grad_norm": 0.5274946689605713,
|
|
"learning_rate": 0.00013318266205628517,
|
|
"loss": 0.815,
|
|
"mean_token_accuracy": 0.8150236658751965,
|
|
"num_tokens": 15742262.0,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"entropy": 0.8192636791616679,
|
|
"epoch": 1.272538860103627,
|
|
"grad_norm": 0.5230441093444824,
|
|
"learning_rate": 0.0001327512806920459,
|
|
"loss": 0.818,
|
|
"mean_token_accuracy": 0.8141300410032273,
|
|
"num_tokens": 15793507.0,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"entropy": 0.8397725142538548,
|
|
"epoch": 1.2766839378238342,
|
|
"grad_norm": 0.46072137355804443,
|
|
"learning_rate": 0.0001323192155441693,
|
|
"loss": 0.853,
|
|
"mean_token_accuracy": 0.8097899332642555,
|
|
"num_tokens": 15842829.0,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"entropy": 0.8665676057338715,
|
|
"epoch": 1.2808290155440414,
|
|
"grad_norm": 0.5086504220962524,
|
|
"learning_rate": 0.00013188647563334318,
|
|
"loss": 0.8749,
|
|
"mean_token_accuracy": 0.8014019787311554,
|
|
"num_tokens": 15896240.0,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"entropy": 0.8529352061450481,
|
|
"epoch": 1.2849740932642488,
|
|
"grad_norm": 0.6207098364830017,
|
|
"learning_rate": 0.00013145306999434307,
|
|
"loss": 0.851,
|
|
"mean_token_accuracy": 0.8078053623437882,
|
|
"num_tokens": 15948228.0,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"entropy": 0.8836882207542658,
|
|
"epoch": 1.2891191709844558,
|
|
"grad_norm": 0.5001370906829834,
|
|
"learning_rate": 0.0001310190076758437,
|
|
"loss": 0.8963,
|
|
"mean_token_accuracy": 0.8000708214938641,
|
|
"num_tokens": 15999003.0,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"entropy": 0.8294613551348448,
|
|
"epoch": 1.2932642487046633,
|
|
"grad_norm": 0.5259645581245422,
|
|
"learning_rate": 0.0001305842977402299,
|
|
"loss": 0.8277,
|
|
"mean_token_accuracy": 0.8141705945134163,
|
|
"num_tokens": 16049768.0,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"entropy": 0.7990455340594054,
|
|
"epoch": 1.2974093264248705,
|
|
"grad_norm": 0.6949828267097473,
|
|
"learning_rate": 0.00013014894926340756,
|
|
"loss": 0.818,
|
|
"mean_token_accuracy": 0.8159361682832241,
|
|
"num_tokens": 16100649.0,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"entropy": 0.8716464608907699,
|
|
"epoch": 1.3015544041450777,
|
|
"grad_norm": 0.5351895689964294,
|
|
"learning_rate": 0.00012971297133461404,
|
|
"loss": 0.8757,
|
|
"mean_token_accuracy": 0.806228144466877,
|
|
"num_tokens": 16151132.0,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"entropy": 0.8658848643302918,
|
|
"epoch": 1.305699481865285,
|
|
"grad_norm": 0.4988473355770111,
|
|
"learning_rate": 0.00012927637305622854,
|
|
"loss": 0.8759,
|
|
"mean_token_accuracy": 0.8043223187327385,
|
|
"num_tokens": 16199699.0,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"entropy": 0.8393181882798672,
|
|
"epoch": 1.3098445595854922,
|
|
"grad_norm": 0.6557036638259888,
|
|
"learning_rate": 0.00012883916354358187,
|
|
"loss": 0.8646,
|
|
"mean_token_accuracy": 0.8063815765082836,
|
|
"num_tokens": 16248978.0,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"entropy": 0.8659711088985205,
|
|
"epoch": 1.3139896373056996,
|
|
"grad_norm": 0.4643518030643463,
|
|
"learning_rate": 0.00012840135192476623,
|
|
"loss": 0.8722,
|
|
"mean_token_accuracy": 0.8084965907037258,
|
|
"num_tokens": 16301616.0,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"entropy": 0.8105513293296098,
|
|
"epoch": 1.3181347150259066,
|
|
"grad_norm": 0.5949639081954956,
|
|
"learning_rate": 0.00012796294734044472,
|
|
"loss": 0.8248,
|
|
"mean_token_accuracy": 0.8129052989184856,
|
|
"num_tokens": 16353801.0,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"entropy": 0.8505993623286485,
|
|
"epoch": 1.322279792746114,
|
|
"grad_norm": 0.5101528763771057,
|
|
"learning_rate": 0.00012752395894366028,
|
|
"loss": 0.8621,
|
|
"mean_token_accuracy": 0.8028258629143238,
|
|
"num_tokens": 16404445.0,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"entropy": 0.8487086150795221,
|
|
"epoch": 1.3264248704663213,
|
|
"grad_norm": 0.5054559111595154,
|
|
"learning_rate": 0.00012708439589964493,
|
|
"loss": 0.8582,
|
|
"mean_token_accuracy": 0.8066637292504311,
|
|
"num_tokens": 16455827.0,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"entropy": 0.8373073261231184,
|
|
"epoch": 1.3305699481865285,
|
|
"grad_norm": 0.6401093602180481,
|
|
"learning_rate": 0.000126644267385628,
|
|
"loss": 0.8484,
|
|
"mean_token_accuracy": 0.8094668321311473,
|
|
"num_tokens": 16508083.0,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"entropy": 0.8590784061700105,
|
|
"epoch": 1.3347150259067357,
|
|
"grad_norm": 0.5628422498703003,
|
|
"learning_rate": 0.0001262035825906449,
|
|
"loss": 0.8711,
|
|
"mean_token_accuracy": 0.8022596552968025,
|
|
"num_tokens": 16558595.0,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"entropy": 0.8520713247358799,
|
|
"epoch": 1.338860103626943,
|
|
"grad_norm": 0.5349719524383545,
|
|
"learning_rate": 0.00012576235071534505,
|
|
"loss": 0.8449,
|
|
"mean_token_accuracy": 0.8066559001803398,
|
|
"num_tokens": 16610218.0,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"entropy": 0.8428357064723968,
|
|
"epoch": 1.3430051813471502,
|
|
"grad_norm": 0.49081674218177795,
|
|
"learning_rate": 0.00012532058097179986,
|
|
"loss": 0.8535,
|
|
"mean_token_accuracy": 0.8075176730751992,
|
|
"num_tokens": 16661885.0,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"entropy": 0.8258651193231344,
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 0.46557745337486267,
|
|
"learning_rate": 0.0001248782825833104,
|
|
"loss": 0.8356,
|
|
"mean_token_accuracy": 0.8086560435593129,
|
|
"num_tokens": 16714755.0,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"eval_entropy": 0.8672052781412921,
|
|
"eval_loss": 0.8999478220939636,
|
|
"eval_mean_token_accuracy": 0.7994705477333445,
|
|
"eval_num_tokens": 16714755.0,
|
|
"eval_runtime": 198.43,
|
|
"eval_samples_per_second": 20.476,
|
|
"eval_steps_per_second": 2.56,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"entropy": 0.8300185158848763,
|
|
"epoch": 1.3512953367875649,
|
|
"grad_norm": 0.48430800437927246,
|
|
"learning_rate": 0.00012443546478421483,
|
|
"loss": 0.8244,
|
|
"mean_token_accuracy": 0.8106158778071404,
|
|
"num_tokens": 16765887.0,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"entropy": 0.8473370209336281,
|
|
"epoch": 1.355440414507772,
|
|
"grad_norm": 0.5041252374649048,
|
|
"learning_rate": 0.00012399213681969553,
|
|
"loss": 0.8451,
|
|
"mean_token_accuracy": 0.8070519365370273,
|
|
"num_tokens": 16817043.0,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"entropy": 0.8274556834250688,
|
|
"epoch": 1.3595854922279793,
|
|
"grad_norm": 0.600204348564148,
|
|
"learning_rate": 0.00012354830794558634,
|
|
"loss": 0.8387,
|
|
"mean_token_accuracy": 0.8088588923215866,
|
|
"num_tokens": 16868674.0,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"entropy": 0.8054228462278843,
|
|
"epoch": 1.3637305699481865,
|
|
"grad_norm": 0.4903612732887268,
|
|
"learning_rate": 0.0001231039874281789,
|
|
"loss": 0.8119,
|
|
"mean_token_accuracy": 0.8124719470739364,
|
|
"num_tokens": 16919825.0,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"entropy": 0.863025251030922,
|
|
"epoch": 1.3678756476683938,
|
|
"grad_norm": 0.5454948544502258,
|
|
"learning_rate": 0.00012265918454402956,
|
|
"loss": 0.8801,
|
|
"mean_token_accuracy": 0.8062082342803478,
|
|
"num_tokens": 16970445.0,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"entropy": 0.8554578214883805,
|
|
"epoch": 1.372020725388601,
|
|
"grad_norm": 0.45817020535469055,
|
|
"learning_rate": 0.00012221390857976552,
|
|
"loss": 0.854,
|
|
"mean_token_accuracy": 0.805611626803875,
|
|
"num_tokens": 17022081.0,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"entropy": 0.8327577732503414,
|
|
"epoch": 1.3761658031088082,
|
|
"grad_norm": 0.5555339455604553,
|
|
"learning_rate": 0.00012176816883189098,
|
|
"loss": 0.8481,
|
|
"mean_token_accuracy": 0.8094927176833153,
|
|
"num_tokens": 17073856.0,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"entropy": 0.8180656850337982,
|
|
"epoch": 1.3803108808290157,
|
|
"grad_norm": 0.5528201460838318,
|
|
"learning_rate": 0.00012132197460659302,
|
|
"loss": 0.8202,
|
|
"mean_token_accuracy": 0.8145633541047573,
|
|
"num_tokens": 17125812.0,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"entropy": 0.8341928899288178,
|
|
"epoch": 1.3844559585492229,
|
|
"grad_norm": 0.5436283349990845,
|
|
"learning_rate": 0.00012087533521954738,
|
|
"loss": 0.8498,
|
|
"mean_token_accuracy": 0.8051383703947067,
|
|
"num_tokens": 17177882.0,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"entropy": 0.8418808333575726,
|
|
"epoch": 1.38860103626943,
|
|
"grad_norm": 0.4761488437652588,
|
|
"learning_rate": 0.00012042825999572389,
|
|
"loss": 0.854,
|
|
"mean_token_accuracy": 0.8077041938900947,
|
|
"num_tokens": 17228781.0,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"entropy": 0.8426752701401711,
|
|
"epoch": 1.3927461139896373,
|
|
"grad_norm": 0.5098069906234741,
|
|
"learning_rate": 0.00011998075826919186,
|
|
"loss": 0.8386,
|
|
"mean_token_accuracy": 0.8088356919586659,
|
|
"num_tokens": 17281237.0,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"entropy": 0.8159036364406347,
|
|
"epoch": 1.3968911917098445,
|
|
"grad_norm": 0.47502851486206055,
|
|
"learning_rate": 0.00011953283938292507,
|
|
"loss": 0.8194,
|
|
"mean_token_accuracy": 0.8093040868639946,
|
|
"num_tokens": 17334918.0,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"entropy": 0.8014930065721273,
|
|
"epoch": 1.4010362694300518,
|
|
"grad_norm": 0.48169076442718506,
|
|
"learning_rate": 0.00011908451268860683,
|
|
"loss": 0.803,
|
|
"mean_token_accuracy": 0.8121360823512077,
|
|
"num_tokens": 17389654.0,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"entropy": 0.8264496996998787,
|
|
"epoch": 1.405181347150259,
|
|
"grad_norm": 0.5647678375244141,
|
|
"learning_rate": 0.00011863578754643477,
|
|
"loss": 0.8424,
|
|
"mean_token_accuracy": 0.8126275010406971,
|
|
"num_tokens": 17439076.0,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"entropy": 0.7977627020329237,
|
|
"epoch": 1.4093264248704664,
|
|
"grad_norm": 0.4393410086631775,
|
|
"learning_rate": 0.00011818667332492519,
|
|
"loss": 0.7995,
|
|
"mean_token_accuracy": 0.8173242390155793,
|
|
"num_tokens": 17490400.0,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"entropy": 0.8283428326249123,
|
|
"epoch": 1.4134715025906734,
|
|
"grad_norm": 0.5114555954933167,
|
|
"learning_rate": 0.00011773717940071775,
|
|
"loss": 0.8415,
|
|
"mean_token_accuracy": 0.8091488346457482,
|
|
"num_tokens": 17542277.0,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"entropy": 0.8346800617873669,
|
|
"epoch": 1.417616580310881,
|
|
"grad_norm": 0.5786413550376892,
|
|
"learning_rate": 0.00011728731515837957,
|
|
"loss": 0.8469,
|
|
"mean_token_accuracy": 0.8066603220999241,
|
|
"num_tokens": 17595279.0,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"entropy": 0.8550420433282853,
|
|
"epoch": 1.4217616580310881,
|
|
"grad_norm": 0.5571143627166748,
|
|
"learning_rate": 0.00011683708999020917,
|
|
"loss": 0.848,
|
|
"mean_token_accuracy": 0.8096319273114204,
|
|
"num_tokens": 17646281.0,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"entropy": 0.8568415675312281,
|
|
"epoch": 1.4259067357512953,
|
|
"grad_norm": 0.5374813675880432,
|
|
"learning_rate": 0.00011638651329604065,
|
|
"loss": 0.8716,
|
|
"mean_token_accuracy": 0.8028912648558617,
|
|
"num_tokens": 17699991.0,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"entropy": 0.8243790756911039,
|
|
"epoch": 1.4300518134715026,
|
|
"grad_norm": 0.5662438273429871,
|
|
"learning_rate": 0.00011593559448304719,
|
|
"loss": 0.8331,
|
|
"mean_token_accuracy": 0.809017975628376,
|
|
"num_tokens": 17752012.0,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"entropy": 0.8257568281143903,
|
|
"epoch": 1.4341968911917098,
|
|
"grad_norm": 0.645889163017273,
|
|
"learning_rate": 0.00011548434296554482,
|
|
"loss": 0.8291,
|
|
"mean_token_accuracy": 0.8112167201936245,
|
|
"num_tokens": 17803848.0,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"entropy": 0.8181511592119932,
|
|
"epoch": 1.4383419689119172,
|
|
"grad_norm": 0.5994529128074646,
|
|
"learning_rate": 0.00011503276816479583,
|
|
"loss": 0.8198,
|
|
"mean_token_accuracy": 0.8139660328626632,
|
|
"num_tokens": 17854838.0,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"entropy": 0.8200897790491581,
|
|
"epoch": 1.4424870466321242,
|
|
"grad_norm": 0.5802255272865295,
|
|
"learning_rate": 0.00011458087950881187,
|
|
"loss": 0.8299,
|
|
"mean_token_accuracy": 0.8112624324858189,
|
|
"num_tokens": 17904416.0,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"entropy": 0.8070213198661804,
|
|
"epoch": 1.4466321243523317,
|
|
"grad_norm": 0.6289516687393188,
|
|
"learning_rate": 0.00011412868643215752,
|
|
"loss": 0.8136,
|
|
"mean_token_accuracy": 0.81369848549366,
|
|
"num_tokens": 17955196.0,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"entropy": 0.8097393732517958,
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 0.4648899435997009,
|
|
"learning_rate": 0.00011367619837575288,
|
|
"loss": 0.8161,
|
|
"mean_token_accuracy": 0.8125620447099209,
|
|
"num_tokens": 18006201.0,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"eval_entropy": 0.8238435296092447,
|
|
"eval_loss": 0.8907586932182312,
|
|
"eval_mean_token_accuracy": 0.8013772557335576,
|
|
"eval_num_tokens": 18006201.0,
|
|
"eval_runtime": 198.4999,
|
|
"eval_samples_per_second": 20.469,
|
|
"eval_steps_per_second": 2.559,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"entropy": 0.8560092329978943,
|
|
"epoch": 1.4549222797927461,
|
|
"grad_norm": 0.6165677309036255,
|
|
"learning_rate": 0.00011322342478667675,
|
|
"loss": 0.8691,
|
|
"mean_token_accuracy": 0.8045230902731418,
|
|
"num_tokens": 18056473.0,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"entropy": 0.843320706486702,
|
|
"epoch": 1.4590673575129534,
|
|
"grad_norm": 0.5168942213058472,
|
|
"learning_rate": 0.00011277037511796935,
|
|
"loss": 0.8527,
|
|
"mean_token_accuracy": 0.807574936747551,
|
|
"num_tokens": 18109011.0,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"entropy": 0.8072948306798935,
|
|
"epoch": 1.4632124352331606,
|
|
"grad_norm": 0.5400416851043701,
|
|
"learning_rate": 0.00011231705882843475,
|
|
"loss": 0.8075,
|
|
"mean_token_accuracy": 0.8139371827244759,
|
|
"num_tokens": 18160257.0,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"entropy": 0.8513644590973855,
|
|
"epoch": 1.4673575129533678,
|
|
"grad_norm": 0.48429426550865173,
|
|
"learning_rate": 0.00011186348538244373,
|
|
"loss": 0.8648,
|
|
"mean_token_accuracy": 0.803517735004425,
|
|
"num_tokens": 18214069.0,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"entropy": 0.7893289439380169,
|
|
"epoch": 1.471502590673575,
|
|
"grad_norm": 0.5499273538589478,
|
|
"learning_rate": 0.00011140966424973587,
|
|
"loss": 0.7918,
|
|
"mean_token_accuracy": 0.8176083937287331,
|
|
"num_tokens": 18264693.0,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"entropy": 0.8045220617204905,
|
|
"epoch": 1.4756476683937825,
|
|
"grad_norm": 0.5206645727157593,
|
|
"learning_rate": 0.00011095560490522206,
|
|
"loss": 0.8077,
|
|
"mean_token_accuracy": 0.8162005722522736,
|
|
"num_tokens": 18314907.0,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"entropy": 0.8232816342264414,
|
|
"epoch": 1.4797927461139897,
|
|
"grad_norm": 0.538935124874115,
|
|
"learning_rate": 0.00011050131682878653,
|
|
"loss": 0.8332,
|
|
"mean_token_accuracy": 0.8122490800917148,
|
|
"num_tokens": 18364529.0,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"entropy": 0.8094791114330292,
|
|
"epoch": 1.483937823834197,
|
|
"grad_norm": 0.6132236123085022,
|
|
"learning_rate": 0.00011004680950508902,
|
|
"loss": 0.8276,
|
|
"mean_token_accuracy": 0.8141122475266457,
|
|
"num_tokens": 18417472.0,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"entropy": 0.829077123478055,
|
|
"epoch": 1.4880829015544041,
|
|
"grad_norm": 0.5404674410820007,
|
|
"learning_rate": 0.00010959209242336675,
|
|
"loss": 0.8227,
|
|
"mean_token_accuracy": 0.8100025370717049,
|
|
"num_tokens": 18469801.0,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"entropy": 0.8118694521486759,
|
|
"epoch": 1.4922279792746114,
|
|
"grad_norm": 0.5093076229095459,
|
|
"learning_rate": 0.00010913717507723618,
|
|
"loss": 0.8165,
|
|
"mean_token_accuracy": 0.8142033599317073,
|
|
"num_tokens": 18517522.0,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"entropy": 0.8019976295530796,
|
|
"epoch": 1.4963730569948186,
|
|
"grad_norm": 0.4714727997779846,
|
|
"learning_rate": 0.0001086820669644951,
|
|
"loss": 0.8078,
|
|
"mean_token_accuracy": 0.8166663497686386,
|
|
"num_tokens": 18568632.0,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"entropy": 0.83862883746624,
|
|
"epoch": 1.5005181347150258,
|
|
"grad_norm": 0.5765607357025146,
|
|
"learning_rate": 0.00010822677758692392,
|
|
"loss": 0.8576,
|
|
"mean_token_accuracy": 0.8069118566811084,
|
|
"num_tokens": 18620019.0,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"entropy": 0.812782196700573,
|
|
"epoch": 1.5046632124352333,
|
|
"grad_norm": 0.6457787156105042,
|
|
"learning_rate": 0.00010777131645008771,
|
|
"loss": 0.818,
|
|
"mean_token_accuracy": 0.8110380724072457,
|
|
"num_tokens": 18671835.0,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"entropy": 0.8287965424358845,
|
|
"epoch": 1.5088082901554403,
|
|
"grad_norm": 0.5896804332733154,
|
|
"learning_rate": 0.0001073156930631374,
|
|
"loss": 0.829,
|
|
"mean_token_accuracy": 0.80876934081316,
|
|
"num_tokens": 18723224.0,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"entropy": 0.8180272746831179,
|
|
"epoch": 1.5129533678756477,
|
|
"grad_norm": 0.5445335507392883,
|
|
"learning_rate": 0.0001068599169386114,
|
|
"loss": 0.8323,
|
|
"mean_token_accuracy": 0.8124563902616501,
|
|
"num_tokens": 18773617.0,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"entropy": 0.8370382864028215,
|
|
"epoch": 1.517098445595855,
|
|
"grad_norm": 0.6012756824493408,
|
|
"learning_rate": 0.0001064039975922371,
|
|
"loss": 0.8294,
|
|
"mean_token_accuracy": 0.8105267629027366,
|
|
"num_tokens": 18824266.0,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"entropy": 0.8495032925158739,
|
|
"epoch": 1.5212435233160622,
|
|
"grad_norm": 0.5166526436805725,
|
|
"learning_rate": 0.00010594794454273193,
|
|
"loss": 0.8628,
|
|
"mean_token_accuracy": 0.8086043260991573,
|
|
"num_tokens": 18873655.0,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"entropy": 0.8049097340553999,
|
|
"epoch": 1.5253886010362694,
|
|
"grad_norm": 0.5029107928276062,
|
|
"learning_rate": 0.00010549176731160497,
|
|
"loss": 0.8187,
|
|
"mean_token_accuracy": 0.8109171070158482,
|
|
"num_tokens": 18928282.0,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"entropy": 0.8324394032359124,
|
|
"epoch": 1.5295336787564766,
|
|
"grad_norm": 0.5992510914802551,
|
|
"learning_rate": 0.00010503547542295777,
|
|
"loss": 0.8476,
|
|
"mean_token_accuracy": 0.8105188995599747,
|
|
"num_tokens": 18980358.0,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"entropy": 0.8332845911383628,
|
|
"epoch": 1.533678756476684,
|
|
"grad_norm": 0.6292238235473633,
|
|
"learning_rate": 0.00010457907840328582,
|
|
"loss": 0.8277,
|
|
"mean_token_accuracy": 0.8115433685481548,
|
|
"num_tokens": 19032878.0,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"entropy": 0.840051831305027,
|
|
"epoch": 1.537823834196891,
|
|
"grad_norm": 0.5706473588943481,
|
|
"learning_rate": 0.00010412258578127958,
|
|
"loss": 0.8584,
|
|
"mean_token_accuracy": 0.8070104755461216,
|
|
"num_tokens": 19087084.0,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"entropy": 0.8229773003607989,
|
|
"epoch": 1.5419689119170985,
|
|
"grad_norm": 0.5656697154045105,
|
|
"learning_rate": 0.00010366600708762539,
|
|
"loss": 0.8325,
|
|
"mean_token_accuracy": 0.811124337464571,
|
|
"num_tokens": 19139345.0,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"entropy": 0.8013437777757645,
|
|
"epoch": 1.5461139896373057,
|
|
"grad_norm": 0.6003351211547852,
|
|
"learning_rate": 0.00010320935185480672,
|
|
"loss": 0.8096,
|
|
"mean_token_accuracy": 0.8137698873877526,
|
|
"num_tokens": 19191604.0,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"entropy": 0.8188879381865263,
|
|
"epoch": 1.550259067357513,
|
|
"grad_norm": 0.6367331147193909,
|
|
"learning_rate": 0.00010275262961690493,
|
|
"loss": 0.8178,
|
|
"mean_token_accuracy": 0.8128097750246525,
|
|
"num_tokens": 19242492.0,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"entropy": 0.841957900673151,
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 0.5700961351394653,
|
|
"learning_rate": 0.00010229584990940033,
|
|
"loss": 0.8468,
|
|
"mean_token_accuracy": 0.8067762546241284,
|
|
"num_tokens": 19293849.0,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"eval_entropy": 0.833337394036646,
|
|
"eval_loss": 0.8787927031517029,
|
|
"eval_mean_token_accuracy": 0.803317973932882,
|
|
"eval_num_tokens": 19293849.0,
|
|
"eval_runtime": 198.8156,
|
|
"eval_samples_per_second": 20.436,
|
|
"eval_steps_per_second": 2.555,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"entropy": 0.820938353985548,
|
|
"epoch": 1.5585492227979274,
|
|
"grad_norm": 0.5340061187744141,
|
|
"learning_rate": 0.00010183902226897318,
|
|
"loss": 0.8313,
|
|
"mean_token_accuracy": 0.809401934593916,
|
|
"num_tokens": 19345635.0,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"entropy": 0.8386820316314697,
|
|
"epoch": 1.5626943005181348,
|
|
"grad_norm": 0.672240138053894,
|
|
"learning_rate": 0.00010138215623330438,
|
|
"loss": 0.847,
|
|
"mean_token_accuracy": 0.8100010626018047,
|
|
"num_tokens": 19397330.0,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"entropy": 0.8033540561795235,
|
|
"epoch": 1.5668393782383419,
|
|
"grad_norm": 0.5205754041671753,
|
|
"learning_rate": 0.0001009252613408765,
|
|
"loss": 0.7943,
|
|
"mean_token_accuracy": 0.8171277180314064,
|
|
"num_tokens": 19450201.0,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"entropy": 0.8175629302859306,
|
|
"epoch": 1.5709844559585493,
|
|
"grad_norm": 0.5606580376625061,
|
|
"learning_rate": 0.00010046834713077458,
|
|
"loss": 0.8325,
|
|
"mean_token_accuracy": 0.8103730753064156,
|
|
"num_tokens": 19502473.0,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"entropy": 0.7871093209832907,
|
|
"epoch": 1.5751295336787565,
|
|
"grad_norm": 0.6882584691047668,
|
|
"learning_rate": 0.00010001142314248695,
|
|
"loss": 0.7874,
|
|
"mean_token_accuracy": 0.8198158241808414,
|
|
"num_tokens": 19553258.0,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"entropy": 0.7693354304879904,
|
|
"epoch": 1.5792746113989637,
|
|
"grad_norm": 0.5484011769294739,
|
|
"learning_rate": 9.955449891570613e-05,
|
|
"loss": 0.7779,
|
|
"mean_token_accuracy": 0.8201031595468521,
|
|
"num_tokens": 19602942.0,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"entropy": 0.8091827511787415,
|
|
"epoch": 1.583419689119171,
|
|
"grad_norm": 0.49855074286460876,
|
|
"learning_rate": 9.909758399012953e-05,
|
|
"loss": 0.8099,
|
|
"mean_token_accuracy": 0.8144584089517594,
|
|
"num_tokens": 19656050.0,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"entropy": 0.8198324736207724,
|
|
"epoch": 1.5875647668393782,
|
|
"grad_norm": 0.59278804063797,
|
|
"learning_rate": 9.864068790526054e-05,
|
|
"loss": 0.8377,
|
|
"mean_token_accuracy": 0.8094165749847889,
|
|
"num_tokens": 19706790.0,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"entropy": 0.7976933725178241,
|
|
"epoch": 1.5917098445595856,
|
|
"grad_norm": 0.6395507454872131,
|
|
"learning_rate": 9.818382020020901e-05,
|
|
"loss": 0.8034,
|
|
"mean_token_accuracy": 0.8143622942268849,
|
|
"num_tokens": 19756538.0,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"entropy": 0.7928529791533947,
|
|
"epoch": 1.5958549222797926,
|
|
"grad_norm": 0.5840646624565125,
|
|
"learning_rate": 9.772699041349238e-05,
|
|
"loss": 0.8076,
|
|
"mean_token_accuracy": 0.816139267385006,
|
|
"num_tokens": 19810315.0,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"entropy": 0.8421904467046261,
|
|
"epoch": 1.6,
|
|
"grad_norm": 0.5608034729957581,
|
|
"learning_rate": 9.727020808283638e-05,
|
|
"loss": 0.85,
|
|
"mean_token_accuracy": 0.8074178583920002,
|
|
"num_tokens": 19862739.0,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"entropy": 0.8121896721422672,
|
|
"epoch": 1.604145077720207,
|
|
"grad_norm": 0.6539454460144043,
|
|
"learning_rate": 9.681348274497602e-05,
|
|
"loss": 0.8117,
|
|
"mean_token_accuracy": 0.8129927277565002,
|
|
"num_tokens": 19914136.0,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"entropy": 0.804261787980795,
|
|
"epoch": 1.6082901554404145,
|
|
"grad_norm": 0.5365179777145386,
|
|
"learning_rate": 9.635682393545629e-05,
|
|
"loss": 0.8052,
|
|
"mean_token_accuracy": 0.8143087387084961,
|
|
"num_tokens": 19966215.0,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"entropy": 0.8292149394750595,
|
|
"epoch": 1.6124352331606218,
|
|
"grad_norm": 0.577548623085022,
|
|
"learning_rate": 9.590024118843332e-05,
|
|
"loss": 0.8248,
|
|
"mean_token_accuracy": 0.810042142868042,
|
|
"num_tokens": 20019259.0,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"entropy": 0.7938210744410753,
|
|
"epoch": 1.616580310880829,
|
|
"grad_norm": 0.5613548755645752,
|
|
"learning_rate": 9.544374403647512e-05,
|
|
"loss": 0.8026,
|
|
"mean_token_accuracy": 0.8159350425004959,
|
|
"num_tokens": 20073659.0,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"entropy": 0.825110399723053,
|
|
"epoch": 1.6207253886010364,
|
|
"grad_norm": 0.5206952095031738,
|
|
"learning_rate": 9.498734201036267e-05,
|
|
"loss": 0.8231,
|
|
"mean_token_accuracy": 0.81042909771204,
|
|
"num_tokens": 20126433.0,
|
|
"step": 7820
|
|
},
|
|
{
|
|
"entropy": 0.7997011609375477,
|
|
"epoch": 1.6248704663212434,
|
|
"grad_norm": 0.6118007898330688,
|
|
"learning_rate": 9.453104463889097e-05,
|
|
"loss": 0.8103,
|
|
"mean_token_accuracy": 0.8118245296180249,
|
|
"num_tokens": 20177818.0,
|
|
"step": 7840
|
|
},
|
|
{
|
|
"entropy": 0.8241588916629553,
|
|
"epoch": 1.6290155440414509,
|
|
"grad_norm": 0.6169880628585815,
|
|
"learning_rate": 9.407486144866982e-05,
|
|
"loss": 0.8345,
|
|
"mean_token_accuracy": 0.8105017945170403,
|
|
"num_tokens": 20227713.0,
|
|
"step": 7860
|
|
},
|
|
{
|
|
"entropy": 0.8396770048886537,
|
|
"epoch": 1.6331606217616579,
|
|
"grad_norm": 0.625164270401001,
|
|
"learning_rate": 9.361880196392541e-05,
|
|
"loss": 0.839,
|
|
"mean_token_accuracy": 0.8064838156104088,
|
|
"num_tokens": 20278266.0,
|
|
"step": 7880
|
|
},
|
|
{
|
|
"entropy": 0.8021301727741956,
|
|
"epoch": 1.6373056994818653,
|
|
"grad_norm": 0.5141489505767822,
|
|
"learning_rate": 9.316287570630098e-05,
|
|
"loss": 0.8076,
|
|
"mean_token_accuracy": 0.8155386179685593,
|
|
"num_tokens": 20329177.0,
|
|
"step": 7900
|
|
},
|
|
{
|
|
"entropy": 0.8296167310327291,
|
|
"epoch": 1.6414507772020726,
|
|
"grad_norm": 0.5505626797676086,
|
|
"learning_rate": 9.270709219465842e-05,
|
|
"loss": 0.831,
|
|
"mean_token_accuracy": 0.8111650675535202,
|
|
"num_tokens": 20379258.0,
|
|
"step": 7920
|
|
},
|
|
{
|
|
"entropy": 0.8242552451789379,
|
|
"epoch": 1.6455958549222798,
|
|
"grad_norm": 0.5838463306427002,
|
|
"learning_rate": 9.22514609448792e-05,
|
|
"loss": 0.8309,
|
|
"mean_token_accuracy": 0.814516019821167,
|
|
"num_tokens": 20428945.0,
|
|
"step": 7940
|
|
},
|
|
{
|
|
"entropy": 0.7963455855846405,
|
|
"epoch": 1.649740932642487,
|
|
"grad_norm": 0.5433127880096436,
|
|
"learning_rate": 9.179599146966598e-05,
|
|
"loss": 0.8086,
|
|
"mean_token_accuracy": 0.8120452761650085,
|
|
"num_tokens": 20480831.0,
|
|
"step": 7960
|
|
},
|
|
{
|
|
"entropy": 0.7983941353857518,
|
|
"epoch": 1.6538860103626942,
|
|
"grad_norm": 0.5573062300682068,
|
|
"learning_rate": 9.134069327834377e-05,
|
|
"loss": 0.7935,
|
|
"mean_token_accuracy": 0.8119946666061878,
|
|
"num_tokens": 20534496.0,
|
|
"step": 7980
|
|
},
|
|
{
|
|
"entropy": 0.8061387918889522,
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 0.6268565058708191,
|
|
"learning_rate": 9.088557587666166e-05,
|
|
"loss": 0.822,
|
|
"mean_token_accuracy": 0.8143616132438183,
|
|
"num_tokens": 20586773.0,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"eval_entropy": 0.8331183724281356,
|
|
"eval_loss": 0.8691816926002502,
|
|
"eval_mean_token_accuracy": 0.8051465577262593,
|
|
"eval_num_tokens": 20586773.0,
|
|
"eval_runtime": 199.1925,
|
|
"eval_samples_per_second": 20.397,
|
|
"eval_steps_per_second": 2.55,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"entropy": 0.8063231449574232,
|
|
"epoch": 1.6621761658031087,
|
|
"grad_norm": 0.6503937840461731,
|
|
"learning_rate": 9.043064876659395e-05,
|
|
"loss": 0.8142,
|
|
"mean_token_accuracy": 0.8132365435361862,
|
|
"num_tokens": 20639113.0,
|
|
"step": 8020
|
|
},
|
|
{
|
|
"entropy": 0.791298009455204,
|
|
"epoch": 1.6663212435233161,
|
|
"grad_norm": 0.5850557088851929,
|
|
"learning_rate": 8.997592144614232e-05,
|
|
"loss": 0.7926,
|
|
"mean_token_accuracy": 0.819820100069046,
|
|
"num_tokens": 20690614.0,
|
|
"step": 8040
|
|
},
|
|
{
|
|
"entropy": 0.8223494410514831,
|
|
"epoch": 1.6704663212435233,
|
|
"grad_norm": 0.6517923474311829,
|
|
"learning_rate": 8.952140340913696e-05,
|
|
"loss": 0.8292,
|
|
"mean_token_accuracy": 0.8117071449756622,
|
|
"num_tokens": 20742718.0,
|
|
"step": 8060
|
|
},
|
|
{
|
|
"entropy": 0.7764448471367359,
|
|
"epoch": 1.6746113989637306,
|
|
"grad_norm": 0.5607268810272217,
|
|
"learning_rate": 8.906710414503882e-05,
|
|
"loss": 0.7809,
|
|
"mean_token_accuracy": 0.8177466176450252,
|
|
"num_tokens": 20794750.0,
|
|
"step": 8080
|
|
},
|
|
{
|
|
"entropy": 0.8049763035029173,
|
|
"epoch": 1.6787564766839378,
|
|
"grad_norm": 0.4749673008918762,
|
|
"learning_rate": 8.861303313874114e-05,
|
|
"loss": 0.8123,
|
|
"mean_token_accuracy": 0.8124122641980648,
|
|
"num_tokens": 20846038.0,
|
|
"step": 8100
|
|
},
|
|
{
|
|
"entropy": 0.765482772141695,
|
|
"epoch": 1.682901554404145,
|
|
"grad_norm": 0.5713556408882141,
|
|
"learning_rate": 8.815919987037173e-05,
|
|
"loss": 0.7701,
|
|
"mean_token_accuracy": 0.8197274900972843,
|
|
"num_tokens": 20897102.0,
|
|
"step": 8120
|
|
},
|
|
{
|
|
"entropy": 0.8134225759655237,
|
|
"epoch": 1.6870466321243525,
|
|
"grad_norm": 0.5361589789390564,
|
|
"learning_rate": 8.770561381509473e-05,
|
|
"loss": 0.8148,
|
|
"mean_token_accuracy": 0.8147925660014153,
|
|
"num_tokens": 20948316.0,
|
|
"step": 8140
|
|
},
|
|
{
|
|
"entropy": 0.8131828509271145,
|
|
"epoch": 1.6911917098445595,
|
|
"grad_norm": 0.7416291236877441,
|
|
"learning_rate": 8.725228444291311e-05,
|
|
"loss": 0.8202,
|
|
"mean_token_accuracy": 0.8145058192312717,
|
|
"num_tokens": 20999357.0,
|
|
"step": 8160
|
|
},
|
|
{
|
|
"entropy": 0.8027627006173134,
|
|
"epoch": 1.695336787564767,
|
|
"grad_norm": 0.5522140264511108,
|
|
"learning_rate": 8.679922121847063e-05,
|
|
"loss": 0.806,
|
|
"mean_token_accuracy": 0.8165438532829284,
|
|
"num_tokens": 21049581.0,
|
|
"step": 8180
|
|
},
|
|
{
|
|
"entropy": 0.8006887208670378,
|
|
"epoch": 1.6994818652849741,
|
|
"grad_norm": 0.5689117312431335,
|
|
"learning_rate": 8.634643360085457e-05,
|
|
"loss": 0.8095,
|
|
"mean_token_accuracy": 0.817089269310236,
|
|
"num_tokens": 21099851.0,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"entropy": 0.797663002833724,
|
|
"epoch": 1.7036269430051814,
|
|
"grad_norm": 0.6648983359336853,
|
|
"learning_rate": 8.589393104339785e-05,
|
|
"loss": 0.8015,
|
|
"mean_token_accuracy": 0.8169760525226593,
|
|
"num_tokens": 21149296.0,
|
|
"step": 8220
|
|
},
|
|
{
|
|
"entropy": 0.8056201867759227,
|
|
"epoch": 1.7077720207253886,
|
|
"grad_norm": 0.642423689365387,
|
|
"learning_rate": 8.544172299348216e-05,
|
|
"loss": 0.812,
|
|
"mean_token_accuracy": 0.810999334603548,
|
|
"num_tokens": 21202075.0,
|
|
"step": 8240
|
|
},
|
|
{
|
|
"entropy": 0.7819869294762611,
|
|
"epoch": 1.7119170984455958,
|
|
"grad_norm": 0.6682912707328796,
|
|
"learning_rate": 8.498981889234016e-05,
|
|
"loss": 0.7891,
|
|
"mean_token_accuracy": 0.8156447999179364,
|
|
"num_tokens": 21256533.0,
|
|
"step": 8260
|
|
},
|
|
{
|
|
"entropy": 0.8356943935155868,
|
|
"epoch": 1.7160621761658033,
|
|
"grad_norm": 0.6799805164337158,
|
|
"learning_rate": 8.453822817485886e-05,
|
|
"loss": 0.8433,
|
|
"mean_token_accuracy": 0.8074328951537609,
|
|
"num_tokens": 21306660.0,
|
|
"step": 8280
|
|
},
|
|
{
|
|
"entropy": 0.8379206340759993,
|
|
"epoch": 1.7202072538860103,
|
|
"grad_norm": 0.6007653474807739,
|
|
"learning_rate": 8.40869602693823e-05,
|
|
"loss": 0.8414,
|
|
"mean_token_accuracy": 0.8075645297765732,
|
|
"num_tokens": 21357659.0,
|
|
"step": 8300
|
|
},
|
|
{
|
|
"entropy": 0.7662032548338175,
|
|
"epoch": 1.7243523316062177,
|
|
"grad_norm": 0.626533031463623,
|
|
"learning_rate": 8.363602459751484e-05,
|
|
"loss": 0.7748,
|
|
"mean_token_accuracy": 0.8235852107405662,
|
|
"num_tokens": 21405296.0,
|
|
"step": 8320
|
|
},
|
|
{
|
|
"entropy": 0.8069826323539019,
|
|
"epoch": 1.7284974093264247,
|
|
"grad_norm": 0.518573522567749,
|
|
"learning_rate": 8.318543057392456e-05,
|
|
"loss": 0.8188,
|
|
"mean_token_accuracy": 0.8129658289253712,
|
|
"num_tokens": 21455035.0,
|
|
"step": 8340
|
|
},
|
|
{
|
|
"entropy": 0.8016500160098076,
|
|
"epoch": 1.7326424870466322,
|
|
"grad_norm": 0.671558678150177,
|
|
"learning_rate": 8.273518760614637e-05,
|
|
"loss": 0.8108,
|
|
"mean_token_accuracy": 0.8182107858359814,
|
|
"num_tokens": 21505216.0,
|
|
"step": 8360
|
|
},
|
|
{
|
|
"entropy": 0.807530952617526,
|
|
"epoch": 1.7367875647668394,
|
|
"grad_norm": 0.6064499616622925,
|
|
"learning_rate": 8.228530509438605e-05,
|
|
"loss": 0.8098,
|
|
"mean_token_accuracy": 0.8161074809730053,
|
|
"num_tokens": 21555851.0,
|
|
"step": 8380
|
|
},
|
|
{
|
|
"entropy": 0.8076930336654187,
|
|
"epoch": 1.7409326424870466,
|
|
"grad_norm": 0.6009045839309692,
|
|
"learning_rate": 8.183579243132355e-05,
|
|
"loss": 0.7954,
|
|
"mean_token_accuracy": 0.8193715222179889,
|
|
"num_tokens": 21603028.0,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"entropy": 0.8195010025054217,
|
|
"epoch": 1.745077720207254,
|
|
"grad_norm": 0.500089704990387,
|
|
"learning_rate": 8.138665900191721e-05,
|
|
"loss": 0.8277,
|
|
"mean_token_accuracy": 0.8070785038173198,
|
|
"num_tokens": 21656324.0,
|
|
"step": 8420
|
|
},
|
|
{
|
|
"entropy": 0.7939011812210083,
|
|
"epoch": 1.749222797927461,
|
|
"grad_norm": 0.5857884287834167,
|
|
"learning_rate": 8.093791418320766e-05,
|
|
"loss": 0.7991,
|
|
"mean_token_accuracy": 0.8177553959190845,
|
|
"num_tokens": 21705745.0,
|
|
"step": 8440
|
|
},
|
|
{
|
|
"entropy": 0.8071985386312008,
|
|
"epoch": 1.7533678756476685,
|
|
"grad_norm": 0.5542448163032532,
|
|
"learning_rate": 8.048956734412213e-05,
|
|
"loss": 0.8103,
|
|
"mean_token_accuracy": 0.8151128917932511,
|
|
"num_tokens": 21757859.0,
|
|
"step": 8460
|
|
},
|
|
{
|
|
"entropy": 0.771114907041192,
|
|
"epoch": 1.7575129533678755,
|
|
"grad_norm": 0.7207659482955933,
|
|
"learning_rate": 8.00416278452787e-05,
|
|
"loss": 0.7719,
|
|
"mean_token_accuracy": 0.8224960990250111,
|
|
"num_tokens": 21806318.0,
|
|
"step": 8480
|
|
},
|
|
{
|
|
"entropy": 0.7873483639210462,
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 0.707039475440979,
|
|
"learning_rate": 7.95941050387911e-05,
|
|
"loss": 0.7864,
|
|
"mean_token_accuracy": 0.8166319273412228,
|
|
"num_tokens": 21857727.0,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"eval_entropy": 0.8191909030901166,
|
|
"eval_loss": 0.8602352142333984,
|
|
"eval_mean_token_accuracy": 0.8063250250938371,
|
|
"eval_num_tokens": 21857727.0,
|
|
"eval_runtime": 199.5985,
|
|
"eval_samples_per_second": 20.356,
|
|
"eval_steps_per_second": 2.545,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"entropy": 0.8139069590717554,
|
|
"epoch": 1.7658031088082902,
|
|
"grad_norm": 0.7291368842124939,
|
|
"learning_rate": 7.914700826807323e-05,
|
|
"loss": 0.8285,
|
|
"mean_token_accuracy": 0.810819711536169,
|
|
"num_tokens": 21907657.0,
|
|
"step": 8520
|
|
},
|
|
{
|
|
"entropy": 0.797875202819705,
|
|
"epoch": 1.7699481865284974,
|
|
"grad_norm": 0.6266984939575195,
|
|
"learning_rate": 7.870034686764427e-05,
|
|
"loss": 0.8044,
|
|
"mean_token_accuracy": 0.8135039299726486,
|
|
"num_tokens": 21960321.0,
|
|
"step": 8540
|
|
},
|
|
{
|
|
"entropy": 0.7745360296219588,
|
|
"epoch": 1.7740932642487046,
|
|
"grad_norm": 0.5916459560394287,
|
|
"learning_rate": 7.825413016293355e-05,
|
|
"loss": 0.7741,
|
|
"mean_token_accuracy": 0.8170041099190712,
|
|
"num_tokens": 22011861.0,
|
|
"step": 8560
|
|
},
|
|
{
|
|
"entropy": 0.7811613772064447,
|
|
"epoch": 1.7782383419689118,
|
|
"grad_norm": 0.5927022099494934,
|
|
"learning_rate": 7.780836747008626e-05,
|
|
"loss": 0.796,
|
|
"mean_token_accuracy": 0.8198545664548874,
|
|
"num_tokens": 22064882.0,
|
|
"step": 8580
|
|
},
|
|
{
|
|
"entropy": 0.8116177171468735,
|
|
"epoch": 1.7823834196891193,
|
|
"grad_norm": 0.5992716550827026,
|
|
"learning_rate": 7.736306809576842e-05,
|
|
"loss": 0.807,
|
|
"mean_token_accuracy": 0.8128923654556275,
|
|
"num_tokens": 22114736.0,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"entropy": 0.798346196487546,
|
|
"epoch": 1.7865284974093263,
|
|
"grad_norm": 0.6956632733345032,
|
|
"learning_rate": 7.691824133697307e-05,
|
|
"loss": 0.813,
|
|
"mean_token_accuracy": 0.8101338863372802,
|
|
"num_tokens": 22167035.0,
|
|
"step": 8620
|
|
},
|
|
{
|
|
"entropy": 0.7884280022233725,
|
|
"epoch": 1.7906735751295337,
|
|
"grad_norm": 0.6857265830039978,
|
|
"learning_rate": 7.647389648082581e-05,
|
|
"loss": 0.7863,
|
|
"mean_token_accuracy": 0.8167264245450496,
|
|
"num_tokens": 22217133.0,
|
|
"step": 8640
|
|
},
|
|
{
|
|
"entropy": 0.816360642015934,
|
|
"epoch": 1.794818652849741,
|
|
"grad_norm": 0.5161186456680298,
|
|
"learning_rate": 7.603004280439115e-05,
|
|
"loss": 0.8211,
|
|
"mean_token_accuracy": 0.811207789182663,
|
|
"num_tokens": 22270418.0,
|
|
"step": 8660
|
|
},
|
|
{
|
|
"entropy": 0.8121461659669876,
|
|
"epoch": 1.7989637305699482,
|
|
"grad_norm": 0.6177785992622375,
|
|
"learning_rate": 7.558668957447862e-05,
|
|
"loss": 0.8195,
|
|
"mean_token_accuracy": 0.814345795661211,
|
|
"num_tokens": 22319452.0,
|
|
"step": 8680
|
|
},
|
|
{
|
|
"entropy": 0.7978278145194053,
|
|
"epoch": 1.8031088082901554,
|
|
"grad_norm": 0.6711212396621704,
|
|
"learning_rate": 7.514384604744945e-05,
|
|
"loss": 0.7942,
|
|
"mean_token_accuracy": 0.8158060103654862,
|
|
"num_tokens": 22371990.0,
|
|
"step": 8700
|
|
},
|
|
{
|
|
"entropy": 0.7888606559485197,
|
|
"epoch": 1.8072538860103626,
|
|
"grad_norm": 0.5402470827102661,
|
|
"learning_rate": 7.470152146902325e-05,
|
|
"loss": 0.8015,
|
|
"mean_token_accuracy": 0.8196404658257961,
|
|
"num_tokens": 22424374.0,
|
|
"step": 8720
|
|
},
|
|
{
|
|
"entropy": 0.815355260670185,
|
|
"epoch": 1.81139896373057,
|
|
"grad_norm": 0.5406270623207092,
|
|
"learning_rate": 7.425972507408489e-05,
|
|
"loss": 0.8253,
|
|
"mean_token_accuracy": 0.8108999341726303,
|
|
"num_tokens": 22476209.0,
|
|
"step": 8740
|
|
},
|
|
{
|
|
"entropy": 0.8312284335494041,
|
|
"epoch": 1.815544041450777,
|
|
"grad_norm": 0.6402141451835632,
|
|
"learning_rate": 7.381846608649198e-05,
|
|
"loss": 0.8377,
|
|
"mean_token_accuracy": 0.8085312664508819,
|
|
"num_tokens": 22528702.0,
|
|
"step": 8760
|
|
},
|
|
{
|
|
"entropy": 0.7654821410775184,
|
|
"epoch": 1.8196891191709845,
|
|
"grad_norm": 0.8056541681289673,
|
|
"learning_rate": 7.337775371888188e-05,
|
|
"loss": 0.7758,
|
|
"mean_token_accuracy": 0.8195617265999318,
|
|
"num_tokens": 22581034.0,
|
|
"step": 8780
|
|
},
|
|
{
|
|
"entropy": 0.805729741603136,
|
|
"epoch": 1.8238341968911918,
|
|
"grad_norm": 0.6341525316238403,
|
|
"learning_rate": 7.293759717247977e-05,
|
|
"loss": 0.8063,
|
|
"mean_token_accuracy": 0.8120145253837109,
|
|
"num_tokens": 22635651.0,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"entropy": 0.7698429528623819,
|
|
"epoch": 1.827979274611399,
|
|
"grad_norm": 0.6195350885391235,
|
|
"learning_rate": 7.249800563690618e-05,
|
|
"loss": 0.7793,
|
|
"mean_token_accuracy": 0.8217138454318047,
|
|
"num_tokens": 22687614.0,
|
|
"step": 8820
|
|
},
|
|
{
|
|
"entropy": 0.7681546911597252,
|
|
"epoch": 1.8321243523316062,
|
|
"grad_norm": 0.6474044322967529,
|
|
"learning_rate": 7.205898828998547e-05,
|
|
"loss": 0.7688,
|
|
"mean_token_accuracy": 0.8173581101000309,
|
|
"num_tokens": 22741534.0,
|
|
"step": 8840
|
|
},
|
|
{
|
|
"entropy": 0.8100545316934585,
|
|
"epoch": 1.8362694300518134,
|
|
"grad_norm": 0.5901336669921875,
|
|
"learning_rate": 7.162055429755392e-05,
|
|
"loss": 0.8156,
|
|
"mean_token_accuracy": 0.813131357729435,
|
|
"num_tokens": 22796467.0,
|
|
"step": 8860
|
|
},
|
|
{
|
|
"entropy": 0.7731993589550257,
|
|
"epoch": 1.8404145077720209,
|
|
"grad_norm": 0.6105223894119263,
|
|
"learning_rate": 7.118271281326857e-05,
|
|
"loss": 0.7993,
|
|
"mean_token_accuracy": 0.8200972005724907,
|
|
"num_tokens": 22847814.0,
|
|
"step": 8880
|
|
},
|
|
{
|
|
"entropy": 0.818979337438941,
|
|
"epoch": 1.8445595854922279,
|
|
"grad_norm": 0.6697989702224731,
|
|
"learning_rate": 7.074547297841597e-05,
|
|
"loss": 0.7947,
|
|
"mean_token_accuracy": 0.8152140490710735,
|
|
"num_tokens": 22900697.0,
|
|
"step": 8900
|
|
},
|
|
{
|
|
"entropy": 0.7861097157001495,
|
|
"epoch": 1.8487046632124353,
|
|
"grad_norm": 0.5355159640312195,
|
|
"learning_rate": 7.030884392172148e-05,
|
|
"loss": 0.7987,
|
|
"mean_token_accuracy": 0.8170608669519425,
|
|
"num_tokens": 22950925.0,
|
|
"step": 8920
|
|
},
|
|
{
|
|
"entropy": 0.7655485037714242,
|
|
"epoch": 1.8528497409326425,
|
|
"grad_norm": 0.6905961632728577,
|
|
"learning_rate": 6.987283475915841e-05,
|
|
"loss": 0.778,
|
|
"mean_token_accuracy": 0.8229250572621822,
|
|
"num_tokens": 23001662.0,
|
|
"step": 8940
|
|
},
|
|
{
|
|
"entropy": 0.7826405435800552,
|
|
"epoch": 1.8569948186528498,
|
|
"grad_norm": 0.670516848564148,
|
|
"learning_rate": 6.943745459375815e-05,
|
|
"loss": 0.7768,
|
|
"mean_token_accuracy": 0.8207223378121853,
|
|
"num_tokens": 23052161.0,
|
|
"step": 8960
|
|
},
|
|
{
|
|
"entropy": 0.7672661807388067,
|
|
"epoch": 1.861139896373057,
|
|
"grad_norm": 0.6080722808837891,
|
|
"learning_rate": 6.900271251541954e-05,
|
|
"loss": 0.7693,
|
|
"mean_token_accuracy": 0.822633370012045,
|
|
"num_tokens": 23103953.0,
|
|
"step": 8980
|
|
},
|
|
{
|
|
"entropy": 0.7867002800107002,
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 0.6100406646728516,
|
|
"learning_rate": 6.856861760071963e-05,
|
|
"loss": 0.7984,
|
|
"mean_token_accuracy": 0.8164176814258098,
|
|
"num_tokens": 23158219.0,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"eval_entropy": 0.8072302305088268,
|
|
"eval_loss": 0.8477412462234497,
|
|
"eval_mean_token_accuracy": 0.8091423881100858,
|
|
"eval_num_tokens": 23158219.0,
|
|
"eval_runtime": 199.3762,
|
|
"eval_samples_per_second": 20.379,
|
|
"eval_steps_per_second": 2.548,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"entropy": 0.7598585288971662,
|
|
"epoch": 1.8694300518134717,
|
|
"grad_norm": 0.6759816408157349,
|
|
"learning_rate": 6.81351789127238e-05,
|
|
"loss": 0.7643,
|
|
"mean_token_accuracy": 0.8221842713654042,
|
|
"num_tokens": 23210056.0,
|
|
"step": 9020
|
|
},
|
|
{
|
|
"entropy": 0.7827660325914622,
|
|
"epoch": 1.8735751295336787,
|
|
"grad_norm": 0.6666163802146912,
|
|
"learning_rate": 6.770240550079676e-05,
|
|
"loss": 0.7831,
|
|
"mean_token_accuracy": 0.8202085882425308,
|
|
"num_tokens": 23260263.0,
|
|
"step": 9040
|
|
},
|
|
{
|
|
"entropy": 0.7963905472308397,
|
|
"epoch": 1.877720207253886,
|
|
"grad_norm": 0.7692875862121582,
|
|
"learning_rate": 6.727030640041348e-05,
|
|
"loss": 0.8143,
|
|
"mean_token_accuracy": 0.8150147773325443,
|
|
"num_tokens": 23312197.0,
|
|
"step": 9060
|
|
},
|
|
{
|
|
"entropy": 0.7809346288442611,
|
|
"epoch": 1.881865284974093,
|
|
"grad_norm": 0.6786936521530151,
|
|
"learning_rate": 6.683889063297064e-05,
|
|
"loss": 0.7762,
|
|
"mean_token_accuracy": 0.8172829367220402,
|
|
"num_tokens": 23368158.0,
|
|
"step": 9080
|
|
},
|
|
{
|
|
"entropy": 0.7630620401352644,
|
|
"epoch": 1.8860103626943006,
|
|
"grad_norm": 0.5788915157318115,
|
|
"learning_rate": 6.640816720559822e-05,
|
|
"loss": 0.7692,
|
|
"mean_token_accuracy": 0.8216680750250817,
|
|
"num_tokens": 23418983.0,
|
|
"step": 9100
|
|
},
|
|
{
|
|
"entropy": 0.776552353054285,
|
|
"epoch": 1.8901554404145078,
|
|
"grad_norm": 0.6300600171089172,
|
|
"learning_rate": 6.597814511097156e-05,
|
|
"loss": 0.7759,
|
|
"mean_token_accuracy": 0.8195004962384701,
|
|
"num_tokens": 23473575.0,
|
|
"step": 9120
|
|
},
|
|
{
|
|
"entropy": 0.7825426395982504,
|
|
"epoch": 1.894300518134715,
|
|
"grad_norm": 0.5763329863548279,
|
|
"learning_rate": 6.55488333271233e-05,
|
|
"loss": 0.7932,
|
|
"mean_token_accuracy": 0.8186860151588917,
|
|
"num_tokens": 23525829.0,
|
|
"step": 9140
|
|
},
|
|
{
|
|
"entropy": 0.7654791139066219,
|
|
"epoch": 1.8984455958549222,
|
|
"grad_norm": 0.6289655566215515,
|
|
"learning_rate": 6.512024081725649e-05,
|
|
"loss": 0.7669,
|
|
"mean_token_accuracy": 0.8211936444044113,
|
|
"num_tokens": 23577826.0,
|
|
"step": 9160
|
|
},
|
|
{
|
|
"entropy": 0.7603258680552244,
|
|
"epoch": 1.9025906735751295,
|
|
"grad_norm": 0.6766859889030457,
|
|
"learning_rate": 6.469237652955682e-05,
|
|
"loss": 0.7576,
|
|
"mean_token_accuracy": 0.8232448793947696,
|
|
"num_tokens": 23628129.0,
|
|
"step": 9180
|
|
},
|
|
{
|
|
"entropy": 0.7688217446208,
|
|
"epoch": 1.906735751295337,
|
|
"grad_norm": 0.8477441668510437,
|
|
"learning_rate": 6.426524939700626e-05,
|
|
"loss": 0.7773,
|
|
"mean_token_accuracy": 0.8188046269118786,
|
|
"num_tokens": 23679531.0,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"entropy": 0.7668525300920009,
|
|
"epoch": 1.910880829015544,
|
|
"grad_norm": 0.6910899877548218,
|
|
"learning_rate": 6.383886833719638e-05,
|
|
"loss": 0.7705,
|
|
"mean_token_accuracy": 0.8227375738322735,
|
|
"num_tokens": 23732146.0,
|
|
"step": 9220
|
|
},
|
|
{
|
|
"entropy": 0.7733695317059756,
|
|
"epoch": 1.9150259067357513,
|
|
"grad_norm": 0.5750974416732788,
|
|
"learning_rate": 6.341324225214219e-05,
|
|
"loss": 0.7751,
|
|
"mean_token_accuracy": 0.8216360554099083,
|
|
"num_tokens": 23782259.0,
|
|
"step": 9240
|
|
},
|
|
{
|
|
"entropy": 0.7474736385047436,
|
|
"epoch": 1.9191709844559586,
|
|
"grad_norm": 0.7330994009971619,
|
|
"learning_rate": 6.298838002809626e-05,
|
|
"loss": 0.745,
|
|
"mean_token_accuracy": 0.8256187126040458,
|
|
"num_tokens": 23836311.0,
|
|
"step": 9260
|
|
},
|
|
{
|
|
"entropy": 0.7911654248833656,
|
|
"epoch": 1.9233160621761658,
|
|
"grad_norm": 0.7114558219909668,
|
|
"learning_rate": 6.256429053536319e-05,
|
|
"loss": 0.8006,
|
|
"mean_token_accuracy": 0.8159124657511712,
|
|
"num_tokens": 23889732.0,
|
|
"step": 9280
|
|
},
|
|
{
|
|
"entropy": 0.7705597594380379,
|
|
"epoch": 1.927461139896373,
|
|
"grad_norm": 0.674951434135437,
|
|
"learning_rate": 6.214098262811455e-05,
|
|
"loss": 0.7772,
|
|
"mean_token_accuracy": 0.8165130980312825,
|
|
"num_tokens": 23943183.0,
|
|
"step": 9300
|
|
},
|
|
{
|
|
"entropy": 0.813532643392682,
|
|
"epoch": 1.9316062176165802,
|
|
"grad_norm": 0.6601293683052063,
|
|
"learning_rate": 6.171846514420376e-05,
|
|
"loss": 0.8134,
|
|
"mean_token_accuracy": 0.8136617749929428,
|
|
"num_tokens": 23992601.0,
|
|
"step": 9320
|
|
},
|
|
{
|
|
"entropy": 0.7866529859602451,
|
|
"epoch": 1.9357512953367877,
|
|
"grad_norm": 0.7155274748802185,
|
|
"learning_rate": 6.129674690498186e-05,
|
|
"loss": 0.8018,
|
|
"mean_token_accuracy": 0.8170430913567543,
|
|
"num_tokens": 24043063.0,
|
|
"step": 9340
|
|
},
|
|
{
|
|
"entropy": 0.8019247651100159,
|
|
"epoch": 1.9398963730569947,
|
|
"grad_norm": 0.6180950403213501,
|
|
"learning_rate": 6.087583671511308e-05,
|
|
"loss": 0.8102,
|
|
"mean_token_accuracy": 0.8140971504151822,
|
|
"num_tokens": 24094348.0,
|
|
"step": 9360
|
|
},
|
|
{
|
|
"entropy": 0.7875942759215832,
|
|
"epoch": 1.9440414507772021,
|
|
"grad_norm": 0.6453153491020203,
|
|
"learning_rate": 6.045574336239126e-05,
|
|
"loss": 0.7913,
|
|
"mean_token_accuracy": 0.8203319162130356,
|
|
"num_tokens": 24144708.0,
|
|
"step": 9380
|
|
},
|
|
{
|
|
"entropy": 0.7748208403587341,
|
|
"epoch": 1.9481865284974094,
|
|
"grad_norm": 0.908160388469696,
|
|
"learning_rate": 6.003647561755612e-05,
|
|
"loss": 0.7759,
|
|
"mean_token_accuracy": 0.8210457645356655,
|
|
"num_tokens": 24196754.0,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"entropy": 0.7494080271571875,
|
|
"epoch": 1.9523316062176166,
|
|
"grad_norm": 0.6796506643295288,
|
|
"learning_rate": 5.961804223411042e-05,
|
|
"loss": 0.7599,
|
|
"mean_token_accuracy": 0.8223151490092278,
|
|
"num_tokens": 24249843.0,
|
|
"step": 9420
|
|
},
|
|
{
|
|
"entropy": 0.766802679002285,
|
|
"epoch": 1.9564766839378238,
|
|
"grad_norm": 0.6993657946586609,
|
|
"learning_rate": 5.9200451948136924e-05,
|
|
"loss": 0.7806,
|
|
"mean_token_accuracy": 0.8238598793745041,
|
|
"num_tokens": 24301242.0,
|
|
"step": 9440
|
|
},
|
|
{
|
|
"entropy": 0.7636256385594606,
|
|
"epoch": 1.960621761658031,
|
|
"grad_norm": 0.5855976343154907,
|
|
"learning_rate": 5.8783713478116285e-05,
|
|
"loss": 0.7693,
|
|
"mean_token_accuracy": 0.8205041140317917,
|
|
"num_tokens": 24354158.0,
|
|
"step": 9460
|
|
},
|
|
{
|
|
"entropy": 0.7989416006952524,
|
|
"epoch": 1.9647668393782385,
|
|
"grad_norm": 0.612602174282074,
|
|
"learning_rate": 5.836783552474473e-05,
|
|
"loss": 0.8043,
|
|
"mean_token_accuracy": 0.8161830216646194,
|
|
"num_tokens": 24404992.0,
|
|
"step": 9480
|
|
},
|
|
{
|
|
"entropy": 0.7820379823446274,
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 0.6877280473709106,
|
|
"learning_rate": 5.7952826770752666e-05,
|
|
"loss": 0.7815,
|
|
"mean_token_accuracy": 0.8213707759976387,
|
|
"num_tokens": 24457830.0,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"eval_entropy": 0.7865860647573246,
|
|
"eval_loss": 0.8369172215461731,
|
|
"eval_mean_token_accuracy": 0.81087670058716,
|
|
"eval_num_tokens": 24457830.0,
|
|
"eval_runtime": 199.0099,
|
|
"eval_samples_per_second": 20.416,
|
|
"eval_steps_per_second": 2.553,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"entropy": 0.7840311020612717,
|
|
"epoch": 1.973056994818653,
|
|
"grad_norm": 0.591534435749054,
|
|
"learning_rate": 5.753869588072325e-05,
|
|
"loss": 0.7981,
|
|
"mean_token_accuracy": 0.8175319261848927,
|
|
"num_tokens": 24508627.0,
|
|
"step": 9520
|
|
},
|
|
{
|
|
"entropy": 0.7821144435554743,
|
|
"epoch": 1.9772020725388602,
|
|
"grad_norm": 0.6599027514457703,
|
|
"learning_rate": 5.7125451500911595e-05,
|
|
"loss": 0.7844,
|
|
"mean_token_accuracy": 0.8220554009079933,
|
|
"num_tokens": 24560779.0,
|
|
"step": 9540
|
|
},
|
|
{
|
|
"entropy": 0.7700827412307263,
|
|
"epoch": 1.9813471502590674,
|
|
"grad_norm": 0.6830772161483765,
|
|
"learning_rate": 5.6713102259064054e-05,
|
|
"loss": 0.7743,
|
|
"mean_token_accuracy": 0.823592484742403,
|
|
"num_tokens": 24611482.0,
|
|
"step": 9560
|
|
},
|
|
{
|
|
"entropy": 0.7665214616805315,
|
|
"epoch": 1.9854922279792746,
|
|
"grad_norm": 0.6822047233581543,
|
|
"learning_rate": 5.6301656764238366e-05,
|
|
"loss": 0.773,
|
|
"mean_token_accuracy": 0.8229187518358231,
|
|
"num_tokens": 24662747.0,
|
|
"step": 9580
|
|
},
|
|
{
|
|
"entropy": 0.8019340179860592,
|
|
"epoch": 1.9896373056994818,
|
|
"grad_norm": 0.7325703501701355,
|
|
"learning_rate": 5.5891123606623605e-05,
|
|
"loss": 0.8033,
|
|
"mean_token_accuracy": 0.8131180450320243,
|
|
"num_tokens": 24715245.0,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"entropy": 0.7869971726089716,
|
|
"epoch": 1.9937823834196893,
|
|
"grad_norm": 0.6309528350830078,
|
|
"learning_rate": 5.548151135736126e-05,
|
|
"loss": 0.7781,
|
|
"mean_token_accuracy": 0.8171296060085297,
|
|
"num_tokens": 24769125.0,
|
|
"step": 9620
|
|
},
|
|
{
|
|
"entropy": 0.7535249210894108,
|
|
"epoch": 1.9979274611398963,
|
|
"grad_norm": 0.6659754514694214,
|
|
"learning_rate": 5.50728285683658e-05,
|
|
"loss": 0.7572,
|
|
"mean_token_accuracy": 0.8270934030413628,
|
|
"num_tokens": 24817793.0,
|
|
"step": 9640
|
|
},
|
|
{
|
|
"entropy": 0.7477992571890354,
|
|
"epoch": 2.0020725388601037,
|
|
"grad_norm": 0.696112871170044,
|
|
"learning_rate": 5.4665083772146385e-05,
|
|
"loss": 0.7213,
|
|
"mean_token_accuracy": 0.8279822461307049,
|
|
"num_tokens": 24867726.0,
|
|
"step": 9660
|
|
},
|
|
{
|
|
"entropy": 0.7088031362742185,
|
|
"epoch": 2.0062176165803107,
|
|
"grad_norm": 0.8436769247055054,
|
|
"learning_rate": 5.425828548162879e-05,
|
|
"loss": 0.6939,
|
|
"mean_token_accuracy": 0.8358024321496487,
|
|
"num_tokens": 24921316.0,
|
|
"step": 9680
|
|
},
|
|
{
|
|
"entropy": 0.6361858382821083,
|
|
"epoch": 2.010362694300518,
|
|
"grad_norm": 0.6807072758674622,
|
|
"learning_rate": 5.3852442189977504e-05,
|
|
"loss": 0.6268,
|
|
"mean_token_accuracy": 0.8482733227312564,
|
|
"num_tokens": 24974266.0,
|
|
"step": 9700
|
|
},
|
|
{
|
|
"entropy": 0.6577429339289665,
|
|
"epoch": 2.014507772020725,
|
|
"grad_norm": 0.7923240661621094,
|
|
"learning_rate": 5.344756237041855e-05,
|
|
"loss": 0.6503,
|
|
"mean_token_accuracy": 0.8418660826981068,
|
|
"num_tokens": 25028181.0,
|
|
"step": 9720
|
|
},
|
|
{
|
|
"entropy": 0.6667120568454266,
|
|
"epoch": 2.0186528497409326,
|
|
"grad_norm": 0.9293539524078369,
|
|
"learning_rate": 5.3043654476062375e-05,
|
|
"loss": 0.6562,
|
|
"mean_token_accuracy": 0.8416152134537697,
|
|
"num_tokens": 25079404.0,
|
|
"step": 9740
|
|
},
|
|
{
|
|
"entropy": 0.6501641254872084,
|
|
"epoch": 2.02279792746114,
|
|
"grad_norm": 0.6982747912406921,
|
|
"learning_rate": 5.264072693972767e-05,
|
|
"loss": 0.6432,
|
|
"mean_token_accuracy": 0.8453519389033317,
|
|
"num_tokens": 25131819.0,
|
|
"step": 9760
|
|
},
|
|
{
|
|
"entropy": 0.6610196366906166,
|
|
"epoch": 2.026943005181347,
|
|
"grad_norm": 0.6927458047866821,
|
|
"learning_rate": 5.2238788173764966e-05,
|
|
"loss": 0.6638,
|
|
"mean_token_accuracy": 0.8402243196964264,
|
|
"num_tokens": 25183851.0,
|
|
"step": 9780
|
|
},
|
|
{
|
|
"entropy": 0.7120028961449861,
|
|
"epoch": 2.0310880829015545,
|
|
"grad_norm": 0.9179397821426392,
|
|
"learning_rate": 5.18378465698813e-05,
|
|
"loss": 0.7068,
|
|
"mean_token_accuracy": 0.8339078485965729,
|
|
"num_tokens": 25234249.0,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"entropy": 0.6806752648204565,
|
|
"epoch": 2.0352331606217615,
|
|
"grad_norm": 0.65874844789505,
|
|
"learning_rate": 5.143791049896485e-05,
|
|
"loss": 0.6707,
|
|
"mean_token_accuracy": 0.841441460698843,
|
|
"num_tokens": 25285865.0,
|
|
"step": 9820
|
|
},
|
|
{
|
|
"entropy": 0.671997120976448,
|
|
"epoch": 2.039378238341969,
|
|
"grad_norm": 0.7928804159164429,
|
|
"learning_rate": 5.103898831091025e-05,
|
|
"loss": 0.6727,
|
|
"mean_token_accuracy": 0.8379430681467056,
|
|
"num_tokens": 25338736.0,
|
|
"step": 9840
|
|
},
|
|
{
|
|
"entropy": 0.707481624558568,
|
|
"epoch": 2.043523316062176,
|
|
"grad_norm": 0.6195916533470154,
|
|
"learning_rate": 5.0641088334444074e-05,
|
|
"loss": 0.7054,
|
|
"mean_token_accuracy": 0.831365679949522,
|
|
"num_tokens": 25392940.0,
|
|
"step": 9860
|
|
},
|
|
{
|
|
"entropy": 0.6877768516540528,
|
|
"epoch": 2.0476683937823834,
|
|
"grad_norm": 0.9123510122299194,
|
|
"learning_rate": 5.024421887695121e-05,
|
|
"loss": 0.6796,
|
|
"mean_token_accuracy": 0.8372841715812683,
|
|
"num_tokens": 25444740.0,
|
|
"step": 9880
|
|
},
|
|
{
|
|
"entropy": 0.6668030392378569,
|
|
"epoch": 2.051813471502591,
|
|
"grad_norm": 0.7575181722640991,
|
|
"learning_rate": 4.9848388224301266e-05,
|
|
"loss": 0.6751,
|
|
"mean_token_accuracy": 0.8396980598568916,
|
|
"num_tokens": 25495066.0,
|
|
"step": 9900
|
|
},
|
|
{
|
|
"entropy": 0.6915552038699389,
|
|
"epoch": 2.055958549222798,
|
|
"grad_norm": 0.8226049542427063,
|
|
"learning_rate": 4.945360464067566e-05,
|
|
"loss": 0.688,
|
|
"mean_token_accuracy": 0.8372002221643925,
|
|
"num_tokens": 25545692.0,
|
|
"step": 9920
|
|
},
|
|
{
|
|
"entropy": 0.6767883218824864,
|
|
"epoch": 2.0601036269430053,
|
|
"grad_norm": 0.8825149536132812,
|
|
"learning_rate": 4.9059876368394865e-05,
|
|
"loss": 0.6678,
|
|
"mean_token_accuracy": 0.8408191554248333,
|
|
"num_tokens": 25596578.0,
|
|
"step": 9940
|
|
},
|
|
{
|
|
"entropy": 0.685994666814804,
|
|
"epoch": 2.0642487046632123,
|
|
"grad_norm": 0.8278037309646606,
|
|
"learning_rate": 4.866721162774668e-05,
|
|
"loss": 0.6869,
|
|
"mean_token_accuracy": 0.8369470968842506,
|
|
"num_tokens": 25648744.0,
|
|
"step": 9960
|
|
},
|
|
{
|
|
"entropy": 0.627468366548419,
|
|
"epoch": 2.0683937823834198,
|
|
"grad_norm": 0.9138433933258057,
|
|
"learning_rate": 4.827561861681424e-05,
|
|
"loss": 0.6113,
|
|
"mean_token_accuracy": 0.8540245234966278,
|
|
"num_tokens": 25698194.0,
|
|
"step": 9980
|
|
},
|
|
{
|
|
"entropy": 0.6912321873009205,
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 0.9565804600715637,
|
|
"learning_rate": 4.788510551130517e-05,
|
|
"loss": 0.6889,
|
|
"mean_token_accuracy": 0.8333763852715492,
|
|
"num_tokens": 25750022.0,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"eval_entropy": 0.715560236021759,
|
|
"eval_loss": 0.8467245101928711,
|
|
"eval_mean_token_accuracy": 0.811145214935926,
|
|
"eval_num_tokens": 25750022.0,
|
|
"eval_runtime": 198.9613,
|
|
"eval_samples_per_second": 20.421,
|
|
"eval_steps_per_second": 2.553,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"entropy": 0.6547453593462705,
|
|
"epoch": 2.076683937823834,
|
|
"grad_norm": 0.8177805542945862,
|
|
"learning_rate": 4.749568046438066e-05,
|
|
"loss": 0.652,
|
|
"mean_token_accuracy": 0.8448099605739117,
|
|
"num_tokens": 25800064.0,
|
|
"step": 10020
|
|
},
|
|
{
|
|
"entropy": 0.6505116693675518,
|
|
"epoch": 2.0808290155440416,
|
|
"grad_norm": 0.820354700088501,
|
|
"learning_rate": 4.710735160648544e-05,
|
|
"loss": 0.643,
|
|
"mean_token_accuracy": 0.8450466863811016,
|
|
"num_tokens": 25850195.0,
|
|
"step": 10040
|
|
},
|
|
{
|
|
"entropy": 0.6796498652547598,
|
|
"epoch": 2.0849740932642487,
|
|
"grad_norm": 0.7562269568443298,
|
|
"learning_rate": 4.672012704517775e-05,
|
|
"loss": 0.6733,
|
|
"mean_token_accuracy": 0.8402103535830975,
|
|
"num_tokens": 25900471.0,
|
|
"step": 10060
|
|
},
|
|
{
|
|
"entropy": 0.6429915092885494,
|
|
"epoch": 2.089119170984456,
|
|
"grad_norm": 0.8847933411598206,
|
|
"learning_rate": 4.6334014864960385e-05,
|
|
"loss": 0.6447,
|
|
"mean_token_accuracy": 0.8441250063478947,
|
|
"num_tokens": 25951630.0,
|
|
"step": 10080
|
|
},
|
|
{
|
|
"entropy": 0.6648316029459238,
|
|
"epoch": 2.093264248704663,
|
|
"grad_norm": 0.7468392848968506,
|
|
"learning_rate": 4.594902312711177e-05,
|
|
"loss": 0.6566,
|
|
"mean_token_accuracy": 0.8426650799810886,
|
|
"num_tokens": 26003997.0,
|
|
"step": 10100
|
|
},
|
|
{
|
|
"entropy": 0.6597115486860275,
|
|
"epoch": 2.0974093264248705,
|
|
"grad_norm": 0.8459360599517822,
|
|
"learning_rate": 4.5565159869517535e-05,
|
|
"loss": 0.6502,
|
|
"mean_token_accuracy": 0.8434385687112809,
|
|
"num_tokens": 26055270.0,
|
|
"step": 10120
|
|
},
|
|
{
|
|
"entropy": 0.6247927419841289,
|
|
"epoch": 2.1015544041450775,
|
|
"grad_norm": 0.8427799344062805,
|
|
"learning_rate": 4.5182433106502955e-05,
|
|
"loss": 0.6247,
|
|
"mean_token_accuracy": 0.8491715729236603,
|
|
"num_tokens": 26106133.0,
|
|
"step": 10140
|
|
},
|
|
{
|
|
"entropy": 0.6583450727164746,
|
|
"epoch": 2.105699481865285,
|
|
"grad_norm": 0.6963675022125244,
|
|
"learning_rate": 4.480085082866535e-05,
|
|
"loss": 0.6568,
|
|
"mean_token_accuracy": 0.8423917785286903,
|
|
"num_tokens": 26158488.0,
|
|
"step": 10160
|
|
},
|
|
{
|
|
"entropy": 0.694509357213974,
|
|
"epoch": 2.1098445595854924,
|
|
"grad_norm": 0.7538948655128479,
|
|
"learning_rate": 4.442042100270753e-05,
|
|
"loss": 0.6997,
|
|
"mean_token_accuracy": 0.8305174142122269,
|
|
"num_tokens": 26209379.0,
|
|
"step": 10180
|
|
},
|
|
{
|
|
"entropy": 0.6648438408970833,
|
|
"epoch": 2.1139896373056994,
|
|
"grad_norm": 0.7609308362007141,
|
|
"learning_rate": 4.4041151571271234e-05,
|
|
"loss": 0.6648,
|
|
"mean_token_accuracy": 0.84227564483881,
|
|
"num_tokens": 26260734.0,
|
|
"step": 10200
|
|
},
|
|
{
|
|
"entropy": 0.6773408968001604,
|
|
"epoch": 2.118134715025907,
|
|
"grad_norm": 0.8334757685661316,
|
|
"learning_rate": 4.366305045277148e-05,
|
|
"loss": 0.6743,
|
|
"mean_token_accuracy": 0.8413719512522221,
|
|
"num_tokens": 26313221.0,
|
|
"step": 10220
|
|
},
|
|
{
|
|
"entropy": 0.6706547629088163,
|
|
"epoch": 2.122279792746114,
|
|
"grad_norm": 0.7924262881278992,
|
|
"learning_rate": 4.328612554123106e-05,
|
|
"loss": 0.6646,
|
|
"mean_token_accuracy": 0.8384681299328804,
|
|
"num_tokens": 26363940.0,
|
|
"step": 10240
|
|
},
|
|
{
|
|
"entropy": 0.6400670412927866,
|
|
"epoch": 2.1264248704663213,
|
|
"grad_norm": 0.8543500304222107,
|
|
"learning_rate": 4.29103847061159e-05,
|
|
"loss": 0.6307,
|
|
"mean_token_accuracy": 0.847849340736866,
|
|
"num_tokens": 26414819.0,
|
|
"step": 10260
|
|
},
|
|
{
|
|
"entropy": 0.6826432276517153,
|
|
"epoch": 2.1305699481865283,
|
|
"grad_norm": 0.7541687488555908,
|
|
"learning_rate": 4.25358357921707e-05,
|
|
"loss": 0.6856,
|
|
"mean_token_accuracy": 0.836744848638773,
|
|
"num_tokens": 26466266.0,
|
|
"step": 10280
|
|
},
|
|
{
|
|
"entropy": 0.6441164668649435,
|
|
"epoch": 2.134715025906736,
|
|
"grad_norm": 0.8502160310745239,
|
|
"learning_rate": 4.216248661925514e-05,
|
|
"loss": 0.6411,
|
|
"mean_token_accuracy": 0.842371879518032,
|
|
"num_tokens": 26515981.0,
|
|
"step": 10300
|
|
},
|
|
{
|
|
"entropy": 0.6789159405976534,
|
|
"epoch": 2.138860103626943,
|
|
"grad_norm": 0.8485208749771118,
|
|
"learning_rate": 4.179034498218053e-05,
|
|
"loss": 0.6691,
|
|
"mean_token_accuracy": 0.8399071894586086,
|
|
"num_tokens": 26566563.0,
|
|
"step": 10320
|
|
},
|
|
{
|
|
"entropy": 0.6637649640440941,
|
|
"epoch": 2.1430051813471502,
|
|
"grad_norm": 0.83147132396698,
|
|
"learning_rate": 4.141941865054731e-05,
|
|
"loss": 0.6709,
|
|
"mean_token_accuracy": 0.8435050114989281,
|
|
"num_tokens": 26619271.0,
|
|
"step": 10340
|
|
},
|
|
{
|
|
"entropy": 0.6490921720862388,
|
|
"epoch": 2.1471502590673577,
|
|
"grad_norm": 0.8916320204734802,
|
|
"learning_rate": 4.104971536858253e-05,
|
|
"loss": 0.6483,
|
|
"mean_token_accuracy": 0.8482758186757564,
|
|
"num_tokens": 26670431.0,
|
|
"step": 10360
|
|
},
|
|
{
|
|
"entropy": 0.6531984589993953,
|
|
"epoch": 2.1512953367875647,
|
|
"grad_norm": 0.8996282815933228,
|
|
"learning_rate": 4.0681242854978415e-05,
|
|
"loss": 0.6474,
|
|
"mean_token_accuracy": 0.8454214997589589,
|
|
"num_tokens": 26721831.0,
|
|
"step": 10380
|
|
},
|
|
{
|
|
"entropy": 0.6747349046170712,
|
|
"epoch": 2.155440414507772,
|
|
"grad_norm": 0.9146754145622253,
|
|
"learning_rate": 4.031400880273111e-05,
|
|
"loss": 0.6692,
|
|
"mean_token_accuracy": 0.8392128713428975,
|
|
"num_tokens": 26773142.0,
|
|
"step": 10400
|
|
},
|
|
{
|
|
"entropy": 0.6591500837355853,
|
|
"epoch": 2.159585492227979,
|
|
"grad_norm": 0.9349919557571411,
|
|
"learning_rate": 3.994802087898009e-05,
|
|
"loss": 0.6579,
|
|
"mean_token_accuracy": 0.8448481574654579,
|
|
"num_tokens": 26824348.0,
|
|
"step": 10420
|
|
},
|
|
{
|
|
"entropy": 0.6684356812387705,
|
|
"epoch": 2.1637305699481866,
|
|
"grad_norm": 0.738776683807373,
|
|
"learning_rate": 3.958328672484794e-05,
|
|
"loss": 0.6625,
|
|
"mean_token_accuracy": 0.840371286869049,
|
|
"num_tokens": 26875634.0,
|
|
"step": 10440
|
|
},
|
|
{
|
|
"entropy": 0.6448073945939541,
|
|
"epoch": 2.1678756476683936,
|
|
"grad_norm": 0.8352062702178955,
|
|
"learning_rate": 3.9219813955281124e-05,
|
|
"loss": 0.6329,
|
|
"mean_token_accuracy": 0.8479115769267083,
|
|
"num_tokens": 26924294.0,
|
|
"step": 10460
|
|
},
|
|
{
|
|
"entropy": 0.6653367314487696,
|
|
"epoch": 2.172020725388601,
|
|
"grad_norm": 0.8592702150344849,
|
|
"learning_rate": 3.8857610158890736e-05,
|
|
"loss": 0.6735,
|
|
"mean_token_accuracy": 0.841375395655632,
|
|
"num_tokens": 26973603.0,
|
|
"step": 10480
|
|
},
|
|
{
|
|
"entropy": 0.6571714913472533,
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 0.8268009424209595,
|
|
"learning_rate": 3.849668289779418e-05,
|
|
"loss": 0.6488,
|
|
"mean_token_accuracy": 0.8468234062194824,
|
|
"num_tokens": 27024915.0,
|
|
"step": 10500
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"eval_entropy": 0.7116222533536708,
|
|
"eval_loss": 0.8363224267959595,
|
|
"eval_mean_token_accuracy": 0.8132013156423419,
|
|
"eval_num_tokens": 27024915.0,
|
|
"eval_runtime": 199.6557,
|
|
"eval_samples_per_second": 20.35,
|
|
"eval_steps_per_second": 2.544,
|
|
"step": 10500
|
|
},
|
|
{
|
|
"entropy": 0.6502834629267454,
|
|
"epoch": 2.1803108808290155,
|
|
"grad_norm": 0.7395395636558533,
|
|
"learning_rate": 3.813703970745724e-05,
|
|
"loss": 0.65,
|
|
"mean_token_accuracy": 0.8468669965863228,
|
|
"num_tokens": 27078794.0,
|
|
"step": 10520
|
|
},
|
|
{
|
|
"entropy": 0.6460639111697674,
|
|
"epoch": 2.184455958549223,
|
|
"grad_norm": 0.7782710790634155,
|
|
"learning_rate": 3.777868809653675e-05,
|
|
"loss": 0.6401,
|
|
"mean_token_accuracy": 0.8450980074703693,
|
|
"num_tokens": 27128749.0,
|
|
"step": 10540
|
|
},
|
|
{
|
|
"entropy": 0.6804705254733563,
|
|
"epoch": 2.18860103626943,
|
|
"grad_norm": 0.8695915937423706,
|
|
"learning_rate": 3.742163554672392e-05,
|
|
"loss": 0.6795,
|
|
"mean_token_accuracy": 0.8381233468651772,
|
|
"num_tokens": 27181625.0,
|
|
"step": 10560
|
|
},
|
|
{
|
|
"entropy": 0.6647137206047773,
|
|
"epoch": 2.1927461139896374,
|
|
"grad_norm": 0.6998477578163147,
|
|
"learning_rate": 3.7065889512588036e-05,
|
|
"loss": 0.6604,
|
|
"mean_token_accuracy": 0.8439219489693641,
|
|
"num_tokens": 27235110.0,
|
|
"step": 10580
|
|
},
|
|
{
|
|
"entropy": 0.6688916310667992,
|
|
"epoch": 2.1968911917098444,
|
|
"grad_norm": 0.641112744808197,
|
|
"learning_rate": 3.671145742142091e-05,
|
|
"loss": 0.6525,
|
|
"mean_token_accuracy": 0.8456299722194671,
|
|
"num_tokens": 27287836.0,
|
|
"step": 10600
|
|
},
|
|
{
|
|
"entropy": 0.6387653794139624,
|
|
"epoch": 2.201036269430052,
|
|
"grad_norm": 0.7407101392745972,
|
|
"learning_rate": 3.635834667308162e-05,
|
|
"loss": 0.6379,
|
|
"mean_token_accuracy": 0.8462458416819573,
|
|
"num_tokens": 27338477.0,
|
|
"step": 10620
|
|
},
|
|
{
|
|
"entropy": 0.6488260589540005,
|
|
"epoch": 2.2051813471502593,
|
|
"grad_norm": 0.8521301746368408,
|
|
"learning_rate": 3.60065646398423e-05,
|
|
"loss": 0.6455,
|
|
"mean_token_accuracy": 0.8410644203424453,
|
|
"num_tokens": 27390845.0,
|
|
"step": 10640
|
|
},
|
|
{
|
|
"entropy": 0.6286099251359701,
|
|
"epoch": 2.2093264248704663,
|
|
"grad_norm": 0.6972314119338989,
|
|
"learning_rate": 3.565611866623393e-05,
|
|
"loss": 0.6225,
|
|
"mean_token_accuracy": 0.8491987183690071,
|
|
"num_tokens": 27442346.0,
|
|
"step": 10660
|
|
},
|
|
{
|
|
"entropy": 0.6526692811399698,
|
|
"epoch": 2.2134715025906737,
|
|
"grad_norm": 0.8410720229148865,
|
|
"learning_rate": 3.530701606889332e-05,
|
|
"loss": 0.646,
|
|
"mean_token_accuracy": 0.8417918317019939,
|
|
"num_tokens": 27494127.0,
|
|
"step": 10680
|
|
},
|
|
{
|
|
"entropy": 0.6453199919313193,
|
|
"epoch": 2.2176165803108807,
|
|
"grad_norm": 0.8838449120521545,
|
|
"learning_rate": 3.4959264136409985e-05,
|
|
"loss": 0.6346,
|
|
"mean_token_accuracy": 0.847700871527195,
|
|
"num_tokens": 27545548.0,
|
|
"step": 10700
|
|
},
|
|
{
|
|
"entropy": 0.6914776533842086,
|
|
"epoch": 2.221761658031088,
|
|
"grad_norm": 0.9546701312065125,
|
|
"learning_rate": 3.461287012917433e-05,
|
|
"loss": 0.6931,
|
|
"mean_token_accuracy": 0.8350898310542106,
|
|
"num_tokens": 27598167.0,
|
|
"step": 10720
|
|
},
|
|
{
|
|
"entropy": 0.648529326915741,
|
|
"epoch": 2.225906735751295,
|
|
"grad_norm": 0.8443200588226318,
|
|
"learning_rate": 3.42678412792257e-05,
|
|
"loss": 0.6422,
|
|
"mean_token_accuracy": 0.8480332620441914,
|
|
"num_tokens": 27648442.0,
|
|
"step": 10740
|
|
},
|
|
{
|
|
"entropy": 0.6543070763349533,
|
|
"epoch": 2.2300518134715026,
|
|
"grad_norm": 0.9341285824775696,
|
|
"learning_rate": 3.3924184790101775e-05,
|
|
"loss": 0.6623,
|
|
"mean_token_accuracy": 0.8438093818724155,
|
|
"num_tokens": 27699256.0,
|
|
"step": 10760
|
|
},
|
|
{
|
|
"entropy": 0.6474853783845902,
|
|
"epoch": 2.2341968911917096,
|
|
"grad_norm": 0.7555035948753357,
|
|
"learning_rate": 3.358190783668787e-05,
|
|
"loss": 0.6362,
|
|
"mean_token_accuracy": 0.8459377720952034,
|
|
"num_tokens": 27750160.0,
|
|
"step": 10780
|
|
},
|
|
{
|
|
"entropy": 0.663433950766921,
|
|
"epoch": 2.238341968911917,
|
|
"grad_norm": 1.0860207080841064,
|
|
"learning_rate": 3.324101756506737e-05,
|
|
"loss": 0.6593,
|
|
"mean_token_accuracy": 0.8423758171498775,
|
|
"num_tokens": 27803456.0,
|
|
"step": 10800
|
|
},
|
|
{
|
|
"entropy": 0.6378001000732183,
|
|
"epoch": 2.2424870466321245,
|
|
"grad_norm": 0.6700264811515808,
|
|
"learning_rate": 3.2901521092372223e-05,
|
|
"loss": 0.6313,
|
|
"mean_token_accuracy": 0.8495178803801536,
|
|
"num_tokens": 27854717.0,
|
|
"step": 10820
|
|
},
|
|
{
|
|
"entropy": 0.6556556399911642,
|
|
"epoch": 2.2466321243523315,
|
|
"grad_norm": 0.8560746908187866,
|
|
"learning_rate": 3.256342550663475e-05,
|
|
"loss": 0.6467,
|
|
"mean_token_accuracy": 0.8435241214931011,
|
|
"num_tokens": 27902912.0,
|
|
"step": 10840
|
|
},
|
|
{
|
|
"entropy": 0.6649228014051914,
|
|
"epoch": 2.250777202072539,
|
|
"grad_norm": 0.774558961391449,
|
|
"learning_rate": 3.222673786663925e-05,
|
|
"loss": 0.6539,
|
|
"mean_token_accuracy": 0.8418220199644566,
|
|
"num_tokens": 27953282.0,
|
|
"step": 10860
|
|
},
|
|
{
|
|
"entropy": 0.650136835873127,
|
|
"epoch": 2.254922279792746,
|
|
"grad_norm": 0.700840413570404,
|
|
"learning_rate": 3.189146520177505e-05,
|
|
"loss": 0.6473,
|
|
"mean_token_accuracy": 0.842687302082777,
|
|
"num_tokens": 28004462.0,
|
|
"step": 10880
|
|
},
|
|
{
|
|
"entropy": 0.6371558636426926,
|
|
"epoch": 2.2590673575129534,
|
|
"grad_norm": 0.8537139892578125,
|
|
"learning_rate": 3.1557614511889334e-05,
|
|
"loss": 0.6342,
|
|
"mean_token_accuracy": 0.8491271406412124,
|
|
"num_tokens": 28056107.0,
|
|
"step": 10900
|
|
},
|
|
{
|
|
"entropy": 0.6595030531287194,
|
|
"epoch": 2.263212435233161,
|
|
"grad_norm": 0.9336352348327637,
|
|
"learning_rate": 3.12251927671413e-05,
|
|
"loss": 0.6484,
|
|
"mean_token_accuracy": 0.8411009892821312,
|
|
"num_tokens": 28107879.0,
|
|
"step": 10920
|
|
},
|
|
{
|
|
"entropy": 0.6330884475260973,
|
|
"epoch": 2.267357512953368,
|
|
"grad_norm": 0.8189968466758728,
|
|
"learning_rate": 3.0894206907856435e-05,
|
|
"loss": 0.6377,
|
|
"mean_token_accuracy": 0.8458311021327972,
|
|
"num_tokens": 28160337.0,
|
|
"step": 10940
|
|
},
|
|
{
|
|
"entropy": 0.6687283236533403,
|
|
"epoch": 2.2715025906735753,
|
|
"grad_norm": 0.7569630146026611,
|
|
"learning_rate": 3.056466384438178e-05,
|
|
"loss": 0.6646,
|
|
"mean_token_accuracy": 0.8376108758151531,
|
|
"num_tokens": 28212990.0,
|
|
"step": 10960
|
|
},
|
|
{
|
|
"entropy": 0.6340575184673071,
|
|
"epoch": 2.2756476683937823,
|
|
"grad_norm": 0.8644912242889404,
|
|
"learning_rate": 3.0236570456941592e-05,
|
|
"loss": 0.6363,
|
|
"mean_token_accuracy": 0.8497839488089085,
|
|
"num_tokens": 28263342.0,
|
|
"step": 10980
|
|
},
|
|
{
|
|
"entropy": 0.6397471319884062,
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 0.7490459084510803,
|
|
"learning_rate": 2.9909933595493578e-05,
|
|
"loss": 0.6374,
|
|
"mean_token_accuracy": 0.84698770865798,
|
|
"num_tokens": 28315353.0,
|
|
"step": 11000
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"eval_entropy": 0.7070391678551989,
|
|
"eval_loss": 0.8294876217842102,
|
|
"eval_mean_token_accuracy": 0.814408174297941,
|
|
"eval_num_tokens": 28315353.0,
|
|
"eval_runtime": 200.1537,
|
|
"eval_samples_per_second": 20.299,
|
|
"eval_steps_per_second": 2.538,
|
|
"step": 11000
|
|
},
|
|
{
|
|
"entropy": 0.627053041383624,
|
|
"epoch": 2.2839378238341967,
|
|
"grad_norm": 0.8536275029182434,
|
|
"learning_rate": 2.9584760079586127e-05,
|
|
"loss": 0.6223,
|
|
"mean_token_accuracy": 0.8513742074370384,
|
|
"num_tokens": 28365641.0,
|
|
"step": 11020
|
|
},
|
|
{
|
|
"entropy": 0.6089077012613415,
|
|
"epoch": 2.288082901554404,
|
|
"grad_norm": 0.8392689824104309,
|
|
"learning_rate": 2.9261056698215618e-05,
|
|
"loss": 0.5995,
|
|
"mean_token_accuracy": 0.851662990450859,
|
|
"num_tokens": 28417996.0,
|
|
"step": 11040
|
|
},
|
|
{
|
|
"entropy": 0.6475376274436713,
|
|
"epoch": 2.292227979274611,
|
|
"grad_norm": 0.846416711807251,
|
|
"learning_rate": 2.8938830209685087e-05,
|
|
"loss": 0.6429,
|
|
"mean_token_accuracy": 0.8415305115282535,
|
|
"num_tokens": 28471367.0,
|
|
"step": 11060
|
|
},
|
|
{
|
|
"entropy": 0.6429150599986315,
|
|
"epoch": 2.2963730569948186,
|
|
"grad_norm": 0.8654851317405701,
|
|
"learning_rate": 2.8618087341462685e-05,
|
|
"loss": 0.6535,
|
|
"mean_token_accuracy": 0.8458620354533195,
|
|
"num_tokens": 28522514.0,
|
|
"step": 11080
|
|
},
|
|
{
|
|
"entropy": 0.6514936197549105,
|
|
"epoch": 2.300518134715026,
|
|
"grad_norm": 0.7821703553199768,
|
|
"learning_rate": 2.8298834790041552e-05,
|
|
"loss": 0.6425,
|
|
"mean_token_accuracy": 0.8462134875357151,
|
|
"num_tokens": 28574064.0,
|
|
"step": 11100
|
|
},
|
|
{
|
|
"entropy": 0.6469828557223082,
|
|
"epoch": 2.304663212435233,
|
|
"grad_norm": 0.9377132058143616,
|
|
"learning_rate": 2.798107922079979e-05,
|
|
"loss": 0.6403,
|
|
"mean_token_accuracy": 0.8460097543895244,
|
|
"num_tokens": 28624996.0,
|
|
"step": 11120
|
|
},
|
|
{
|
|
"entropy": 0.661920753121376,
|
|
"epoch": 2.3088082901554405,
|
|
"grad_norm": 0.9121088981628418,
|
|
"learning_rate": 2.766482726786148e-05,
|
|
"loss": 0.6687,
|
|
"mean_token_accuracy": 0.840888224542141,
|
|
"num_tokens": 28677811.0,
|
|
"step": 11140
|
|
},
|
|
{
|
|
"entropy": 0.6738702572882176,
|
|
"epoch": 2.3129533678756475,
|
|
"grad_norm": 0.8303149938583374,
|
|
"learning_rate": 2.735008553395807e-05,
|
|
"loss": 0.6776,
|
|
"mean_token_accuracy": 0.8401521392166614,
|
|
"num_tokens": 28729641.0,
|
|
"step": 11160
|
|
},
|
|
{
|
|
"entropy": 0.6398084193468094,
|
|
"epoch": 2.317098445595855,
|
|
"grad_norm": 0.908401370048523,
|
|
"learning_rate": 2.7036860590290535e-05,
|
|
"loss": 0.6195,
|
|
"mean_token_accuracy": 0.8494696602225303,
|
|
"num_tokens": 28781662.0,
|
|
"step": 11180
|
|
},
|
|
{
|
|
"entropy": 0.6473023805767297,
|
|
"epoch": 2.321243523316062,
|
|
"grad_norm": 0.8027302622795105,
|
|
"learning_rate": 2.6725158976392173e-05,
|
|
"loss": 0.6513,
|
|
"mean_token_accuracy": 0.8457495465874671,
|
|
"num_tokens": 28830808.0,
|
|
"step": 11200
|
|
},
|
|
{
|
|
"entropy": 0.6902273558080196,
|
|
"epoch": 2.3253886010362694,
|
|
"grad_norm": 0.8590432405471802,
|
|
"learning_rate": 2.6414987199992148e-05,
|
|
"loss": 0.691,
|
|
"mean_token_accuracy": 0.8374306775629521,
|
|
"num_tokens": 28883459.0,
|
|
"step": 11220
|
|
},
|
|
{
|
|
"entropy": 0.6321788892149925,
|
|
"epoch": 2.329533678756477,
|
|
"grad_norm": 0.8151998519897461,
|
|
"learning_rate": 2.610635173687943e-05,
|
|
"loss": 0.6172,
|
|
"mean_token_accuracy": 0.8519966520369053,
|
|
"num_tokens": 28935267.0,
|
|
"step": 11240
|
|
},
|
|
{
|
|
"entropy": 0.6105965875089169,
|
|
"epoch": 2.333678756476684,
|
|
"grad_norm": 0.9420289397239685,
|
|
"learning_rate": 2.5799259030767964e-05,
|
|
"loss": 0.6139,
|
|
"mean_token_accuracy": 0.849336089938879,
|
|
"num_tokens": 28987596.0,
|
|
"step": 11260
|
|
},
|
|
{
|
|
"entropy": 0.6410636026412249,
|
|
"epoch": 2.3378238341968913,
|
|
"grad_norm": 1.0593581199645996,
|
|
"learning_rate": 2.5493715493161675e-05,
|
|
"loss": 0.6319,
|
|
"mean_token_accuracy": 0.8488616354763507,
|
|
"num_tokens": 29037838.0,
|
|
"step": 11280
|
|
},
|
|
{
|
|
"entropy": 0.658510209992528,
|
|
"epoch": 2.3419689119170983,
|
|
"grad_norm": 0.7236459255218506,
|
|
"learning_rate": 2.518972750322096e-05,
|
|
"loss": 0.6676,
|
|
"mean_token_accuracy": 0.8431073188781738,
|
|
"num_tokens": 29088873.0,
|
|
"step": 11300
|
|
},
|
|
{
|
|
"entropy": 0.6631653629243374,
|
|
"epoch": 2.3461139896373058,
|
|
"grad_norm": 0.7426918148994446,
|
|
"learning_rate": 2.4887301407629294e-05,
|
|
"loss": 0.6545,
|
|
"mean_token_accuracy": 0.8448002658784389,
|
|
"num_tokens": 29139781.0,
|
|
"step": 11320
|
|
},
|
|
{
|
|
"entropy": 0.6609065797179937,
|
|
"epoch": 2.3502590673575128,
|
|
"grad_norm": 0.9877640008926392,
|
|
"learning_rate": 2.458644352046089e-05,
|
|
"loss": 0.6638,
|
|
"mean_token_accuracy": 0.8403280735015869,
|
|
"num_tokens": 29191244.0,
|
|
"step": 11340
|
|
},
|
|
{
|
|
"entropy": 0.6530089993029833,
|
|
"epoch": 2.3544041450777202,
|
|
"grad_norm": 0.8895524144172668,
|
|
"learning_rate": 2.4287160123048748e-05,
|
|
"loss": 0.652,
|
|
"mean_token_accuracy": 0.8447512105107308,
|
|
"num_tokens": 29242515.0,
|
|
"step": 11360
|
|
},
|
|
{
|
|
"entropy": 0.6493765514343977,
|
|
"epoch": 2.3585492227979277,
|
|
"grad_norm": 0.7405846118927002,
|
|
"learning_rate": 2.3989457463853592e-05,
|
|
"loss": 0.6411,
|
|
"mean_token_accuracy": 0.8460350595414639,
|
|
"num_tokens": 29293645.0,
|
|
"step": 11380
|
|
},
|
|
{
|
|
"entropy": 0.667537048831582,
|
|
"epoch": 2.3626943005181347,
|
|
"grad_norm": 0.7325462102890015,
|
|
"learning_rate": 2.3693341758333278e-05,
|
|
"loss": 0.6685,
|
|
"mean_token_accuracy": 0.8410760566592217,
|
|
"num_tokens": 29345634.0,
|
|
"step": 11400
|
|
},
|
|
{
|
|
"entropy": 0.6467866998165845,
|
|
"epoch": 2.366839378238342,
|
|
"grad_norm": 0.819976806640625,
|
|
"learning_rate": 2.339881918881327e-05,
|
|
"loss": 0.6406,
|
|
"mean_token_accuracy": 0.8478086508810521,
|
|
"num_tokens": 29395059.0,
|
|
"step": 11420
|
|
},
|
|
{
|
|
"entropy": 0.6170216646045447,
|
|
"epoch": 2.370984455958549,
|
|
"grad_norm": 0.8028335571289062,
|
|
"learning_rate": 2.310589590435729e-05,
|
|
"loss": 0.6136,
|
|
"mean_token_accuracy": 0.8523507431149483,
|
|
"num_tokens": 29445517.0,
|
|
"step": 11440
|
|
},
|
|
{
|
|
"entropy": 0.61521628908813,
|
|
"epoch": 2.3751295336787566,
|
|
"grad_norm": 1.0430378913879395,
|
|
"learning_rate": 2.2814578020639143e-05,
|
|
"loss": 0.6203,
|
|
"mean_token_accuracy": 0.852887948602438,
|
|
"num_tokens": 29494967.0,
|
|
"step": 11460
|
|
},
|
|
{
|
|
"entropy": 0.6518917117267847,
|
|
"epoch": 2.3792746113989636,
|
|
"grad_norm": 0.8299168944358826,
|
|
"learning_rate": 2.2524871619814993e-05,
|
|
"loss": 0.6421,
|
|
"mean_token_accuracy": 0.8435743443667889,
|
|
"num_tokens": 29546691.0,
|
|
"step": 11480
|
|
},
|
|
{
|
|
"entropy": 0.6700312186032533,
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 0.9835838079452515,
|
|
"learning_rate": 2.2236782750396213e-05,
|
|
"loss": 0.6601,
|
|
"mean_token_accuracy": 0.8435488626360893,
|
|
"num_tokens": 29599532.0,
|
|
"step": 11500
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"eval_entropy": 0.6985438240794685,
|
|
"eval_loss": 0.8230946660041809,
|
|
"eval_mean_token_accuracy": 0.8159553369903189,
|
|
"eval_num_tokens": 29599532.0,
|
|
"eval_runtime": 199.7844,
|
|
"eval_samples_per_second": 20.337,
|
|
"eval_steps_per_second": 2.543,
|
|
"step": 11500
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 14475,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"EarlyStoppingCallback": {
|
|
"args": {
|
|
"early_stopping_patience": 4,
|
|
"early_stopping_threshold": 0.005
|
|
},
|
|
"attributes": {
|
|
"early_stopping_patience_counter": 0
|
|
}
|
|
},
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 2.0389130720590848e+18,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|