1448 lines
41 KiB
JSON
1448 lines
41 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 100,
|
|
"global_step": 1275,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 0.7889615148305893,
|
|
"epoch": 0.023557126030624265,
|
|
"grad_norm": 8.1875,
|
|
"learning_rate": 1.40625e-06,
|
|
"loss": 0.8571792602539062,
|
|
"mean_token_accuracy": 0.8150893449783325,
|
|
"num_tokens": 64215.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 0.8182223863899708,
|
|
"epoch": 0.04711425206124853,
|
|
"grad_norm": 7.75,
|
|
"learning_rate": 2.96875e-06,
|
|
"loss": 0.875468635559082,
|
|
"mean_token_accuracy": 0.8121306523680687,
|
|
"num_tokens": 125069.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 0.774581016600132,
|
|
"epoch": 0.0706713780918728,
|
|
"grad_norm": 5.6875,
|
|
"learning_rate": 4.53125e-06,
|
|
"loss": 0.7942372322082519,
|
|
"mean_token_accuracy": 0.8235767468810081,
|
|
"num_tokens": 190122.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 0.7172340139746666,
|
|
"epoch": 0.09422850412249706,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 6.093750000000001e-06,
|
|
"loss": 0.7248775959014893,
|
|
"mean_token_accuracy": 0.834468886256218,
|
|
"num_tokens": 253014.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 0.6181357510387897,
|
|
"epoch": 0.11778563015312132,
|
|
"grad_norm": 4.15625,
|
|
"learning_rate": 7.656250000000001e-06,
|
|
"loss": 0.6276291370391845,
|
|
"mean_token_accuracy": 0.8509871512651443,
|
|
"num_tokens": 313920.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 0.5261967420578003,
|
|
"epoch": 0.1413427561837456,
|
|
"grad_norm": 2.875,
|
|
"learning_rate": 9.21875e-06,
|
|
"loss": 0.5425624847412109,
|
|
"mean_token_accuracy": 0.8646206513047219,
|
|
"num_tokens": 375393.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 0.4926342010498047,
|
|
"epoch": 0.16489988221436985,
|
|
"grad_norm": 3.015625,
|
|
"learning_rate": 1.0781250000000001e-05,
|
|
"loss": 0.4834623336791992,
|
|
"mean_token_accuracy": 0.8720198556780815,
|
|
"num_tokens": 438328.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 0.4710537105798721,
|
|
"epoch": 0.18845700824499412,
|
|
"grad_norm": 2.921875,
|
|
"learning_rate": 1.234375e-05,
|
|
"loss": 0.4642228603363037,
|
|
"mean_token_accuracy": 0.877609857916832,
|
|
"num_tokens": 497477.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 0.4559089481830597,
|
|
"epoch": 0.21201413427561838,
|
|
"grad_norm": 2.4375,
|
|
"learning_rate": 1.3906250000000001e-05,
|
|
"loss": 0.42686986923217773,
|
|
"mean_token_accuracy": 0.8810460358858109,
|
|
"num_tokens": 563258.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 0.44573221281170844,
|
|
"epoch": 0.23557126030624265,
|
|
"grad_norm": 3.125,
|
|
"learning_rate": 1.546875e-05,
|
|
"loss": 0.4036698818206787,
|
|
"mean_token_accuracy": 0.8818169787526131,
|
|
"num_tokens": 630849.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.23557126030624265,
|
|
"eval_entropy": 0.4385891295141644,
|
|
"eval_loss": 0.40417641401290894,
|
|
"eval_mean_token_accuracy": 0.8824950125482347,
|
|
"eval_num_tokens": 630849.0,
|
|
"eval_runtime": 18.2759,
|
|
"eval_samples_per_second": 19.589,
|
|
"eval_steps_per_second": 4.925,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 0.42985599488019943,
|
|
"epoch": 0.2591283863368669,
|
|
"grad_norm": 2.84375,
|
|
"learning_rate": 1.703125e-05,
|
|
"loss": 0.3886325597763062,
|
|
"mean_token_accuracy": 0.887049525976181,
|
|
"num_tokens": 693995.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 0.44139461889863013,
|
|
"epoch": 0.2826855123674912,
|
|
"grad_norm": 2.734375,
|
|
"learning_rate": 1.859375e-05,
|
|
"loss": 0.38890700340270995,
|
|
"mean_token_accuracy": 0.8885618790984153,
|
|
"num_tokens": 756897.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 0.4030730716884136,
|
|
"epoch": 0.30624263839811544,
|
|
"grad_norm": 2.8125,
|
|
"learning_rate": 1.9999962490381906e-05,
|
|
"loss": 0.3493493556976318,
|
|
"mean_token_accuracy": 0.894849705696106,
|
|
"num_tokens": 822752.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 0.3679922290146351,
|
|
"epoch": 0.3297997644287397,
|
|
"grad_norm": 3.046875,
|
|
"learning_rate": 1.999546167668739e-05,
|
|
"loss": 0.3169102668762207,
|
|
"mean_token_accuracy": 0.9010762587189675,
|
|
"num_tokens": 884853.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 0.3625841185450554,
|
|
"epoch": 0.35335689045936397,
|
|
"grad_norm": 2.4375,
|
|
"learning_rate": 1.998346280806768e-05,
|
|
"loss": 0.32173867225646974,
|
|
"mean_token_accuracy": 0.8979647248983383,
|
|
"num_tokens": 947499.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 0.36609998121857645,
|
|
"epoch": 0.37691401648998824,
|
|
"grad_norm": 2.625,
|
|
"learning_rate": 1.9963974885425267e-05,
|
|
"loss": 0.3519533634185791,
|
|
"mean_token_accuracy": 0.8926548227667809,
|
|
"num_tokens": 1008262.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 0.3507001116871834,
|
|
"epoch": 0.4004711425206125,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 1.9937012527546064e-05,
|
|
"loss": 0.32005207538604735,
|
|
"mean_token_accuracy": 0.9024498045444489,
|
|
"num_tokens": 1071249.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 0.3369211222976446,
|
|
"epoch": 0.42402826855123676,
|
|
"grad_norm": 2.703125,
|
|
"learning_rate": 1.9902595960133182e-05,
|
|
"loss": 0.323317813873291,
|
|
"mean_token_accuracy": 0.8968252062797546,
|
|
"num_tokens": 1131234.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 0.34537971392273903,
|
|
"epoch": 0.44758539458186103,
|
|
"grad_norm": 2.6875,
|
|
"learning_rate": 1.986075100063469e-05,
|
|
"loss": 0.32833309173583985,
|
|
"mean_token_accuracy": 0.8981048464775085,
|
|
"num_tokens": 1193301.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 0.3510992005467415,
|
|
"epoch": 0.4711425206124853,
|
|
"grad_norm": 3.046875,
|
|
"learning_rate": 1.9811509038876772e-05,
|
|
"loss": 0.33012425899505615,
|
|
"mean_token_accuracy": 0.896586000919342,
|
|
"num_tokens": 1253516.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.4711425206124853,
|
|
"eval_entropy": 0.3400075112779935,
|
|
"eval_loss": 0.3215847909450531,
|
|
"eval_mean_token_accuracy": 0.8995050960116916,
|
|
"eval_num_tokens": 1253516.0,
|
|
"eval_runtime": 17.9992,
|
|
"eval_samples_per_second": 19.89,
|
|
"eval_steps_per_second": 5.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 0.3474044375121593,
|
|
"epoch": 0.49469964664310956,
|
|
"grad_norm": 2.890625,
|
|
"learning_rate": 1.9754907013516774e-05,
|
|
"loss": 0.33171815872192384,
|
|
"mean_token_accuracy": 0.8964410588145256,
|
|
"num_tokens": 1316463.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 0.3416020147502422,
|
|
"epoch": 0.5182567726737338,
|
|
"grad_norm": 2.625,
|
|
"learning_rate": 1.9690987384333823e-05,
|
|
"loss": 0.31979641914367674,
|
|
"mean_token_accuracy": 0.901093739271164,
|
|
"num_tokens": 1377534.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 0.340639504045248,
|
|
"epoch": 0.541813898704358,
|
|
"grad_norm": 2.75,
|
|
"learning_rate": 1.961979810037778e-05,
|
|
"loss": 0.32565860748291015,
|
|
"mean_token_accuracy": 0.8986324697732926,
|
|
"num_tokens": 1442756.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 0.3141076397150755,
|
|
"epoch": 0.5653710247349824,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 1.954139256400049e-05,
|
|
"loss": 0.29269728660583494,
|
|
"mean_token_accuracy": 0.9061810851097107,
|
|
"num_tokens": 1507239.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 0.33452659361064435,
|
|
"epoch": 0.5889281507656066,
|
|
"grad_norm": 2.796875,
|
|
"learning_rate": 1.9455829590796184e-05,
|
|
"loss": 0.32290034294128417,
|
|
"mean_token_accuracy": 0.8977762594819069,
|
|
"num_tokens": 1568670.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 0.31916009560227393,
|
|
"epoch": 0.6124852767962309,
|
|
"grad_norm": 2.84375,
|
|
"learning_rate": 1.9363173365481223e-05,
|
|
"loss": 0.3089333772659302,
|
|
"mean_token_accuracy": 0.9027115389704704,
|
|
"num_tokens": 1631924.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 0.32317030280828474,
|
|
"epoch": 0.6360424028268551,
|
|
"grad_norm": 2.84375,
|
|
"learning_rate": 1.92634933937462e-05,
|
|
"loss": 0.30866892337799073,
|
|
"mean_token_accuracy": 0.9009129598736763,
|
|
"num_tokens": 1694459.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 0.3136075004935265,
|
|
"epoch": 0.6595995288574794,
|
|
"grad_norm": 2.21875,
|
|
"learning_rate": 1.915686445011651e-05,
|
|
"loss": 0.2930387020111084,
|
|
"mean_token_accuracy": 0.9052153035998345,
|
|
"num_tokens": 1754072.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 0.3060571104288101,
|
|
"epoch": 0.6831566548881036,
|
|
"grad_norm": 2.0625,
|
|
"learning_rate": 1.9043366521860558e-05,
|
|
"loss": 0.2982192516326904,
|
|
"mean_token_accuracy": 0.9027306795120239,
|
|
"num_tokens": 1816651.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 0.29948526062071323,
|
|
"epoch": 0.7067137809187279,
|
|
"grad_norm": 2.171875,
|
|
"learning_rate": 1.892308474898763e-05,
|
|
"loss": 0.2839794635772705,
|
|
"mean_token_accuracy": 0.9067387148737908,
|
|
"num_tokens": 1878801.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.7067137809187279,
|
|
"eval_entropy": 0.32328523662355213,
|
|
"eval_loss": 0.30643436312675476,
|
|
"eval_mean_token_accuracy": 0.9020023855898115,
|
|
"eval_num_tokens": 1878801.0,
|
|
"eval_runtime": 18.2117,
|
|
"eval_samples_per_second": 19.658,
|
|
"eval_steps_per_second": 4.942,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 0.33191716447472575,
|
|
"epoch": 0.7302709069493521,
|
|
"grad_norm": 2.375,
|
|
"learning_rate": 1.879610936038046e-05,
|
|
"loss": 0.321528697013855,
|
|
"mean_token_accuracy": 0.8981613501906395,
|
|
"num_tokens": 1940429.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 0.3285808339715004,
|
|
"epoch": 0.7538280329799765,
|
|
"grad_norm": 3.0,
|
|
"learning_rate": 1.86625356061104e-05,
|
|
"loss": 0.31851098537445066,
|
|
"mean_token_accuracy": 0.8980434089899063,
|
|
"num_tokens": 2001961.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 0.3324954815208912,
|
|
"epoch": 0.7773851590106007,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 1.852246368598597e-05,
|
|
"loss": 0.3293052911758423,
|
|
"mean_token_accuracy": 0.8975612118840217,
|
|
"num_tokens": 2065780.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 0.31062053591012956,
|
|
"epoch": 0.800942285041225,
|
|
"grad_norm": 1.9453125,
|
|
"learning_rate": 1.837599867438838e-05,
|
|
"loss": 0.2968944787979126,
|
|
"mean_token_accuracy": 0.9042825505137444,
|
|
"num_tokens": 2128375.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 0.3136239554733038,
|
|
"epoch": 0.8244994110718492,
|
|
"grad_norm": 3.453125,
|
|
"learning_rate": 1.822325044145041e-05,
|
|
"loss": 0.3048269033432007,
|
|
"mean_token_accuracy": 0.9041909068822861,
|
|
"num_tokens": 2191745.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 0.29147293269634245,
|
|
"epoch": 0.8480565371024735,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 1.8064333570637768e-05,
|
|
"loss": 0.2808744192123413,
|
|
"mean_token_accuracy": 0.9083063289523124,
|
|
"num_tokens": 2257044.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 0.319144606217742,
|
|
"epoch": 0.8716136631330977,
|
|
"grad_norm": 3.03125,
|
|
"learning_rate": 1.7899367272794777e-05,
|
|
"loss": 0.31203126907348633,
|
|
"mean_token_accuracy": 0.9014263987541199,
|
|
"num_tokens": 2319051.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 0.312490488961339,
|
|
"epoch": 0.8951707891637221,
|
|
"grad_norm": 2.75,
|
|
"learning_rate": 1.7728475296718846e-05,
|
|
"loss": 0.3043776273727417,
|
|
"mean_token_accuracy": 0.9026625797152519,
|
|
"num_tokens": 2382991.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 0.3071211285889149,
|
|
"epoch": 0.9187279151943463,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 1.7551785836330814e-05,
|
|
"loss": 0.30926945209503176,
|
|
"mean_token_accuracy": 0.9017301961779595,
|
|
"num_tokens": 2444896.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 0.30839360542595384,
|
|
"epoch": 0.9422850412249706,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 1.7369431434510793e-05,
|
|
"loss": 0.2929032802581787,
|
|
"mean_token_accuracy": 0.9051709935069084,
|
|
"num_tokens": 2503544.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.9422850412249706,
|
|
"eval_entropy": 0.30972004185120267,
|
|
"eval_loss": 0.29919198155403137,
|
|
"eval_mean_token_accuracy": 0.9038941383361816,
|
|
"eval_num_tokens": 2503544.0,
|
|
"eval_runtime": 17.7094,
|
|
"eval_samples_per_second": 20.215,
|
|
"eval_steps_per_second": 5.082,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 0.311648141592741,
|
|
"epoch": 0.9658421672555948,
|
|
"grad_norm": 2.25,
|
|
"learning_rate": 1.718154888367167e-05,
|
|
"loss": 0.30500121116638185,
|
|
"mean_token_accuracy": 0.9008665040135384,
|
|
"num_tokens": 2568084.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 0.2945106942206621,
|
|
"epoch": 0.9893992932862191,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 1.698827912314484e-05,
|
|
"loss": 0.2944274663925171,
|
|
"mean_token_accuracy": 0.9072879686951637,
|
|
"num_tokens": 2626844.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 0.2838428891018817,
|
|
"epoch": 1.0117785630153122,
|
|
"grad_norm": 2.0,
|
|
"learning_rate": 1.678976713345513e-05,
|
|
"loss": 0.2622137784957886,
|
|
"mean_token_accuracy": 0.9127975890510961,
|
|
"num_tokens": 2683705.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 0.2841747090220451,
|
|
"epoch": 1.0353356890459364,
|
|
"grad_norm": 2.375,
|
|
"learning_rate": 1.658616182756426e-05,
|
|
"loss": 0.26954827308654783,
|
|
"mean_token_accuracy": 0.9122204825282096,
|
|
"num_tokens": 2746010.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 0.26254382394254205,
|
|
"epoch": 1.0588928150765606,
|
|
"grad_norm": 2.53125,
|
|
"learning_rate": 1.6377615939164387e-05,
|
|
"loss": 0.2567893505096436,
|
|
"mean_token_accuracy": 0.915389546751976,
|
|
"num_tokens": 2806689.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 0.2845174681395292,
|
|
"epoch": 1.0824499411071848,
|
|
"grad_norm": 2.59375,
|
|
"learning_rate": 1.6164285908105526e-05,
|
|
"loss": 0.28064637184143065,
|
|
"mean_token_accuracy": 0.9090600132942199,
|
|
"num_tokens": 2867295.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 0.2866407621651888,
|
|
"epoch": 1.1060070671378093,
|
|
"grad_norm": 2.671875,
|
|
"learning_rate": 1.594633176304287e-05,
|
|
"loss": 0.2659857988357544,
|
|
"mean_token_accuracy": 0.9122302174568176,
|
|
"num_tokens": 2933048.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 0.2710740964859724,
|
|
"epoch": 1.1295641931684335,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 1.5723917001391912e-05,
|
|
"loss": 0.2589958906173706,
|
|
"mean_token_accuracy": 0.9144651293754578,
|
|
"num_tokens": 2996555.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 0.2907809391617775,
|
|
"epoch": 1.1531213191990577,
|
|
"grad_norm": 2.78125,
|
|
"learning_rate": 1.5497208466681547e-05,
|
|
"loss": 0.2864827156066895,
|
|
"mean_token_accuracy": 0.9080412805080413,
|
|
"num_tokens": 3057517.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 0.2771881382912397,
|
|
"epoch": 1.176678445229682,
|
|
"grad_norm": 2.765625,
|
|
"learning_rate": 1.526637622339706e-05,
|
|
"loss": 0.2643341302871704,
|
|
"mean_token_accuracy": 0.912305337190628,
|
|
"num_tokens": 3120296.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 1.176678445229682,
|
|
"eval_entropy": 0.2753614894217915,
|
|
"eval_loss": 0.29845961928367615,
|
|
"eval_mean_token_accuracy": 0.904228851530287,
|
|
"eval_num_tokens": 3120296.0,
|
|
"eval_runtime": 17.8418,
|
|
"eval_samples_per_second": 20.065,
|
|
"eval_steps_per_second": 5.044,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 0.2737978909164667,
|
|
"epoch": 1.2002355712603063,
|
|
"grad_norm": 2.34375,
|
|
"learning_rate": 1.5031593429406964e-05,
|
|
"loss": 0.2711649417877197,
|
|
"mean_token_accuracy": 0.9123903706669807,
|
|
"num_tokens": 3181634.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"entropy": 0.274722558632493,
|
|
"epoch": 1.2237926972909305,
|
|
"grad_norm": 2.78125,
|
|
"learning_rate": 1.4793036206069298e-05,
|
|
"loss": 0.2646583795547485,
|
|
"mean_token_accuracy": 0.9136801406741142,
|
|
"num_tokens": 3243733.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"entropy": 0.27263724245131016,
|
|
"epoch": 1.2473498233215548,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 1.455088350611494e-05,
|
|
"loss": 0.26004538536071775,
|
|
"mean_token_accuracy": 0.916094321012497,
|
|
"num_tokens": 3304134.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"entropy": 0.2745048556476831,
|
|
"epoch": 1.270906949352179,
|
|
"grad_norm": 2.703125,
|
|
"learning_rate": 1.430531697940693e-05,
|
|
"loss": 0.26915180683135986,
|
|
"mean_token_accuracy": 0.9127532616257668,
|
|
"num_tokens": 3367557.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"entropy": 0.2687293775379658,
|
|
"epoch": 1.2944640753828032,
|
|
"grad_norm": 2.5625,
|
|
"learning_rate": 1.4056520836676551e-05,
|
|
"loss": 0.2629591703414917,
|
|
"mean_token_accuracy": 0.9146309986710548,
|
|
"num_tokens": 3431087.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"entropy": 0.28919927664101125,
|
|
"epoch": 1.3180212014134276,
|
|
"grad_norm": 2.53125,
|
|
"learning_rate": 1.380468171133842e-05,
|
|
"loss": 0.2779649496078491,
|
|
"mean_token_accuracy": 0.9086012721061707,
|
|
"num_tokens": 3491841.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"entropy": 0.2814686447381973,
|
|
"epoch": 1.3415783274440518,
|
|
"grad_norm": 2.125,
|
|
"learning_rate": 1.354998851948818e-05,
|
|
"loss": 0.27813720703125,
|
|
"mean_token_accuracy": 0.9098659679293633,
|
|
"num_tokens": 3561309.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"entropy": 0.27902866415679456,
|
|
"epoch": 1.365135453474676,
|
|
"grad_norm": 2.78125,
|
|
"learning_rate": 1.3292632318187857e-05,
|
|
"loss": 0.26876034736633303,
|
|
"mean_token_accuracy": 0.9134621784090996,
|
|
"num_tokens": 3624157.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"entropy": 0.26935031823813915,
|
|
"epoch": 1.3886925795053005,
|
|
"grad_norm": 3.0,
|
|
"learning_rate": 1.3032806162145193e-05,
|
|
"loss": 0.26446683406829835,
|
|
"mean_token_accuracy": 0.9133341759443283,
|
|
"num_tokens": 3683984.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"entropy": 0.2716564033180475,
|
|
"epoch": 1.4122497055359247,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 1.2770704958894424e-05,
|
|
"loss": 0.25622873306274413,
|
|
"mean_token_accuracy": 0.9154532626271248,
|
|
"num_tokens": 3746491.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 1.4122497055359247,
|
|
"eval_entropy": 0.2752952078978221,
|
|
"eval_loss": 0.29595252871513367,
|
|
"eval_mean_token_accuracy": 0.904966625240114,
|
|
"eval_num_tokens": 3746491.0,
|
|
"eval_runtime": 17.6989,
|
|
"eval_samples_per_second": 20.227,
|
|
"eval_steps_per_second": 5.085,
|
|
"step": 600
|
|
},
|
|
{
|
|
"entropy": 0.287643176317215,
|
|
"epoch": 1.4358068315665489,
|
|
"grad_norm": 2.3125,
|
|
"learning_rate": 1.2506525322587207e-05,
|
|
"loss": 0.2789506673812866,
|
|
"mean_token_accuracy": 0.9099320113658905,
|
|
"num_tokens": 3805790.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"entropy": 0.25317693017423154,
|
|
"epoch": 1.459363957597173,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 1.2240465426503296e-05,
|
|
"loss": 0.24759247303009033,
|
|
"mean_token_accuracy": 0.9188913211226464,
|
|
"num_tokens": 3871408.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"entropy": 0.265877678245306,
|
|
"epoch": 1.4829210836277973,
|
|
"grad_norm": 2.765625,
|
|
"learning_rate": 1.197272485439167e-05,
|
|
"loss": 0.2643004894256592,
|
|
"mean_token_accuracy": 0.9132170379161835,
|
|
"num_tokens": 3933222.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"entropy": 0.2881245229393244,
|
|
"epoch": 1.5064782096584217,
|
|
"grad_norm": 2.375,
|
|
"learning_rate": 1.1703504450753604e-05,
|
|
"loss": 0.2831957578659058,
|
|
"mean_token_accuracy": 0.9093836858868599,
|
|
"num_tokens": 3997842.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"entropy": 0.26541271023452284,
|
|
"epoch": 1.530035335689046,
|
|
"grad_norm": 2.796875,
|
|
"learning_rate": 1.1433006170179961e-05,
|
|
"loss": 0.2564934253692627,
|
|
"mean_token_accuracy": 0.9148715168237687,
|
|
"num_tokens": 4064436.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"entropy": 0.27074739187955854,
|
|
"epoch": 1.5535924617196701,
|
|
"grad_norm": 2.609375,
|
|
"learning_rate": 1.1161432925855792e-05,
|
|
"loss": 0.26715807914733886,
|
|
"mean_token_accuracy": 0.9134342730045318,
|
|
"num_tokens": 4123228.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"entropy": 0.26959656067192556,
|
|
"epoch": 1.5771495877502946,
|
|
"grad_norm": 2.46875,
|
|
"learning_rate": 1.0888988437345851e-05,
|
|
"loss": 0.25867605209350586,
|
|
"mean_token_accuracy": 0.914973109960556,
|
|
"num_tokens": 4190074.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"entropy": 0.27577902004122734,
|
|
"epoch": 1.6007067137809188,
|
|
"grad_norm": 2.421875,
|
|
"learning_rate": 1.061587707777515e-05,
|
|
"loss": 0.2708859443664551,
|
|
"mean_token_accuracy": 0.9112199231982231,
|
|
"num_tokens": 4253621.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"entropy": 0.27748207114636897,
|
|
"epoch": 1.624263839811543,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 1.0342303720519331e-05,
|
|
"loss": 0.27588458061218263,
|
|
"mean_token_accuracy": 0.9095302507281303,
|
|
"num_tokens": 4319496.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"entropy": 0.28387282863259317,
|
|
"epoch": 1.6478209658421674,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 1.0068473585519714e-05,
|
|
"loss": 0.27500205039978026,
|
|
"mean_token_accuracy": 0.9111377790570259,
|
|
"num_tokens": 4382617.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 1.6478209658421674,
|
|
"eval_entropy": 0.2773133357365926,
|
|
"eval_loss": 0.29400721192359924,
|
|
"eval_mean_token_accuracy": 0.9053900049792396,
|
|
"eval_num_tokens": 4382617.0,
|
|
"eval_runtime": 17.7539,
|
|
"eval_samples_per_second": 20.165,
|
|
"eval_steps_per_second": 5.069,
|
|
"step": 700
|
|
},
|
|
{
|
|
"entropy": 0.262077946215868,
|
|
"epoch": 1.6713780918727914,
|
|
"grad_norm": 2.90625,
|
|
"learning_rate": 9.794592085338401e-06,
|
|
"loss": 0.25229198932647706,
|
|
"mean_token_accuracy": 0.9180409327149391,
|
|
"num_tokens": 4443337.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"entropy": 0.2617459647357464,
|
|
"epoch": 1.6949352179034158,
|
|
"grad_norm": 2.703125,
|
|
"learning_rate": 9.520864671068875e-06,
|
|
"loss": 0.2502692699432373,
|
|
"mean_token_accuracy": 0.9161550804972649,
|
|
"num_tokens": 4506989.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"entropy": 0.25692556127905847,
|
|
"epoch": 1.71849234393404,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 9.247496678217704e-06,
|
|
"loss": 0.24970452785491942,
|
|
"mean_token_accuracy": 0.9180740728974343,
|
|
"num_tokens": 4568818.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"entropy": 0.2656152307987213,
|
|
"epoch": 1.7420494699646643,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 8.974693172672936e-06,
|
|
"loss": 0.2638600587844849,
|
|
"mean_token_accuracy": 0.9151053532958031,
|
|
"num_tokens": 4629719.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"entropy": 0.2762697324156761,
|
|
"epoch": 1.7656065959952887,
|
|
"grad_norm": 3.09375,
|
|
"learning_rate": 8.70265879687476e-06,
|
|
"loss": 0.26587440967559817,
|
|
"mean_token_accuracy": 0.913601940870285,
|
|
"num_tokens": 4689567.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"entropy": 0.2680691860616207,
|
|
"epoch": 1.7891637220259127,
|
|
"grad_norm": 2.4375,
|
|
"learning_rate": 8.431597616303818e-06,
|
|
"loss": 0.258583664894104,
|
|
"mean_token_accuracy": 0.9141830712556839,
|
|
"num_tokens": 4752689.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"entropy": 0.2715546615421772,
|
|
"epoch": 1.812720848056537,
|
|
"grad_norm": 2.875,
|
|
"learning_rate": 8.16171296640231e-06,
|
|
"loss": 0.26168193817138674,
|
|
"mean_token_accuracy": 0.9130914509296417,
|
|
"num_tokens": 4812140.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"entropy": 0.24891340732574463,
|
|
"epoch": 1.8362779740871613,
|
|
"grad_norm": 2.34375,
|
|
"learning_rate": 7.89320730004274e-06,
|
|
"loss": 0.24246461391448976,
|
|
"mean_token_accuracy": 0.9222318306565285,
|
|
"num_tokens": 4877858.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"entropy": 0.2686690021306276,
|
|
"epoch": 1.8598351001177855,
|
|
"grad_norm": 2.328125,
|
|
"learning_rate": 7.626282035658703e-06,
|
|
"loss": 0.26695163249969484,
|
|
"mean_token_accuracy": 0.9125299632549286,
|
|
"num_tokens": 4938326.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"entropy": 0.26906889863312244,
|
|
"epoch": 1.88339222614841,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 7.361137406151706e-06,
|
|
"loss": 0.25990853309631345,
|
|
"mean_token_accuracy": 0.9151310101151466,
|
|
"num_tokens": 5000808.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.88339222614841,
|
|
"eval_entropy": 0.2677851602435112,
|
|
"eval_loss": 0.29366225004196167,
|
|
"eval_mean_token_accuracy": 0.905531241496404,
|
|
"eval_num_tokens": 5000808.0,
|
|
"eval_runtime": 18.2132,
|
|
"eval_samples_per_second": 19.656,
|
|
"eval_steps_per_second": 4.941,
|
|
"step": 800
|
|
},
|
|
{
|
|
"entropy": 0.2672777663916349,
|
|
"epoch": 1.9069493521790342,
|
|
"grad_norm": 2.5625,
|
|
"learning_rate": 7.0979723086872245e-06,
|
|
"loss": 0.26606771945953367,
|
|
"mean_token_accuracy": 0.9127248525619507,
|
|
"num_tokens": 5061451.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"entropy": 0.26827876679599283,
|
|
"epoch": 1.9305064782096584,
|
|
"grad_norm": 2.828125,
|
|
"learning_rate": 6.8369841554928226e-06,
|
|
"loss": 0.266079306602478,
|
|
"mean_token_accuracy": 0.9128642335534096,
|
|
"num_tokens": 5125616.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"entropy": 0.2682387940585613,
|
|
"epoch": 1.9540636042402828,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 6.578368725770182e-06,
|
|
"loss": 0.25919017791748045,
|
|
"mean_token_accuracy": 0.9145669594407082,
|
|
"num_tokens": 5186355.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"entropy": 0.2683764260262251,
|
|
"epoch": 1.9776207302709068,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 6.322320018832125e-06,
|
|
"loss": 0.25967261791229246,
|
|
"mean_token_accuracy": 0.9135927557945251,
|
|
"num_tokens": 5249849.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"entropy": 0.2581090499695979,
|
|
"epoch": 2.0,
|
|
"grad_norm": 3.359375,
|
|
"learning_rate": 6.069030108574806e-06,
|
|
"loss": 0.2496717929840088,
|
|
"mean_token_accuracy": 0.9168546999755659,
|
|
"num_tokens": 5306782.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"entropy": 0.25910058319568635,
|
|
"epoch": 2.0235571260306244,
|
|
"grad_norm": 2.375,
|
|
"learning_rate": 5.8186889993942704e-06,
|
|
"loss": 0.24170937538146972,
|
|
"mean_token_accuracy": 0.9203898578882217,
|
|
"num_tokens": 5368144.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"entropy": 0.2578096315264702,
|
|
"epoch": 2.0471142520612484,
|
|
"grad_norm": 2.25,
|
|
"learning_rate": 5.571484483655443e-06,
|
|
"loss": 0.23918161392211915,
|
|
"mean_token_accuracy": 0.9217669263482093,
|
|
"num_tokens": 5432512.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"entropy": 0.27063696682453153,
|
|
"epoch": 2.070671378091873,
|
|
"grad_norm": 2.046875,
|
|
"learning_rate": 5.327602000820413e-06,
|
|
"loss": 0.251662015914917,
|
|
"mean_token_accuracy": 0.9172984629869461,
|
|
"num_tokens": 5495843.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"entropy": 0.25756302401423453,
|
|
"epoch": 2.094228504122497,
|
|
"grad_norm": 3.203125,
|
|
"learning_rate": 5.087224498341815e-06,
|
|
"loss": 0.24423439502716066,
|
|
"mean_token_accuracy": 0.9185133799910545,
|
|
"num_tokens": 5558561.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"entropy": 0.2523910623043776,
|
|
"epoch": 2.1177856301531213,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 4.8505322944255096e-06,
|
|
"loss": 0.2407623052597046,
|
|
"mean_token_accuracy": 0.920753127336502,
|
|
"num_tokens": 5622036.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 2.1177856301531213,
|
|
"eval_entropy": 0.259268061319987,
|
|
"eval_loss": 0.2943871319293976,
|
|
"eval_mean_token_accuracy": 0.9055050121413337,
|
|
"eval_num_tokens": 5622036.0,
|
|
"eval_runtime": 17.6582,
|
|
"eval_samples_per_second": 20.274,
|
|
"eval_steps_per_second": 5.097,
|
|
"step": 900
|
|
},
|
|
{
|
|
"entropy": 0.2510908890515566,
|
|
"epoch": 2.1413427561837457,
|
|
"grad_norm": 2.53125,
|
|
"learning_rate": 4.617702942765641e-06,
|
|
"loss": 0.24126572608947755,
|
|
"mean_token_accuracy": 0.9219986706972122,
|
|
"num_tokens": 5682573.0,
|
|
"step": 910
|
|
},
|
|
{
|
|
"entropy": 0.25892969965934753,
|
|
"epoch": 2.1648998822143697,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 4.388911099353446e-06,
|
|
"loss": 0.2474604368209839,
|
|
"mean_token_accuracy": 0.9187318086624146,
|
|
"num_tokens": 5747004.0,
|
|
"step": 920
|
|
},
|
|
{
|
|
"entropy": 0.2502060953527689,
|
|
"epoch": 2.188457008244994,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 4.164328391459752e-06,
|
|
"loss": 0.24216799736022948,
|
|
"mean_token_accuracy": 0.9202440962195396,
|
|
"num_tokens": 5806808.0,
|
|
"step": 930
|
|
},
|
|
{
|
|
"entropy": 0.24517580829560756,
|
|
"epoch": 2.2120141342756185,
|
|
"grad_norm": 2.734375,
|
|
"learning_rate": 3.944123288889516e-06,
|
|
"loss": 0.2399127721786499,
|
|
"mean_token_accuracy": 0.9201767787337303,
|
|
"num_tokens": 5865387.0,
|
|
"step": 940
|
|
},
|
|
{
|
|
"entropy": 0.247905682772398,
|
|
"epoch": 2.2355712603062425,
|
|
"grad_norm": 2.75,
|
|
"learning_rate": 3.728460977604852e-06,
|
|
"loss": 0.2392266035079956,
|
|
"mean_token_accuracy": 0.921120385825634,
|
|
"num_tokens": 5926051.0,
|
|
"step": 950
|
|
},
|
|
{
|
|
"entropy": 0.25987638123333456,
|
|
"epoch": 2.259128386336867,
|
|
"grad_norm": 2.25,
|
|
"learning_rate": 3.517503235811468e-06,
|
|
"loss": 0.25406298637390134,
|
|
"mean_token_accuracy": 0.9165161669254303,
|
|
"num_tokens": 5991136.0,
|
|
"step": 960
|
|
},
|
|
{
|
|
"entropy": 0.2581822741776705,
|
|
"epoch": 2.2826855123674914,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 3.3114083126014097e-06,
|
|
"loss": 0.2514220714569092,
|
|
"mean_token_accuracy": 0.9170676961541175,
|
|
"num_tokens": 6057883.0,
|
|
"step": 970
|
|
},
|
|
{
|
|
"entropy": 0.25649656988680364,
|
|
"epoch": 2.3062426383981154,
|
|
"grad_norm": 2.65625,
|
|
"learning_rate": 3.110330809243134e-06,
|
|
"loss": 0.24820396900177003,
|
|
"mean_token_accuracy": 0.9190985530614852,
|
|
"num_tokens": 6121739.0,
|
|
"step": 980
|
|
},
|
|
{
|
|
"entropy": 0.27023419849574565,
|
|
"epoch": 2.32979976442874,
|
|
"grad_norm": 2.8125,
|
|
"learning_rate": 2.914421563208003e-06,
|
|
"loss": 0.26283965110778806,
|
|
"mean_token_accuracy": 0.9136555790901184,
|
|
"num_tokens": 6183942.0,
|
|
"step": 990
|
|
},
|
|
{
|
|
"entropy": 0.2571270074695349,
|
|
"epoch": 2.353356890459364,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 2.7238275350201603e-06,
|
|
"loss": 0.2580463647842407,
|
|
"mean_token_accuracy": 0.915487390756607,
|
|
"num_tokens": 6247108.0,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 2.353356890459364,
|
|
"eval_entropy": 0.2592703622248438,
|
|
"eval_loss": 0.2946915328502655,
|
|
"eval_mean_token_accuracy": 0.9055968953503503,
|
|
"eval_num_tokens": 6247108.0,
|
|
"eval_runtime": 17.7291,
|
|
"eval_samples_per_second": 20.193,
|
|
"eval_steps_per_second": 5.076,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"entropy": 0.2446678452193737,
|
|
"epoch": 2.3769140164899882,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 2.5386916980147126e-06,
|
|
"loss": 0.23602125644683838,
|
|
"mean_token_accuracy": 0.9227104559540749,
|
|
"num_tokens": 6310081.0,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"entropy": 0.252720745652914,
|
|
"epoch": 2.4004711425206127,
|
|
"grad_norm": 2.609375,
|
|
"learning_rate": 2.3591529310868376e-06,
|
|
"loss": 0.2390143871307373,
|
|
"mean_token_accuracy": 0.9209019631147385,
|
|
"num_tokens": 6374657.0,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"entropy": 0.25507905408740045,
|
|
"epoch": 2.4240282685512367,
|
|
"grad_norm": 2.671875,
|
|
"learning_rate": 2.18534591451236e-06,
|
|
"loss": 0.23744599819183348,
|
|
"mean_token_accuracy": 0.9203833773732185,
|
|
"num_tokens": 6436588.0,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"entropy": 0.2791239682585001,
|
|
"epoch": 2.447585394581861,
|
|
"grad_norm": 2.9375,
|
|
"learning_rate": 2.0174010289178914e-06,
|
|
"loss": 0.2733083486557007,
|
|
"mean_token_accuracy": 0.9116560816764832,
|
|
"num_tokens": 6499124.0,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"entropy": 0.2407016299664974,
|
|
"epoch": 2.471142520612485,
|
|
"grad_norm": 2.375,
|
|
"learning_rate": 1.8554442574763298e-06,
|
|
"loss": 0.2296748399734497,
|
|
"mean_token_accuracy": 0.9236260697245597,
|
|
"num_tokens": 6562914.0,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"entropy": 0.2620877519249916,
|
|
"epoch": 2.4946996466431095,
|
|
"grad_norm": 2.4375,
|
|
"learning_rate": 1.699597091401093e-06,
|
|
"loss": 0.25340471267700193,
|
|
"mean_token_accuracy": 0.9164399743080139,
|
|
"num_tokens": 6625628.0,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"entropy": 0.25387019738554956,
|
|
"epoch": 2.518256772673734,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 1.549976438809978e-06,
|
|
"loss": 0.25104894638061526,
|
|
"mean_token_accuracy": 0.9186207503080368,
|
|
"num_tokens": 6688963.0,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"entropy": 0.25147085413336756,
|
|
"epoch": 2.541813898704358,
|
|
"grad_norm": 2.3125,
|
|
"learning_rate": 1.4066945370270225e-06,
|
|
"loss": 0.24287889003753663,
|
|
"mean_token_accuracy": 0.9193637162446976,
|
|
"num_tokens": 6752441.0,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"entropy": 0.240979677811265,
|
|
"epoch": 2.5653710247349824,
|
|
"grad_norm": 2.515625,
|
|
"learning_rate": 1.2698588683881185e-06,
|
|
"loss": 0.23093729019165038,
|
|
"mean_token_accuracy": 0.9233990237116814,
|
|
"num_tokens": 6816965.0,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"entropy": 0.24780453816056253,
|
|
"epoch": 2.5889281507656063,
|
|
"grad_norm": 2.921875,
|
|
"learning_rate": 1.1395720796135868e-06,
|
|
"loss": 0.2352431058883667,
|
|
"mean_token_accuracy": 0.9226407021284103,
|
|
"num_tokens": 6878708.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 2.5889281507656063,
|
|
"eval_entropy": 0.2587818231847551,
|
|
"eval_loss": 0.29461440443992615,
|
|
"eval_mean_token_accuracy": 0.9055061519145966,
|
|
"eval_num_tokens": 6878708.0,
|
|
"eval_runtime": 17.5709,
|
|
"eval_samples_per_second": 20.375,
|
|
"eval_steps_per_second": 5.122,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"entropy": 0.25969714000821115,
|
|
"epoch": 2.6124852767962308,
|
|
"grad_norm": 2.734375,
|
|
"learning_rate": 1.0159319048081463e-06,
|
|
"loss": 0.2466489553451538,
|
|
"mean_token_accuracy": 0.9182975381612778,
|
|
"num_tokens": 6939885.0,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"entropy": 0.24832366406917572,
|
|
"epoch": 2.636042402826855,
|
|
"grad_norm": 2.3125,
|
|
"learning_rate": 8.990310921460865e-07,
|
|
"loss": 0.24750256538391113,
|
|
"mean_token_accuracy": 0.919737757742405,
|
|
"num_tokens": 6999229.0,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"entropy": 0.2580000825226307,
|
|
"epoch": 2.6595995288574796,
|
|
"grad_norm": 2.59375,
|
|
"learning_rate": 7.889573342965884e-07,
|
|
"loss": 0.2480959415435791,
|
|
"mean_token_accuracy": 0.9181187838315964,
|
|
"num_tokens": 7060446.0,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"entropy": 0.2526660870760679,
|
|
"epoch": 2.6831566548881036,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 6.857932026414305e-07,
|
|
"loss": 0.24074335098266603,
|
|
"mean_token_accuracy": 0.9200021162629127,
|
|
"num_tokens": 7123159.0,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"entropy": 0.2564856391400099,
|
|
"epoch": 2.706713780918728,
|
|
"grad_norm": 2.578125,
|
|
"learning_rate": 5.896160853344079e-07,
|
|
"loss": 0.24535846710205078,
|
|
"mean_token_accuracy": 0.9195480927824974,
|
|
"num_tokens": 7185033.0,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"entropy": 0.25719250217080114,
|
|
"epoch": 2.730270906949352,
|
|
"grad_norm": 2.8125,
|
|
"learning_rate": 5.00498129248913e-07,
|
|
"loss": 0.24295554161071778,
|
|
"mean_token_accuracy": 0.9188410475850105,
|
|
"num_tokens": 7248100.0,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"entropy": 0.2540380448102951,
|
|
"epoch": 2.7538280329799765,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 4.1850618585726477e-07,
|
|
"loss": 0.24861540794372558,
|
|
"mean_token_accuracy": 0.9181004285812377,
|
|
"num_tokens": 7309952.0,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"entropy": 0.2551457196474075,
|
|
"epoch": 2.777385159010601,
|
|
"grad_norm": 2.5625,
|
|
"learning_rate": 3.437017610823323e-07,
|
|
"loss": 0.2492974042892456,
|
|
"mean_token_accuracy": 0.9186578243970871,
|
|
"num_tokens": 7370009.0,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"entropy": 0.2607110310345888,
|
|
"epoch": 2.800942285041225,
|
|
"grad_norm": 2.125,
|
|
"learning_rate": 2.7614096915913325e-07,
|
|
"loss": 0.2521533727645874,
|
|
"mean_token_accuracy": 0.9172650650143623,
|
|
"num_tokens": 7432485.0,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"entropy": 0.2432629566639662,
|
|
"epoch": 2.8244994110718493,
|
|
"grad_norm": 2.515625,
|
|
"learning_rate": 2.1587449054095933e-07,
|
|
"loss": 0.23481087684631347,
|
|
"mean_token_accuracy": 0.9234458088874817,
|
|
"num_tokens": 7492335.0,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 2.8244994110718493,
|
|
"eval_entropy": 0.25882798383633293,
|
|
"eval_loss": 0.2944915294647217,
|
|
"eval_mean_token_accuracy": 0.9055290811591679,
|
|
"eval_num_tokens": 7492335.0,
|
|
"eval_runtime": 17.8608,
|
|
"eval_samples_per_second": 20.044,
|
|
"eval_steps_per_second": 5.039,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"entropy": 0.2645870730280876,
|
|
"epoch": 2.8480565371024733,
|
|
"grad_norm": 2.421875,
|
|
"learning_rate": 1.6294753388163886e-07,
|
|
"loss": 0.2564361333847046,
|
|
"mean_token_accuracy": 0.9171825960278511,
|
|
"num_tokens": 7555003.0,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"entropy": 0.26486502699553965,
|
|
"epoch": 2.8716136631330977,
|
|
"grad_norm": 2.609375,
|
|
"learning_rate": 1.1739980212245983e-07,
|
|
"loss": 0.2559476613998413,
|
|
"mean_token_accuracy": 0.9165961742401123,
|
|
"num_tokens": 7616130.0,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"entropy": 0.24283210895955562,
|
|
"epoch": 2.895170789163722,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 7.926546270915603e-08,
|
|
"loss": 0.23642585277557374,
|
|
"mean_token_accuracy": 0.9221925526857376,
|
|
"num_tokens": 7682597.0,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"entropy": 0.2405279938131571,
|
|
"epoch": 2.918727915194346,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 4.8573121961347226e-08,
|
|
"loss": 0.23437602519989015,
|
|
"mean_token_accuracy": 0.9235370129346847,
|
|
"num_tokens": 7744071.0,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"entropy": 0.254606057330966,
|
|
"epoch": 2.9422850412249706,
|
|
"grad_norm": 2.71875,
|
|
"learning_rate": 2.5345803613617826e-08,
|
|
"loss": 0.25019450187683107,
|
|
"mean_token_accuracy": 0.91759652197361,
|
|
"num_tokens": 7805475.0,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"entropy": 0.2431744422763586,
|
|
"epoch": 2.9658421672555946,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 9.600931544362369e-09,
|
|
"loss": 0.23505258560180664,
|
|
"mean_token_accuracy": 0.9224522054195404,
|
|
"num_tokens": 7867869.0,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"entropy": 0.25586013197898866,
|
|
"epoch": 2.989399293286219,
|
|
"grad_norm": 2.53125,
|
|
"learning_rate": 1.3503167053285916e-09,
|
|
"loss": 0.24874160289764405,
|
|
"mean_token_accuracy": 0.9177542105317116,
|
|
"num_tokens": 7930815.0,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"eval_entropy": 0.25879754341310923,
|
|
"eval_loss": 0.2946431338787079,
|
|
"eval_mean_token_accuracy": 0.9054906017250485,
|
|
"eval_num_tokens": 7960173.0,
|
|
"eval_runtime": 17.729,
|
|
"eval_samples_per_second": 20.193,
|
|
"eval_steps_per_second": 5.076,
|
|
"step": 1275
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1275,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.696316897815552e+16,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|