1610 lines
46 KiB
JSON
1610 lines
46 KiB
JSON
{
|
|
"best_global_step": 3000,
|
|
"best_metric": 0.9706581830978394,
|
|
"best_model_checkpoint": "/workspace/tanglish/checkpoints/qwen_lora_v1/lora/checkpoint-3000",
|
|
"epoch": 0.6217616580310881,
|
|
"eval_steps": 500,
|
|
"global_step": 3000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 1.4381825268268584,
|
|
"epoch": 0.004145077720207254,
|
|
"grad_norm": 2.1717000007629395,
|
|
"learning_rate": 5.248618784530387e-06,
|
|
"loss": 4.0762,
|
|
"mean_token_accuracy": 0.4441531464457512,
|
|
"num_tokens": 51480.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 1.5037598952651023,
|
|
"epoch": 0.008290155440414507,
|
|
"grad_norm": 2.724255323410034,
|
|
"learning_rate": 1.0773480662983426e-05,
|
|
"loss": 3.8127,
|
|
"mean_token_accuracy": 0.4525611069053411,
|
|
"num_tokens": 103460.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 1.901321442425251,
|
|
"epoch": 0.012435233160621761,
|
|
"grad_norm": 1.1939035654067993,
|
|
"learning_rate": 1.6298342541436465e-05,
|
|
"loss": 3.1292,
|
|
"mean_token_accuracy": 0.48335910513997077,
|
|
"num_tokens": 154920.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 2.358680176734924,
|
|
"epoch": 0.016580310880829015,
|
|
"grad_norm": 0.7893019318580627,
|
|
"learning_rate": 2.1823204419889505e-05,
|
|
"loss": 2.3084,
|
|
"mean_token_accuracy": 0.554056815057993,
|
|
"num_tokens": 207786.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 1.830135741829872,
|
|
"epoch": 0.02072538860103627,
|
|
"grad_norm": 0.5736128091812134,
|
|
"learning_rate": 2.7348066298342545e-05,
|
|
"loss": 1.708,
|
|
"mean_token_accuracy": 0.6866068348288537,
|
|
"num_tokens": 256847.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 1.4562795095145702,
|
|
"epoch": 0.024870466321243522,
|
|
"grad_norm": 0.9956451058387756,
|
|
"learning_rate": 3.287292817679558e-05,
|
|
"loss": 1.4132,
|
|
"mean_token_accuracy": 0.7327493987977505,
|
|
"num_tokens": 309233.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 1.4073016680777073,
|
|
"epoch": 0.029015544041450778,
|
|
"grad_norm": 0.8126519918441772,
|
|
"learning_rate": 3.8397790055248625e-05,
|
|
"loss": 1.3879,
|
|
"mean_token_accuracy": 0.7353364005684853,
|
|
"num_tokens": 361206.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 1.28325657248497,
|
|
"epoch": 0.03316062176165803,
|
|
"grad_norm": 0.6408252120018005,
|
|
"learning_rate": 4.392265193370166e-05,
|
|
"loss": 1.2517,
|
|
"mean_token_accuracy": 0.7543429806828499,
|
|
"num_tokens": 412093.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 1.219002117216587,
|
|
"epoch": 0.03730569948186528,
|
|
"grad_norm": 0.6377286911010742,
|
|
"learning_rate": 4.94475138121547e-05,
|
|
"loss": 1.2238,
|
|
"mean_token_accuracy": 0.7544910028576851,
|
|
"num_tokens": 463753.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 1.2931843183934688,
|
|
"epoch": 0.04145077720207254,
|
|
"grad_norm": 0.7240749001502991,
|
|
"learning_rate": 5.4972375690607735e-05,
|
|
"loss": 1.3026,
|
|
"mean_token_accuracy": 0.7400020457804203,
|
|
"num_tokens": 515356.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 1.2196356683969498,
|
|
"epoch": 0.04559585492227979,
|
|
"grad_norm": 0.7647127509117126,
|
|
"learning_rate": 6.049723756906077e-05,
|
|
"loss": 1.2411,
|
|
"mean_token_accuracy": 0.7544264450669289,
|
|
"num_tokens": 565236.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 1.171409723162651,
|
|
"epoch": 0.049740932642487044,
|
|
"grad_norm": 0.7827390432357788,
|
|
"learning_rate": 6.602209944751382e-05,
|
|
"loss": 1.1776,
|
|
"mean_token_accuracy": 0.7626889519393444,
|
|
"num_tokens": 615515.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 1.1406699948012828,
|
|
"epoch": 0.0538860103626943,
|
|
"grad_norm": 1.0029950141906738,
|
|
"learning_rate": 7.154696132596685e-05,
|
|
"loss": 1.1546,
|
|
"mean_token_accuracy": 0.766391609609127,
|
|
"num_tokens": 666062.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 1.1219736263155937,
|
|
"epoch": 0.058031088082901555,
|
|
"grad_norm": 1.0503102540969849,
|
|
"learning_rate": 7.70718232044199e-05,
|
|
"loss": 1.1352,
|
|
"mean_token_accuracy": 0.7641930885612964,
|
|
"num_tokens": 719494.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 1.1847783662378788,
|
|
"epoch": 0.06217616580310881,
|
|
"grad_norm": 0.8290141820907593,
|
|
"learning_rate": 8.259668508287292e-05,
|
|
"loss": 1.202,
|
|
"mean_token_accuracy": 0.7585213594138622,
|
|
"num_tokens": 772839.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 1.1723797246813774,
|
|
"epoch": 0.06632124352331606,
|
|
"grad_norm": 0.877815842628479,
|
|
"learning_rate": 8.812154696132597e-05,
|
|
"loss": 1.1825,
|
|
"mean_token_accuracy": 0.7599225588142872,
|
|
"num_tokens": 825189.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 1.1410918802022934,
|
|
"epoch": 0.07046632124352331,
|
|
"grad_norm": 0.7032957673072815,
|
|
"learning_rate": 9.364640883977901e-05,
|
|
"loss": 1.1333,
|
|
"mean_token_accuracy": 0.7682646617293358,
|
|
"num_tokens": 877294.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 1.1239484950900078,
|
|
"epoch": 0.07461139896373056,
|
|
"grad_norm": 0.8480610847473145,
|
|
"learning_rate": 9.917127071823204e-05,
|
|
"loss": 1.1536,
|
|
"mean_token_accuracy": 0.7669501408934594,
|
|
"num_tokens": 927234.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 1.110478015244007,
|
|
"epoch": 0.07875647668393783,
|
|
"grad_norm": 0.7844213843345642,
|
|
"learning_rate": 0.0001046961325966851,
|
|
"loss": 1.1255,
|
|
"mean_token_accuracy": 0.7683746472001076,
|
|
"num_tokens": 980398.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 1.1618236370384694,
|
|
"epoch": 0.08290155440414508,
|
|
"grad_norm": 0.7246690988540649,
|
|
"learning_rate": 0.00011022099447513811,
|
|
"loss": 1.1729,
|
|
"mean_token_accuracy": 0.7629252031445504,
|
|
"num_tokens": 1030335.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 1.1364129185676575,
|
|
"epoch": 0.08704663212435233,
|
|
"grad_norm": 0.7912014722824097,
|
|
"learning_rate": 0.00011574585635359116,
|
|
"loss": 1.152,
|
|
"mean_token_accuracy": 0.7651066802442074,
|
|
"num_tokens": 1083694.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 1.0954272992908956,
|
|
"epoch": 0.09119170984455958,
|
|
"grad_norm": 0.7310007214546204,
|
|
"learning_rate": 0.00012127071823204422,
|
|
"loss": 1.1039,
|
|
"mean_token_accuracy": 0.7700782015919685,
|
|
"num_tokens": 1134346.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 1.1179637335240842,
|
|
"epoch": 0.09533678756476684,
|
|
"grad_norm": 0.7415347099304199,
|
|
"learning_rate": 0.00012679558011049723,
|
|
"loss": 1.1267,
|
|
"mean_token_accuracy": 0.7677439145743847,
|
|
"num_tokens": 1188485.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 1.0841590076684953,
|
|
"epoch": 0.09948186528497409,
|
|
"grad_norm": 0.8192847371101379,
|
|
"learning_rate": 0.00013232044198895027,
|
|
"loss": 1.095,
|
|
"mean_token_accuracy": 0.7726523540914059,
|
|
"num_tokens": 1239371.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 1.0643972344696522,
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 0.8887308239936829,
|
|
"learning_rate": 0.00013784530386740332,
|
|
"loss": 1.0762,
|
|
"mean_token_accuracy": 0.7769059598445892,
|
|
"num_tokens": 1290491.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"eval_entropy": 1.095322513439524,
|
|
"eval_loss": 1.1184215545654297,
|
|
"eval_mean_token_accuracy": 0.7680243507144958,
|
|
"eval_num_tokens": 1290491.0,
|
|
"eval_runtime": 198.0973,
|
|
"eval_samples_per_second": 20.51,
|
|
"eval_steps_per_second": 2.564,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 1.0820845410227775,
|
|
"epoch": 0.1077720207253886,
|
|
"grad_norm": 0.7347564697265625,
|
|
"learning_rate": 0.00014337016574585636,
|
|
"loss": 1.0979,
|
|
"mean_token_accuracy": 0.7743508994579316,
|
|
"num_tokens": 1341205.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"entropy": 1.1001912213861942,
|
|
"epoch": 0.11191709844559586,
|
|
"grad_norm": 0.690479040145874,
|
|
"learning_rate": 0.0001488950276243094,
|
|
"loss": 1.1105,
|
|
"mean_token_accuracy": 0.7694470949470997,
|
|
"num_tokens": 1394461.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"entropy": 1.0999469496309757,
|
|
"epoch": 0.11606217616580311,
|
|
"grad_norm": 0.6917790174484253,
|
|
"learning_rate": 0.00015441988950276245,
|
|
"loss": 1.117,
|
|
"mean_token_accuracy": 0.7682196438312531,
|
|
"num_tokens": 1447828.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"entropy": 1.0735741600394249,
|
|
"epoch": 0.12020725388601036,
|
|
"grad_norm": 0.833886444568634,
|
|
"learning_rate": 0.00015994475138121547,
|
|
"loss": 1.086,
|
|
"mean_token_accuracy": 0.7741858348250389,
|
|
"num_tokens": 1498479.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"entropy": 1.0962664999067784,
|
|
"epoch": 0.12435233160621761,
|
|
"grad_norm": 0.6647179126739502,
|
|
"learning_rate": 0.0001654696132596685,
|
|
"loss": 1.0958,
|
|
"mean_token_accuracy": 0.7741696208715438,
|
|
"num_tokens": 1548137.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"entropy": 1.0694029107689857,
|
|
"epoch": 0.12849740932642487,
|
|
"grad_norm": 0.6861886382102966,
|
|
"learning_rate": 0.00017099447513812155,
|
|
"loss": 1.0829,
|
|
"mean_token_accuracy": 0.7720654770731926,
|
|
"num_tokens": 1599638.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"entropy": 1.075825183838606,
|
|
"epoch": 0.13264248704663212,
|
|
"grad_norm": 0.7000752687454224,
|
|
"learning_rate": 0.0001765193370165746,
|
|
"loss": 1.0825,
|
|
"mean_token_accuracy": 0.7762658178806305,
|
|
"num_tokens": 1650323.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"entropy": 1.083068310469389,
|
|
"epoch": 0.13678756476683937,
|
|
"grad_norm": 0.7568516731262207,
|
|
"learning_rate": 0.00018204419889502764,
|
|
"loss": 1.0918,
|
|
"mean_token_accuracy": 0.7720131345093251,
|
|
"num_tokens": 1701215.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"entropy": 1.0660202227532865,
|
|
"epoch": 0.14093264248704662,
|
|
"grad_norm": 0.8106701374053955,
|
|
"learning_rate": 0.00018756906077348066,
|
|
"loss": 1.073,
|
|
"mean_token_accuracy": 0.7741113729774952,
|
|
"num_tokens": 1751275.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"entropy": 1.00438876375556,
|
|
"epoch": 0.14507772020725387,
|
|
"grad_norm": 0.6197234988212585,
|
|
"learning_rate": 0.0001930939226519337,
|
|
"loss": 1.0234,
|
|
"mean_token_accuracy": 0.7818360298871994,
|
|
"num_tokens": 1803256.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"entropy": 1.1083124563097955,
|
|
"epoch": 0.14922279792746113,
|
|
"grad_norm": 0.6046066284179688,
|
|
"learning_rate": 0.00019861878453038677,
|
|
"loss": 1.122,
|
|
"mean_token_accuracy": 0.7704763405025006,
|
|
"num_tokens": 1855372.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"entropy": 1.097743584215641,
|
|
"epoch": 0.1533678756476684,
|
|
"grad_norm": 0.6518492102622986,
|
|
"learning_rate": 0.00019999941280374287,
|
|
"loss": 1.1116,
|
|
"mean_token_accuracy": 0.7682732604444027,
|
|
"num_tokens": 1906015.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"entropy": 1.1098480015993117,
|
|
"epoch": 0.15751295336787566,
|
|
"grad_norm": 0.6173190474510193,
|
|
"learning_rate": 0.00019999680305650554,
|
|
"loss": 1.1173,
|
|
"mean_token_accuracy": 0.7627138867974281,
|
|
"num_tokens": 1959638.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"entropy": 1.0928526185452938,
|
|
"epoch": 0.1616580310880829,
|
|
"grad_norm": 0.6099253296852112,
|
|
"learning_rate": 0.00019999210556868917,
|
|
"loss": 1.1156,
|
|
"mean_token_accuracy": 0.7711292572319508,
|
|
"num_tokens": 2010476.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"entropy": 1.0808037273585795,
|
|
"epoch": 0.16580310880829016,
|
|
"grad_norm": 0.6099389791488647,
|
|
"learning_rate": 0.0001999853204383683,
|
|
"loss": 1.087,
|
|
"mean_token_accuracy": 0.7749647930264473,
|
|
"num_tokens": 2060494.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"entropy": 1.048035841435194,
|
|
"epoch": 0.1699481865284974,
|
|
"grad_norm": 0.5170192122459412,
|
|
"learning_rate": 0.0001999764478072033,
|
|
"loss": 1.0642,
|
|
"mean_token_accuracy": 0.7727734588086606,
|
|
"num_tokens": 2113260.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"entropy": 1.0665065109729768,
|
|
"epoch": 0.17409326424870467,
|
|
"grad_norm": 0.6461181640625,
|
|
"learning_rate": 0.00019996548786043768,
|
|
"loss": 1.0795,
|
|
"mean_token_accuracy": 0.777741952240467,
|
|
"num_tokens": 2164157.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"entropy": 1.0095063984394073,
|
|
"epoch": 0.17823834196891192,
|
|
"grad_norm": 0.5376838445663452,
|
|
"learning_rate": 0.00019995244082689396,
|
|
"loss": 1.0238,
|
|
"mean_token_accuracy": 0.7842500291764736,
|
|
"num_tokens": 2215501.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"entropy": 1.0482122428715228,
|
|
"epoch": 0.18238341968911917,
|
|
"grad_norm": 0.5362586379051208,
|
|
"learning_rate": 0.00019993730697896912,
|
|
"loss": 1.0567,
|
|
"mean_token_accuracy": 0.7757732585072518,
|
|
"num_tokens": 2265908.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"entropy": 1.0375815689563752,
|
|
"epoch": 0.18652849740932642,
|
|
"grad_norm": 0.6826100945472717,
|
|
"learning_rate": 0.0001999200866326287,
|
|
"loss": 1.055,
|
|
"mean_token_accuracy": 0.777643445879221,
|
|
"num_tokens": 2316578.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"entropy": 1.0592326551675797,
|
|
"epoch": 0.19067357512953367,
|
|
"grad_norm": 0.4966115355491638,
|
|
"learning_rate": 0.00019990078014740036,
|
|
"loss": 1.0682,
|
|
"mean_token_accuracy": 0.7757245123386383,
|
|
"num_tokens": 2367491.0,
|
|
"step": 920
|
|
},
|
|
{
|
|
"entropy": 1.0522546865046025,
|
|
"epoch": 0.19481865284974093,
|
|
"grad_norm": 0.5403271317481995,
|
|
"learning_rate": 0.00019987938792636632,
|
|
"loss": 1.0597,
|
|
"mean_token_accuracy": 0.7824762858450413,
|
|
"num_tokens": 2418531.0,
|
|
"step": 940
|
|
},
|
|
{
|
|
"entropy": 1.0112147249281407,
|
|
"epoch": 0.19896373056994818,
|
|
"grad_norm": 0.591804027557373,
|
|
"learning_rate": 0.00019985591041615496,
|
|
"loss": 1.0179,
|
|
"mean_token_accuracy": 0.7824166387319564,
|
|
"num_tokens": 2468312.0,
|
|
"step": 960
|
|
},
|
|
{
|
|
"entropy": 1.0457121133804321,
|
|
"epoch": 0.20310880829015543,
|
|
"grad_norm": 0.4792396128177643,
|
|
"learning_rate": 0.00019983034810693137,
|
|
"loss": 1.0567,
|
|
"mean_token_accuracy": 0.7786681324243545,
|
|
"num_tokens": 2518124.0,
|
|
"step": 980
|
|
},
|
|
{
|
|
"entropy": 1.0062039978802204,
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 0.6134971976280212,
|
|
"learning_rate": 0.00019980270153238742,
|
|
"loss": 1.0284,
|
|
"mean_token_accuracy": 0.7829495541751385,
|
|
"num_tokens": 2569706.0,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"eval_entropy": 1.0445965941496722,
|
|
"eval_loss": 1.059064507484436,
|
|
"eval_mean_token_accuracy": 0.7763740899055962,
|
|
"eval_num_tokens": 2569706.0,
|
|
"eval_runtime": 197.7332,
|
|
"eval_samples_per_second": 20.548,
|
|
"eval_steps_per_second": 2.569,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"entropy": 1.0440333500504493,
|
|
"epoch": 0.21139896373056996,
|
|
"grad_norm": 0.6377447843551636,
|
|
"learning_rate": 0.0001997729712697303,
|
|
"loss": 1.0571,
|
|
"mean_token_accuracy": 0.7803341060876846,
|
|
"num_tokens": 2618883.0,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"entropy": 1.021172496676445,
|
|
"epoch": 0.2155440414507772,
|
|
"grad_norm": 0.5175086855888367,
|
|
"learning_rate": 0.0001997411579396706,
|
|
"loss": 1.0292,
|
|
"mean_token_accuracy": 0.7807957075536252,
|
|
"num_tokens": 2670732.0,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"entropy": 1.0275346338748932,
|
|
"epoch": 0.21968911917098446,
|
|
"grad_norm": 0.5990492701530457,
|
|
"learning_rate": 0.00019970726220640929,
|
|
"loss": 1.0447,
|
|
"mean_token_accuracy": 0.77860803976655,
|
|
"num_tokens": 2722930.0,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"entropy": 1.0109625704586507,
|
|
"epoch": 0.22383419689119172,
|
|
"grad_norm": 0.4519039988517761,
|
|
"learning_rate": 0.00019967128477762404,
|
|
"loss": 1.0286,
|
|
"mean_token_accuracy": 0.7781259723007679,
|
|
"num_tokens": 2775645.0,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"entropy": 1.024503505975008,
|
|
"epoch": 0.22797927461139897,
|
|
"grad_norm": 0.5599638819694519,
|
|
"learning_rate": 0.00019963322640445424,
|
|
"loss": 1.0358,
|
|
"mean_token_accuracy": 0.7802064262330533,
|
|
"num_tokens": 2829031.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"entropy": 1.000456291437149,
|
|
"epoch": 0.23212435233160622,
|
|
"grad_norm": 0.42756351828575134,
|
|
"learning_rate": 0.0001995930878814854,
|
|
"loss": 1.0071,
|
|
"mean_token_accuracy": 0.7876958273351192,
|
|
"num_tokens": 2882439.0,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"entropy": 1.0316657554358244,
|
|
"epoch": 0.23626943005181347,
|
|
"grad_norm": 0.4877125322818756,
|
|
"learning_rate": 0.0001995508700467325,
|
|
"loss": 1.0422,
|
|
"mean_token_accuracy": 0.7804410368204117,
|
|
"num_tokens": 2935155.0,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"entropy": 1.0400867745280267,
|
|
"epoch": 0.24041450777202072,
|
|
"grad_norm": 0.501750111579895,
|
|
"learning_rate": 0.00019950657378162262,
|
|
"loss": 1.06,
|
|
"mean_token_accuracy": 0.7792659178376198,
|
|
"num_tokens": 2986224.0,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"entropy": 1.0030486956238747,
|
|
"epoch": 0.24455958549222798,
|
|
"grad_norm": 0.4440039396286011,
|
|
"learning_rate": 0.00019946020001097644,
|
|
"loss": 1.0081,
|
|
"mean_token_accuracy": 0.7861276485025883,
|
|
"num_tokens": 3037391.0,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"entropy": 1.0251952454447746,
|
|
"epoch": 0.24870466321243523,
|
|
"grad_norm": 0.6198555827140808,
|
|
"learning_rate": 0.00019941174970298888,
|
|
"loss": 1.036,
|
|
"mean_token_accuracy": 0.7791207812726497,
|
|
"num_tokens": 3087069.0,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"entropy": 1.0355947606265545,
|
|
"epoch": 0.2528497409326425,
|
|
"grad_norm": 0.43304187059402466,
|
|
"learning_rate": 0.0001993612238692091,
|
|
"loss": 1.0457,
|
|
"mean_token_accuracy": 0.7767570614814758,
|
|
"num_tokens": 3140164.0,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"entropy": 1.0067585781216621,
|
|
"epoch": 0.25699481865284973,
|
|
"grad_norm": 0.5553335547447205,
|
|
"learning_rate": 0.0001993086235645191,
|
|
"loss": 1.0238,
|
|
"mean_token_accuracy": 0.7811847984790802,
|
|
"num_tokens": 3192879.0,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"entropy": 1.0447653152048588,
|
|
"epoch": 0.261139896373057,
|
|
"grad_norm": 0.5924249291419983,
|
|
"learning_rate": 0.0001992539498871119,
|
|
"loss": 1.049,
|
|
"mean_token_accuracy": 0.7775058671832085,
|
|
"num_tokens": 3244981.0,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"entropy": 1.0076527014374732,
|
|
"epoch": 0.26528497409326424,
|
|
"grad_norm": 0.5087696313858032,
|
|
"learning_rate": 0.00019919720397846855,
|
|
"loss": 1.018,
|
|
"mean_token_accuracy": 0.784369270503521,
|
|
"num_tokens": 3293733.0,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"entropy": 1.0454526104032993,
|
|
"epoch": 0.2694300518134715,
|
|
"grad_norm": 0.5536513924598694,
|
|
"learning_rate": 0.00019913838702333432,
|
|
"loss": 1.0648,
|
|
"mean_token_accuracy": 0.7793882772326469,
|
|
"num_tokens": 3343268.0,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"entropy": 1.0259532511234284,
|
|
"epoch": 0.27357512953367874,
|
|
"grad_norm": 0.5604416728019714,
|
|
"learning_rate": 0.00019907750024969386,
|
|
"loss": 1.0366,
|
|
"mean_token_accuracy": 0.779184664785862,
|
|
"num_tokens": 3395934.0,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"entropy": 1.0157927133142948,
|
|
"epoch": 0.277720207253886,
|
|
"grad_norm": 0.45751333236694336,
|
|
"learning_rate": 0.00019901454492874572,
|
|
"loss": 1.0379,
|
|
"mean_token_accuracy": 0.7799046277999878,
|
|
"num_tokens": 3448448.0,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"entropy": 1.013536487519741,
|
|
"epoch": 0.28186528497409324,
|
|
"grad_norm": 0.4128531813621521,
|
|
"learning_rate": 0.00019894952237487566,
|
|
"loss": 1.0201,
|
|
"mean_token_accuracy": 0.7843268230557442,
|
|
"num_tokens": 3499346.0,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"entropy": 1.0443965464830398,
|
|
"epoch": 0.2860103626943005,
|
|
"grad_norm": 0.49815455079078674,
|
|
"learning_rate": 0.0001988824339456294,
|
|
"loss": 1.0593,
|
|
"mean_token_accuracy": 0.7785378187894821,
|
|
"num_tokens": 3551298.0,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"entropy": 1.011622816324234,
|
|
"epoch": 0.29015544041450775,
|
|
"grad_norm": 0.46742376685142517,
|
|
"learning_rate": 0.000198813281041684,
|
|
"loss": 1.0193,
|
|
"mean_token_accuracy": 0.7825383462011815,
|
|
"num_tokens": 3602656.0,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"entropy": 0.9844559028744697,
|
|
"epoch": 0.29430051813471503,
|
|
"grad_norm": 0.40691128373146057,
|
|
"learning_rate": 0.0001987420651068189,
|
|
"loss": 1.0042,
|
|
"mean_token_accuracy": 0.7844808354973793,
|
|
"num_tokens": 3654678.0,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"entropy": 1.0059320218861103,
|
|
"epoch": 0.29844559585492225,
|
|
"grad_norm": 0.42339223623275757,
|
|
"learning_rate": 0.00019866878762788562,
|
|
"loss": 1.0142,
|
|
"mean_token_accuracy": 0.7823195405304432,
|
|
"num_tokens": 3705949.0,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"entropy": 0.9958507776260376,
|
|
"epoch": 0.30259067357512953,
|
|
"grad_norm": 0.48249927163124084,
|
|
"learning_rate": 0.00019859345013477665,
|
|
"loss": 1.0121,
|
|
"mean_token_accuracy": 0.7889884993433952,
|
|
"num_tokens": 3756251.0,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"entropy": 0.9966832913458348,
|
|
"epoch": 0.3067357512953368,
|
|
"grad_norm": 0.3815440237522125,
|
|
"learning_rate": 0.00019851605420039377,
|
|
"loss": 1.0226,
|
|
"mean_token_accuracy": 0.7830308295786381,
|
|
"num_tokens": 3808938.0,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"entropy": 0.9885330185294151,
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 0.4722138047218323,
|
|
"learning_rate": 0.00019843660144061497,
|
|
"loss": 0.9955,
|
|
"mean_token_accuracy": 0.7874660678207874,
|
|
"num_tokens": 3858248.0,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"eval_entropy": 1.032557885595194,
|
|
"eval_loss": 1.0242446660995483,
|
|
"eval_mean_token_accuracy": 0.7811775283785317,
|
|
"eval_num_tokens": 3858248.0,
|
|
"eval_runtime": 198.2363,
|
|
"eval_samples_per_second": 20.496,
|
|
"eval_steps_per_second": 2.563,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"entropy": 1.0101246651262046,
|
|
"epoch": 0.3150259067357513,
|
|
"grad_norm": 0.42561495304107666,
|
|
"learning_rate": 0.00019835509351426073,
|
|
"loss": 1.0177,
|
|
"mean_token_accuracy": 0.7840446695685387,
|
|
"num_tokens": 3909740.0,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"entropy": 0.9913592956960201,
|
|
"epoch": 0.31917098445595854,
|
|
"grad_norm": 0.4773913025856018,
|
|
"learning_rate": 0.00019827153212305954,
|
|
"loss": 1.0043,
|
|
"mean_token_accuracy": 0.7856849782168865,
|
|
"num_tokens": 3958454.0,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"entropy": 1.0170710057020187,
|
|
"epoch": 0.3233160621761658,
|
|
"grad_norm": 0.4388096034526825,
|
|
"learning_rate": 0.00019818591901161225,
|
|
"loss": 1.0284,
|
|
"mean_token_accuracy": 0.7777703814208508,
|
|
"num_tokens": 4010172.0,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"entropy": 1.0317365549504758,
|
|
"epoch": 0.32746113989637304,
|
|
"grad_norm": 0.4321315884590149,
|
|
"learning_rate": 0.00019809825596735567,
|
|
"loss": 1.0507,
|
|
"mean_token_accuracy": 0.782105403393507,
|
|
"num_tokens": 4061342.0,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"entropy": 1.0285033784806727,
|
|
"epoch": 0.3316062176165803,
|
|
"grad_norm": 0.4316249489784241,
|
|
"learning_rate": 0.00019800854482052526,
|
|
"loss": 1.053,
|
|
"mean_token_accuracy": 0.7812598653137683,
|
|
"num_tokens": 4112308.0,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"entropy": 0.959014667570591,
|
|
"epoch": 0.33575129533678755,
|
|
"grad_norm": 0.4981176555156708,
|
|
"learning_rate": 0.0001979167874441169,
|
|
"loss": 0.9713,
|
|
"mean_token_accuracy": 0.7900767177343369,
|
|
"num_tokens": 4163721.0,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"entropy": 0.9863104440271855,
|
|
"epoch": 0.3398963730569948,
|
|
"grad_norm": 0.5106677412986755,
|
|
"learning_rate": 0.00019782298575384783,
|
|
"loss": 0.9888,
|
|
"mean_token_accuracy": 0.786780058592558,
|
|
"num_tokens": 4214428.0,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"entropy": 0.9868629761040211,
|
|
"epoch": 0.34404145077720205,
|
|
"grad_norm": 0.4558473527431488,
|
|
"learning_rate": 0.00019772714170811662,
|
|
"loss": 1.0034,
|
|
"mean_token_accuracy": 0.7846312940120697,
|
|
"num_tokens": 4267755.0,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"entropy": 1.0307603180408478,
|
|
"epoch": 0.34818652849740933,
|
|
"grad_norm": 0.4738333225250244,
|
|
"learning_rate": 0.00019762925730796224,
|
|
"loss": 1.0414,
|
|
"mean_token_accuracy": 0.7802501529455185,
|
|
"num_tokens": 4317809.0,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"entropy": 1.0127912051975727,
|
|
"epoch": 0.35233160621761656,
|
|
"grad_norm": 0.48111966252326965,
|
|
"learning_rate": 0.0001975293345970224,
|
|
"loss": 1.0147,
|
|
"mean_token_accuracy": 0.7798892751336097,
|
|
"num_tokens": 4370912.0,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"entropy": 0.9599951460957528,
|
|
"epoch": 0.35647668393782384,
|
|
"grad_norm": 0.4724443256855011,
|
|
"learning_rate": 0.00019742737566149078,
|
|
"loss": 0.9585,
|
|
"mean_token_accuracy": 0.7909531377255916,
|
|
"num_tokens": 4423603.0,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"entropy": 1.0278802808374166,
|
|
"epoch": 0.3606217616580311,
|
|
"grad_norm": 0.5354254245758057,
|
|
"learning_rate": 0.0001973233826300735,
|
|
"loss": 1.035,
|
|
"mean_token_accuracy": 0.7811824880540371,
|
|
"num_tokens": 4476943.0,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"entropy": 0.9914717882871628,
|
|
"epoch": 0.36476683937823834,
|
|
"grad_norm": 0.47376251220703125,
|
|
"learning_rate": 0.00019721735767394468,
|
|
"loss": 1.0099,
|
|
"mean_token_accuracy": 0.781830509006977,
|
|
"num_tokens": 4527823.0,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"entropy": 1.020705473423004,
|
|
"epoch": 0.3689119170984456,
|
|
"grad_norm": 0.4668932259082794,
|
|
"learning_rate": 0.00019710930300670108,
|
|
"loss": 1.0274,
|
|
"mean_token_accuracy": 0.7800539046525955,
|
|
"num_tokens": 4581082.0,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"entropy": 1.0137963436543942,
|
|
"epoch": 0.37305699481865284,
|
|
"grad_norm": 0.4402451515197754,
|
|
"learning_rate": 0.00019699922088431605,
|
|
"loss": 1.0194,
|
|
"mean_token_accuracy": 0.7841047510504723,
|
|
"num_tokens": 4630834.0,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"entropy": 0.9691961273550987,
|
|
"epoch": 0.3772020725388601,
|
|
"grad_norm": 0.4267026484012604,
|
|
"learning_rate": 0.00019688711360509213,
|
|
"loss": 0.9716,
|
|
"mean_token_accuracy": 0.7890592306852341,
|
|
"num_tokens": 4682795.0,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"entropy": 0.9769839994609356,
|
|
"epoch": 0.38134715025906735,
|
|
"grad_norm": 0.3717665374279022,
|
|
"learning_rate": 0.00019677298350961333,
|
|
"loss": 0.9986,
|
|
"mean_token_accuracy": 0.7845027431845665,
|
|
"num_tokens": 4734983.0,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"entropy": 1.0078977264463902,
|
|
"epoch": 0.3854922279792746,
|
|
"grad_norm": 0.5216906070709229,
|
|
"learning_rate": 0.00019665683298069616,
|
|
"loss": 1.0144,
|
|
"mean_token_accuracy": 0.7834078282117843,
|
|
"num_tokens": 4786331.0,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"entropy": 0.9985935389995575,
|
|
"epoch": 0.38963730569948185,
|
|
"grad_norm": 0.408572793006897,
|
|
"learning_rate": 0.00019653866444333985,
|
|
"loss": 1.0096,
|
|
"mean_token_accuracy": 0.7860454954206944,
|
|
"num_tokens": 4836661.0,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"entropy": 0.9866756744682789,
|
|
"epoch": 0.39378238341968913,
|
|
"grad_norm": 0.43202802538871765,
|
|
"learning_rate": 0.00019641848036467576,
|
|
"loss": 1.0044,
|
|
"mean_token_accuracy": 0.7888299062848091,
|
|
"num_tokens": 4887993.0,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"entropy": 0.9607140734791756,
|
|
"epoch": 0.39792746113989635,
|
|
"grad_norm": 0.4784650206565857,
|
|
"learning_rate": 0.00019629628325391595,
|
|
"loss": 0.9777,
|
|
"mean_token_accuracy": 0.7884210042655468,
|
|
"num_tokens": 4937865.0,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"entropy": 0.947428410500288,
|
|
"epoch": 0.40207253886010363,
|
|
"grad_norm": 0.5031050443649292,
|
|
"learning_rate": 0.0001961720756623006,
|
|
"loss": 0.9529,
|
|
"mean_token_accuracy": 0.7923657827079296,
|
|
"num_tokens": 4988659.0,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"entropy": 0.9914572931826114,
|
|
"epoch": 0.40621761658031086,
|
|
"grad_norm": 0.46874579787254333,
|
|
"learning_rate": 0.0001960458601830449,
|
|
"loss": 1.0075,
|
|
"mean_token_accuracy": 0.7866203740239144,
|
|
"num_tokens": 5038807.0,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"entropy": 1.0107652366161346,
|
|
"epoch": 0.41036269430051814,
|
|
"grad_norm": 0.5741726160049438,
|
|
"learning_rate": 0.00019591763945128493,
|
|
"loss": 1.0242,
|
|
"mean_token_accuracy": 0.7811738647520542,
|
|
"num_tokens": 5090621.0,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"entropy": 1.022918675094843,
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 0.5132948756217957,
|
|
"learning_rate": 0.00019578741614402245,
|
|
"loss": 1.0264,
|
|
"mean_token_accuracy": 0.7782419688999653,
|
|
"num_tokens": 5143415.0,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"eval_entropy": 0.9987384857624535,
|
|
"eval_loss": 1.0029442310333252,
|
|
"eval_mean_token_accuracy": 0.7838117130628721,
|
|
"eval_num_tokens": 5143415.0,
|
|
"eval_runtime": 197.9171,
|
|
"eval_samples_per_second": 20.529,
|
|
"eval_steps_per_second": 2.567,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"entropy": 0.961260611563921,
|
|
"epoch": 0.41865284974093264,
|
|
"grad_norm": 0.5051520466804504,
|
|
"learning_rate": 0.00019565519298006924,
|
|
"loss": 0.9695,
|
|
"mean_token_accuracy": 0.7914115615189076,
|
|
"num_tokens": 5194400.0,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"entropy": 0.9982391525059938,
|
|
"epoch": 0.4227979274611399,
|
|
"grad_norm": 0.4538373351097107,
|
|
"learning_rate": 0.00019552097271999022,
|
|
"loss": 1.0146,
|
|
"mean_token_accuracy": 0.7841442748904228,
|
|
"num_tokens": 5245625.0,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"entropy": 0.9815789975225926,
|
|
"epoch": 0.42694300518134715,
|
|
"grad_norm": 0.4690403342247009,
|
|
"learning_rate": 0.00019538475816604578,
|
|
"loss": 0.9959,
|
|
"mean_token_accuracy": 0.7872560724616051,
|
|
"num_tokens": 5295237.0,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"entropy": 0.9912715546786786,
|
|
"epoch": 0.4310880829015544,
|
|
"grad_norm": 0.42687225341796875,
|
|
"learning_rate": 0.00019524655216213334,
|
|
"loss": 0.9866,
|
|
"mean_token_accuracy": 0.7859402887523175,
|
|
"num_tokens": 5348418.0,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"entropy": 1.0102518431842327,
|
|
"epoch": 0.43523316062176165,
|
|
"grad_norm": 0.44384920597076416,
|
|
"learning_rate": 0.00019510635759372805,
|
|
"loss": 1.0238,
|
|
"mean_token_accuracy": 0.7797439776360988,
|
|
"num_tokens": 5399166.0,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"entropy": 1.0160183981060982,
|
|
"epoch": 0.43937823834196893,
|
|
"grad_norm": 0.44171077013015747,
|
|
"learning_rate": 0.00019496417738782228,
|
|
"loss": 1.0228,
|
|
"mean_token_accuracy": 0.7803056098520755,
|
|
"num_tokens": 5451958.0,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"entropy": 0.9693700931966305,
|
|
"epoch": 0.44352331606217615,
|
|
"grad_norm": 0.4557332694530487,
|
|
"learning_rate": 0.00019482001451286488,
|
|
"loss": 0.9848,
|
|
"mean_token_accuracy": 0.7881055019795895,
|
|
"num_tokens": 5504481.0,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"entropy": 0.9666793867945671,
|
|
"epoch": 0.44766839378238343,
|
|
"grad_norm": 0.45267391204833984,
|
|
"learning_rate": 0.00019467387197869883,
|
|
"loss": 0.9834,
|
|
"mean_token_accuracy": 0.7889953970909118,
|
|
"num_tokens": 5551843.0,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"entropy": 0.9949551515281201,
|
|
"epoch": 0.45181347150259066,
|
|
"grad_norm": 0.438851535320282,
|
|
"learning_rate": 0.00019452575283649873,
|
|
"loss": 1.0144,
|
|
"mean_token_accuracy": 0.782796959578991,
|
|
"num_tokens": 5604244.0,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"entropy": 0.9963484168052673,
|
|
"epoch": 0.45595854922279794,
|
|
"grad_norm": 0.47502249479293823,
|
|
"learning_rate": 0.00019437566017870678,
|
|
"loss": 1.0137,
|
|
"mean_token_accuracy": 0.7818049520254136,
|
|
"num_tokens": 5655400.0,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"entropy": 0.962771862745285,
|
|
"epoch": 0.46010362694300516,
|
|
"grad_norm": 0.42144647240638733,
|
|
"learning_rate": 0.00019422359713896863,
|
|
"loss": 0.9667,
|
|
"mean_token_accuracy": 0.7884697504341602,
|
|
"num_tokens": 5708666.0,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"entropy": 0.9682818073779345,
|
|
"epoch": 0.46424870466321244,
|
|
"grad_norm": 0.3935494124889374,
|
|
"learning_rate": 0.00019406956689206744,
|
|
"loss": 0.9675,
|
|
"mean_token_accuracy": 0.7886212930083275,
|
|
"num_tokens": 5761629.0,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"entropy": 1.037352129071951,
|
|
"epoch": 0.46839378238341967,
|
|
"grad_norm": 0.4130975902080536,
|
|
"learning_rate": 0.00019391357265385807,
|
|
"loss": 1.0403,
|
|
"mean_token_accuracy": 0.7768575675785542,
|
|
"num_tokens": 5812646.0,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"entropy": 0.9832554623484612,
|
|
"epoch": 0.47253886010362695,
|
|
"grad_norm": 0.3829593360424042,
|
|
"learning_rate": 0.00019375561768119958,
|
|
"loss": 0.9976,
|
|
"mean_token_accuracy": 0.7897411204874516,
|
|
"num_tokens": 5862817.0,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"entropy": 0.9487610541284084,
|
|
"epoch": 0.47668393782383417,
|
|
"grad_norm": 0.4701037108898163,
|
|
"learning_rate": 0.00019359570527188754,
|
|
"loss": 0.9638,
|
|
"mean_token_accuracy": 0.792264959961176,
|
|
"num_tokens": 5912716.0,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"entropy": 0.954827756434679,
|
|
"epoch": 0.48082901554404145,
|
|
"grad_norm": 0.4769448935985565,
|
|
"learning_rate": 0.00019343383876458485,
|
|
"loss": 0.9595,
|
|
"mean_token_accuracy": 0.7927994206547737,
|
|
"num_tokens": 5963750.0,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"entropy": 0.9979561999440193,
|
|
"epoch": 0.48497409326424873,
|
|
"grad_norm": 0.46122485399246216,
|
|
"learning_rate": 0.0001932700215387524,
|
|
"loss": 1.0035,
|
|
"mean_token_accuracy": 0.7901160277426242,
|
|
"num_tokens": 6013594.0,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"entropy": 0.9591921351850032,
|
|
"epoch": 0.48911917098445595,
|
|
"grad_norm": 0.4695828855037689,
|
|
"learning_rate": 0.00019310425701457814,
|
|
"loss": 0.9747,
|
|
"mean_token_accuracy": 0.7882664181292057,
|
|
"num_tokens": 6066506.0,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"entropy": 0.9904223918914795,
|
|
"epoch": 0.49326424870466323,
|
|
"grad_norm": 0.43157315254211426,
|
|
"learning_rate": 0.00019293654865290604,
|
|
"loss": 1.007,
|
|
"mean_token_accuracy": 0.7831035263836383,
|
|
"num_tokens": 6120258.0,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"entropy": 0.9716513626277447,
|
|
"epoch": 0.49740932642487046,
|
|
"grad_norm": 0.4715091586112976,
|
|
"learning_rate": 0.00019276689995516346,
|
|
"loss": 0.9743,
|
|
"mean_token_accuracy": 0.7874019861221313,
|
|
"num_tokens": 6171154.0,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"entropy": 0.9984053768217563,
|
|
"epoch": 0.5015544041450777,
|
|
"grad_norm": 0.4164447486400604,
|
|
"learning_rate": 0.0001925953144632884,
|
|
"loss": 1.0145,
|
|
"mean_token_accuracy": 0.781635707616806,
|
|
"num_tokens": 6222515.0,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"entropy": 0.9804062567651272,
|
|
"epoch": 0.505699481865285,
|
|
"grad_norm": 0.45674899220466614,
|
|
"learning_rate": 0.00019242179575965537,
|
|
"loss": 0.9808,
|
|
"mean_token_accuracy": 0.7865826293826104,
|
|
"num_tokens": 6275939.0,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"entropy": 0.9783611670136452,
|
|
"epoch": 0.5098445595854922,
|
|
"grad_norm": 0.4376664161682129,
|
|
"learning_rate": 0.00019224634746700051,
|
|
"loss": 0.992,
|
|
"mean_token_accuracy": 0.7872013546526432,
|
|
"num_tokens": 6325162.0,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"entropy": 0.9830627031624317,
|
|
"epoch": 0.5139896373056995,
|
|
"grad_norm": 0.3772311508655548,
|
|
"learning_rate": 0.00019206897324834614,
|
|
"loss": 0.9948,
|
|
"mean_token_accuracy": 0.7873731434345246,
|
|
"num_tokens": 6376061.0,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"entropy": 0.9824939467012882,
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 0.4799821376800537,
|
|
"learning_rate": 0.0001918896768069242,
|
|
"loss": 0.988,
|
|
"mean_token_accuracy": 0.7887446887791156,
|
|
"num_tokens": 6428028.0,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"eval_entropy": 0.9617302230023962,
|
|
"eval_loss": 0.9882201552391052,
|
|
"eval_mean_token_accuracy": 0.7864582590230806,
|
|
"eval_num_tokens": 6428028.0,
|
|
"eval_runtime": 197.0794,
|
|
"eval_samples_per_second": 20.616,
|
|
"eval_steps_per_second": 2.578,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"entropy": 0.9796643897891044,
|
|
"epoch": 0.522279792746114,
|
|
"grad_norm": 0.45946526527404785,
|
|
"learning_rate": 0.0001917084618860989,
|
|
"loss": 0.9933,
|
|
"mean_token_accuracy": 0.7855292394757271,
|
|
"num_tokens": 6479048.0,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"entropy": 0.9833916153758764,
|
|
"epoch": 0.5264248704663212,
|
|
"grad_norm": 0.3963780999183655,
|
|
"learning_rate": 0.00019152533226928867,
|
|
"loss": 0.9962,
|
|
"mean_token_accuracy": 0.7817724473774433,
|
|
"num_tokens": 6529826.0,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"entropy": 0.9843226552009583,
|
|
"epoch": 0.5305699481865285,
|
|
"grad_norm": 0.4751962125301361,
|
|
"learning_rate": 0.000191340291779887,
|
|
"loss": 0.9994,
|
|
"mean_token_accuracy": 0.7830713532865048,
|
|
"num_tokens": 6580699.0,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"entropy": 0.9555308140814305,
|
|
"epoch": 0.5347150259067357,
|
|
"grad_norm": 0.40511226654052734,
|
|
"learning_rate": 0.00019115334428118286,
|
|
"loss": 0.9761,
|
|
"mean_token_accuracy": 0.7889509312808514,
|
|
"num_tokens": 6633814.0,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"entropy": 0.9578822962939739,
|
|
"epoch": 0.538860103626943,
|
|
"grad_norm": 0.4532525837421417,
|
|
"learning_rate": 0.00019096449367627977,
|
|
"loss": 0.9656,
|
|
"mean_token_accuracy": 0.7934875927865506,
|
|
"num_tokens": 6684053.0,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"entropy": 0.9569033116102219,
|
|
"epoch": 0.5430051813471503,
|
|
"grad_norm": 0.5332207679748535,
|
|
"learning_rate": 0.00019077374390801455,
|
|
"loss": 0.9788,
|
|
"mean_token_accuracy": 0.7917719252407551,
|
|
"num_tokens": 6733968.0,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"entropy": 0.9321629591286182,
|
|
"epoch": 0.5471502590673575,
|
|
"grad_norm": 0.4724487364292145,
|
|
"learning_rate": 0.00019058109895887478,
|
|
"loss": 0.9322,
|
|
"mean_token_accuracy": 0.7971892558038235,
|
|
"num_tokens": 6785674.0,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"entropy": 0.9211927853524685,
|
|
"epoch": 0.5512953367875648,
|
|
"grad_norm": 0.4107973277568817,
|
|
"learning_rate": 0.0001903865628509158,
|
|
"loss": 0.9379,
|
|
"mean_token_accuracy": 0.7973543092608452,
|
|
"num_tokens": 6837186.0,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"entropy": 0.9783548146486283,
|
|
"epoch": 0.555440414507772,
|
|
"grad_norm": 0.4260256588459015,
|
|
"learning_rate": 0.0001901901396456768,
|
|
"loss": 0.986,
|
|
"mean_token_accuracy": 0.7868110507726669,
|
|
"num_tokens": 6889132.0,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"entropy": 0.9350843414664268,
|
|
"epoch": 0.5595854922279793,
|
|
"grad_norm": 0.43470704555511475,
|
|
"learning_rate": 0.00018999183344409578,
|
|
"loss": 0.9496,
|
|
"mean_token_accuracy": 0.796025338023901,
|
|
"num_tokens": 6940037.0,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"entropy": 0.9442426189780235,
|
|
"epoch": 0.5637305699481865,
|
|
"grad_norm": 0.5161582827568054,
|
|
"learning_rate": 0.00018979164838642417,
|
|
"loss": 0.9485,
|
|
"mean_token_accuracy": 0.7975613526999951,
|
|
"num_tokens": 6990041.0,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"entropy": 0.9514821983873845,
|
|
"epoch": 0.5678756476683938,
|
|
"grad_norm": 0.35760676860809326,
|
|
"learning_rate": 0.00018958958865214026,
|
|
"loss": 0.9622,
|
|
"mean_token_accuracy": 0.7867028988897801,
|
|
"num_tokens": 7044442.0,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"entropy": 0.9858744382858277,
|
|
"epoch": 0.572020725388601,
|
|
"grad_norm": 0.4453827738761902,
|
|
"learning_rate": 0.00018938565845986197,
|
|
"loss": 1.0003,
|
|
"mean_token_accuracy": 0.7861887782812118,
|
|
"num_tokens": 7093249.0,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"entropy": 0.9741125054657459,
|
|
"epoch": 0.5761658031088083,
|
|
"grad_norm": 0.433883935213089,
|
|
"learning_rate": 0.00018917986206725887,
|
|
"loss": 0.9804,
|
|
"mean_token_accuracy": 0.7858184657990932,
|
|
"num_tokens": 7144603.0,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"entropy": 0.9409362945705653,
|
|
"epoch": 0.5803108808290155,
|
|
"grad_norm": 0.41870972514152527,
|
|
"learning_rate": 0.000188972203770963,
|
|
"loss": 0.9447,
|
|
"mean_token_accuracy": 0.7908757150173187,
|
|
"num_tokens": 7196254.0,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"entropy": 0.9704828377813101,
|
|
"epoch": 0.5844559585492228,
|
|
"grad_norm": 0.37623921036720276,
|
|
"learning_rate": 0.00018876268790647956,
|
|
"loss": 0.9889,
|
|
"mean_token_accuracy": 0.7871991626918315,
|
|
"num_tokens": 7248841.0,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"entropy": 0.9590335190296173,
|
|
"epoch": 0.5886010362694301,
|
|
"grad_norm": 0.42611730098724365,
|
|
"learning_rate": 0.00018855131884809608,
|
|
"loss": 0.9564,
|
|
"mean_token_accuracy": 0.7912212461233139,
|
|
"num_tokens": 7300448.0,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"entropy": 0.9557400576770305,
|
|
"epoch": 0.5927461139896373,
|
|
"grad_norm": 0.41997987031936646,
|
|
"learning_rate": 0.00018833810100879128,
|
|
"loss": 0.9629,
|
|
"mean_token_accuracy": 0.7843785919249058,
|
|
"num_tokens": 7353316.0,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"entropy": 0.962028032168746,
|
|
"epoch": 0.5968911917098445,
|
|
"grad_norm": 0.41813725233078003,
|
|
"learning_rate": 0.00018812303884014283,
|
|
"loss": 0.975,
|
|
"mean_token_accuracy": 0.792021531611681,
|
|
"num_tokens": 7402745.0,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"entropy": 0.9566993176937103,
|
|
"epoch": 0.6010362694300518,
|
|
"grad_norm": 0.40095973014831543,
|
|
"learning_rate": 0.00018790613683223437,
|
|
"loss": 0.9677,
|
|
"mean_token_accuracy": 0.7910853162407875,
|
|
"num_tokens": 7453651.0,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"entropy": 0.9973684482276439,
|
|
"epoch": 0.6051813471502591,
|
|
"grad_norm": 0.4190211296081543,
|
|
"learning_rate": 0.00018768739951356198,
|
|
"loss": 1.0137,
|
|
"mean_token_accuracy": 0.7809802234172821,
|
|
"num_tokens": 7507052.0,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"entropy": 0.934837969392538,
|
|
"epoch": 0.6093264248704663,
|
|
"grad_norm": 0.4074060916900635,
|
|
"learning_rate": 0.00018746683145093939,
|
|
"loss": 0.9397,
|
|
"mean_token_accuracy": 0.7906834915280342,
|
|
"num_tokens": 7561383.0,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"entropy": 0.950137410312891,
|
|
"epoch": 0.6134715025906736,
|
|
"grad_norm": 0.397320032119751,
|
|
"learning_rate": 0.0001872444372494028,
|
|
"loss": 0.9578,
|
|
"mean_token_accuracy": 0.7921940423548222,
|
|
"num_tokens": 7611992.0,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"entropy": 0.9245664164423942,
|
|
"epoch": 0.6176165803108808,
|
|
"grad_norm": 0.4212367534637451,
|
|
"learning_rate": 0.00018702022155211458,
|
|
"loss": 0.9547,
|
|
"mean_token_accuracy": 0.7904628373682498,
|
|
"num_tokens": 7664532.0,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"entropy": 0.9380590286105871,
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 0.3710251748561859,
|
|
"learning_rate": 0.00018679418904026654,
|
|
"loss": 0.9341,
|
|
"mean_token_accuracy": 0.7926829554140568,
|
|
"num_tokens": 7715917.0,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"eval_entropy": 0.9522759174033413,
|
|
"eval_loss": 0.9706581830978394,
|
|
"eval_mean_token_accuracy": 0.7886052683113128,
|
|
"eval_num_tokens": 7715917.0,
|
|
"eval_runtime": 197.1854,
|
|
"eval_samples_per_second": 20.605,
|
|
"eval_steps_per_second": 2.576,
|
|
"step": 3000
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 14475,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"EarlyStoppingCallback": {
|
|
"args": {
|
|
"early_stopping_patience": 4,
|
|
"early_stopping_threshold": 0.005
|
|
},
|
|
"attributes": {
|
|
"early_stopping_patience_counter": 0
|
|
}
|
|
},
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 5.328561573660672e+17,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|