1035 lines
27 KiB
JSON
1035 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.0292243848266994,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 10.742142391204833,
|
|
"epoch": 0.000292243848266994,
|
|
"grad_norm": 4.625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 10.7907,
|
|
"mean_token_accuracy": 9.328357991762459e-05,
|
|
"num_tokens": 11126.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"entropy": 10.742114543914795,
|
|
"epoch": 0.000584487696533988,
|
|
"grad_norm": 5.21875,
|
|
"learning_rate": 4.5e-06,
|
|
"loss": 10.763,
|
|
"mean_token_accuracy": 0.0,
|
|
"num_tokens": 20949.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 10.742155933380127,
|
|
"epoch": 0.000876731544800982,
|
|
"grad_norm": 5.125,
|
|
"learning_rate": 7e-06,
|
|
"loss": 10.7441,
|
|
"mean_token_accuracy": 0.00010582010727375746,
|
|
"num_tokens": 31135.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"entropy": 10.74212188720703,
|
|
"epoch": 0.001168975393067976,
|
|
"grad_norm": 4.46875,
|
|
"learning_rate": 9.5e-06,
|
|
"loss": 10.6777,
|
|
"mean_token_accuracy": 9.990009712055326e-05,
|
|
"num_tokens": 40153.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 10.742057132720948,
|
|
"epoch": 0.00146121924133497,
|
|
"grad_norm": 4.84375,
|
|
"learning_rate": 1.2e-05,
|
|
"loss": 10.5935,
|
|
"mean_token_accuracy": 0.005301665072329343,
|
|
"num_tokens": 49826.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"entropy": 10.741645336151123,
|
|
"epoch": 0.001753463089601964,
|
|
"grad_norm": 4.0625,
|
|
"learning_rate": 1.4500000000000002e-05,
|
|
"loss": 10.4734,
|
|
"mean_token_accuracy": 0.037928895093500614,
|
|
"num_tokens": 60054.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 10.740150451660156,
|
|
"epoch": 0.002045706937868958,
|
|
"grad_norm": 3.390625,
|
|
"learning_rate": 1.7000000000000003e-05,
|
|
"loss": 10.3332,
|
|
"mean_token_accuracy": 0.045763476938009265,
|
|
"num_tokens": 69706.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"entropy": 10.735286903381347,
|
|
"epoch": 0.002337950786135952,
|
|
"grad_norm": 2.734375,
|
|
"learning_rate": 1.95e-05,
|
|
"loss": 10.2072,
|
|
"mean_token_accuracy": 0.04907714687287808,
|
|
"num_tokens": 80908.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 10.72396411895752,
|
|
"epoch": 0.0026301946344029457,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 2.2e-05,
|
|
"loss": 10.116,
|
|
"mean_token_accuracy": 0.0504226915538311,
|
|
"num_tokens": 91354.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"entropy": 10.707611751556396,
|
|
"epoch": 0.00292243848266994,
|
|
"grad_norm": 2.09375,
|
|
"learning_rate": 2.4500000000000003e-05,
|
|
"loss": 10.0357,
|
|
"mean_token_accuracy": 0.05158382542431354,
|
|
"num_tokens": 101448.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 10.695165920257569,
|
|
"epoch": 0.0032146823309369336,
|
|
"grad_norm": 1.9453125,
|
|
"learning_rate": 2.7e-05,
|
|
"loss": 9.987,
|
|
"mean_token_accuracy": 0.04974161125719547,
|
|
"num_tokens": 111679.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"entropy": 10.690466690063477,
|
|
"epoch": 0.003506926179203928,
|
|
"grad_norm": 1.8125,
|
|
"learning_rate": 2.95e-05,
|
|
"loss": 9.903,
|
|
"mean_token_accuracy": 0.05024520866572857,
|
|
"num_tokens": 122426.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 10.688275337219238,
|
|
"epoch": 0.0037991700274709215,
|
|
"grad_norm": 2.015625,
|
|
"learning_rate": 3.2e-05,
|
|
"loss": 9.8425,
|
|
"mean_token_accuracy": 0.04879201892763376,
|
|
"num_tokens": 132600.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"entropy": 10.682420635223389,
|
|
"epoch": 0.004091413875737916,
|
|
"grad_norm": 1.7265625,
|
|
"learning_rate": 3.4500000000000005e-05,
|
|
"loss": 9.803,
|
|
"mean_token_accuracy": 0.054092536121606825,
|
|
"num_tokens": 143485.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 10.67214002609253,
|
|
"epoch": 0.00438365772400491,
|
|
"grad_norm": 1.7578125,
|
|
"learning_rate": 3.7e-05,
|
|
"loss": 9.7878,
|
|
"mean_token_accuracy": 0.05034251846373081,
|
|
"num_tokens": 154697.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"entropy": 10.66068525314331,
|
|
"epoch": 0.004675901572271904,
|
|
"grad_norm": 1.78125,
|
|
"learning_rate": 3.95e-05,
|
|
"loss": 9.6894,
|
|
"mean_token_accuracy": 0.05171992778778076,
|
|
"num_tokens": 164883.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 10.651456069946288,
|
|
"epoch": 0.004968145420538897,
|
|
"grad_norm": 1.71875,
|
|
"learning_rate": 4.2000000000000004e-05,
|
|
"loss": 9.6326,
|
|
"mean_token_accuracy": 0.05340392328798771,
|
|
"num_tokens": 175534.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"entropy": 10.638569164276124,
|
|
"epoch": 0.0052603892688058915,
|
|
"grad_norm": 1.7421875,
|
|
"learning_rate": 4.45e-05,
|
|
"loss": 9.6175,
|
|
"mean_token_accuracy": 0.05033664517104626,
|
|
"num_tokens": 187495.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 10.627774047851563,
|
|
"epoch": 0.005552633117072886,
|
|
"grad_norm": 1.703125,
|
|
"learning_rate": 4.7000000000000004e-05,
|
|
"loss": 9.5236,
|
|
"mean_token_accuracy": 0.0469218760728836,
|
|
"num_tokens": 197961.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"entropy": 10.619970226287842,
|
|
"epoch": 0.00584487696533988,
|
|
"grad_norm": 1.6796875,
|
|
"learning_rate": 4.9500000000000004e-05,
|
|
"loss": 9.4519,
|
|
"mean_token_accuracy": 0.05195427779108286,
|
|
"num_tokens": 209553.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 10.597136783599854,
|
|
"epoch": 0.006137120813606874,
|
|
"grad_norm": 1.6015625,
|
|
"learning_rate": 5.2e-05,
|
|
"loss": 9.3622,
|
|
"mean_token_accuracy": 0.0559786681085825,
|
|
"num_tokens": 220019.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"entropy": 10.556843280792236,
|
|
"epoch": 0.006429364661873867,
|
|
"grad_norm": 1.546875,
|
|
"learning_rate": 5.45e-05,
|
|
"loss": 9.2362,
|
|
"mean_token_accuracy": 0.06384762041270733,
|
|
"num_tokens": 231542.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 10.514682292938232,
|
|
"epoch": 0.006721608510140861,
|
|
"grad_norm": 1.6328125,
|
|
"learning_rate": 5.7e-05,
|
|
"loss": 9.1411,
|
|
"mean_token_accuracy": 0.060038824751973155,
|
|
"num_tokens": 242192.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"entropy": 10.46776523590088,
|
|
"epoch": 0.007013852358407856,
|
|
"grad_norm": 1.6015625,
|
|
"learning_rate": 5.9499999999999996e-05,
|
|
"loss": 9.0408,
|
|
"mean_token_accuracy": 0.06044495329260826,
|
|
"num_tokens": 252711.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 10.397676372528077,
|
|
"epoch": 0.00730609620667485,
|
|
"grad_norm": 1.4921875,
|
|
"learning_rate": 6.2e-05,
|
|
"loss": 8.958,
|
|
"mean_token_accuracy": 0.05820495039224625,
|
|
"num_tokens": 262768.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"entropy": 10.315190315246582,
|
|
"epoch": 0.007598340054941843,
|
|
"grad_norm": 1.421875,
|
|
"learning_rate": 6.450000000000001e-05,
|
|
"loss": 8.8458,
|
|
"mean_token_accuracy": 0.06275271475315095,
|
|
"num_tokens": 274374.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 10.261965942382812,
|
|
"epoch": 0.007890583903208837,
|
|
"grad_norm": 1.3203125,
|
|
"learning_rate": 6.7e-05,
|
|
"loss": 8.7538,
|
|
"mean_token_accuracy": 0.06524680852890015,
|
|
"num_tokens": 285449.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"entropy": 10.157284355163574,
|
|
"epoch": 0.008182827751475831,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 6.950000000000001e-05,
|
|
"loss": 8.6492,
|
|
"mean_token_accuracy": 0.06321290582418441,
|
|
"num_tokens": 296785.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 10.050414943695069,
|
|
"epoch": 0.008475071599742826,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 7.2e-05,
|
|
"loss": 8.5334,
|
|
"mean_token_accuracy": 0.06643750704824924,
|
|
"num_tokens": 307781.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"entropy": 9.926141262054443,
|
|
"epoch": 0.00876731544800982,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 7.45e-05,
|
|
"loss": 8.3954,
|
|
"mean_token_accuracy": 0.06747495792806149,
|
|
"num_tokens": 318713.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 9.822235202789306,
|
|
"epoch": 0.009059559296276814,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 7.7e-05,
|
|
"loss": 8.3951,
|
|
"mean_token_accuracy": 0.05827293880283833,
|
|
"num_tokens": 329779.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"entropy": 9.66906967163086,
|
|
"epoch": 0.009351803144543808,
|
|
"grad_norm": 0.953125,
|
|
"learning_rate": 7.950000000000001e-05,
|
|
"loss": 8.2444,
|
|
"mean_token_accuracy": 0.0628633838146925,
|
|
"num_tokens": 339733.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 9.44699068069458,
|
|
"epoch": 0.009644046992810802,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 8.2e-05,
|
|
"loss": 8.1742,
|
|
"mean_token_accuracy": 0.06834175810217857,
|
|
"num_tokens": 350232.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"entropy": 9.28164119720459,
|
|
"epoch": 0.009936290841077795,
|
|
"grad_norm": 0.859375,
|
|
"learning_rate": 8.450000000000001e-05,
|
|
"loss": 8.1224,
|
|
"mean_token_accuracy": 0.06834722384810447,
|
|
"num_tokens": 361051.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 9.148957920074462,
|
|
"epoch": 0.010228534689344789,
|
|
"grad_norm": 0.8515625,
|
|
"learning_rate": 8.7e-05,
|
|
"loss": 8.0815,
|
|
"mean_token_accuracy": 0.0646477747708559,
|
|
"num_tokens": 371699.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"entropy": 8.94444236755371,
|
|
"epoch": 0.010520778537611783,
|
|
"grad_norm": 0.78125,
|
|
"learning_rate": 8.95e-05,
|
|
"loss": 7.9754,
|
|
"mean_token_accuracy": 0.06804421916604042,
|
|
"num_tokens": 381817.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 8.812028980255127,
|
|
"epoch": 0.010813022385878777,
|
|
"grad_norm": 0.7890625,
|
|
"learning_rate": 9.2e-05,
|
|
"loss": 7.9774,
|
|
"mean_token_accuracy": 0.06625252738595008,
|
|
"num_tokens": 393788.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"entropy": 8.69484453201294,
|
|
"epoch": 0.011105266234145771,
|
|
"grad_norm": 0.75,
|
|
"learning_rate": 9.45e-05,
|
|
"loss": 7.8944,
|
|
"mean_token_accuracy": 0.07389901392161846,
|
|
"num_tokens": 404690.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 8.55163288116455,
|
|
"epoch": 0.011397510082412765,
|
|
"grad_norm": 0.82421875,
|
|
"learning_rate": 9.7e-05,
|
|
"loss": 7.855,
|
|
"mean_token_accuracy": 0.06855227462947369,
|
|
"num_tokens": 414627.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"entropy": 8.51447925567627,
|
|
"epoch": 0.01168975393067976,
|
|
"grad_norm": 0.78515625,
|
|
"learning_rate": 9.95e-05,
|
|
"loss": 7.9112,
|
|
"mean_token_accuracy": 0.06506339274346828,
|
|
"num_tokens": 425523.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 8.489240837097167,
|
|
"epoch": 0.011981997778946754,
|
|
"grad_norm": 0.84375,
|
|
"learning_rate": 0.000102,
|
|
"loss": 7.8654,
|
|
"mean_token_accuracy": 0.06927913017570972,
|
|
"num_tokens": 436340.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"entropy": 8.421260166168214,
|
|
"epoch": 0.012274241627213748,
|
|
"grad_norm": 0.6953125,
|
|
"learning_rate": 0.00010449999999999999,
|
|
"loss": 7.875,
|
|
"mean_token_accuracy": 0.07178668864071369,
|
|
"num_tokens": 447677.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 8.410223293304444,
|
|
"epoch": 0.01256648547548074,
|
|
"grad_norm": 0.8984375,
|
|
"learning_rate": 0.000107,
|
|
"loss": 7.852,
|
|
"mean_token_accuracy": 0.06873307004570961,
|
|
"num_tokens": 457470.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"entropy": 8.345960903167725,
|
|
"epoch": 0.012858729323747735,
|
|
"grad_norm": 0.78125,
|
|
"learning_rate": 0.0001095,
|
|
"loss": 7.841,
|
|
"mean_token_accuracy": 0.06865699663758278,
|
|
"num_tokens": 467056.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 8.390186595916749,
|
|
"epoch": 0.013150973172014729,
|
|
"grad_norm": 0.73828125,
|
|
"learning_rate": 0.000112,
|
|
"loss": 7.8266,
|
|
"mean_token_accuracy": 0.07396755889058113,
|
|
"num_tokens": 476964.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"entropy": 8.322624683380127,
|
|
"epoch": 0.013443217020281723,
|
|
"grad_norm": 0.796875,
|
|
"learning_rate": 0.0001145,
|
|
"loss": 7.7907,
|
|
"mean_token_accuracy": 0.07042324244976043,
|
|
"num_tokens": 488273.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 8.322233295440673,
|
|
"epoch": 0.013735460868548717,
|
|
"grad_norm": 0.8125,
|
|
"learning_rate": 0.00011700000000000001,
|
|
"loss": 7.8396,
|
|
"mean_token_accuracy": 0.06669306643307209,
|
|
"num_tokens": 499188.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"entropy": 8.32526388168335,
|
|
"epoch": 0.014027704716815711,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 0.00011949999999999999,
|
|
"loss": 7.7654,
|
|
"mean_token_accuracy": 0.07240047603845597,
|
|
"num_tokens": 509750.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 8.2788516998291,
|
|
"epoch": 0.014319948565082705,
|
|
"grad_norm": 1.2890625,
|
|
"learning_rate": 0.000122,
|
|
"loss": 7.7089,
|
|
"mean_token_accuracy": 0.07294566892087459,
|
|
"num_tokens": 519680.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"entropy": 8.086223363876343,
|
|
"epoch": 0.0146121924133497,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.0001245,
|
|
"loss": 7.8038,
|
|
"mean_token_accuracy": 0.07106421366333962,
|
|
"num_tokens": 531028.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 8.35772066116333,
|
|
"epoch": 0.014904436261616694,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 0.000127,
|
|
"loss": 7.7634,
|
|
"mean_token_accuracy": 0.07067177146673202,
|
|
"num_tokens": 540911.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"entropy": 8.142296409606933,
|
|
"epoch": 0.015196680109883686,
|
|
"grad_norm": 0.828125,
|
|
"learning_rate": 0.0001295,
|
|
"loss": 7.7239,
|
|
"mean_token_accuracy": 0.07250927239656449,
|
|
"num_tokens": 551217.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 8.26462230682373,
|
|
"epoch": 0.01548892395815068,
|
|
"grad_norm": 0.90234375,
|
|
"learning_rate": 0.000132,
|
|
"loss": 7.7371,
|
|
"mean_token_accuracy": 0.06948361918330193,
|
|
"num_tokens": 560741.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"entropy": 8.203502750396728,
|
|
"epoch": 0.015781167806417674,
|
|
"grad_norm": 0.85546875,
|
|
"learning_rate": 0.00013450000000000002,
|
|
"loss": 7.7069,
|
|
"mean_token_accuracy": 0.07292059324681759,
|
|
"num_tokens": 571004.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 8.08554220199585,
|
|
"epoch": 0.01607341165468467,
|
|
"grad_norm": 0.796875,
|
|
"learning_rate": 0.00013700000000000002,
|
|
"loss": 7.6602,
|
|
"mean_token_accuracy": 0.07287065088748931,
|
|
"num_tokens": 581035.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"entropy": 8.230623817443847,
|
|
"epoch": 0.016365655502951663,
|
|
"grad_norm": 0.921875,
|
|
"learning_rate": 0.0001395,
|
|
"loss": 7.7703,
|
|
"mean_token_accuracy": 0.06783514469861984,
|
|
"num_tokens": 591460.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 8.123512077331544,
|
|
"epoch": 0.016657899351218655,
|
|
"grad_norm": 0.91015625,
|
|
"learning_rate": 0.00014199999999999998,
|
|
"loss": 7.7867,
|
|
"mean_token_accuracy": 0.06507540903985501,
|
|
"num_tokens": 602104.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"entropy": 8.162607574462891,
|
|
"epoch": 0.01695014319948565,
|
|
"grad_norm": 0.80078125,
|
|
"learning_rate": 0.0001445,
|
|
"loss": 7.7244,
|
|
"mean_token_accuracy": 0.06926270946860313,
|
|
"num_tokens": 613497.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 8.155611419677735,
|
|
"epoch": 0.017242387047752643,
|
|
"grad_norm": 0.80078125,
|
|
"learning_rate": 0.000147,
|
|
"loss": 7.7376,
|
|
"mean_token_accuracy": 0.06834047213196755,
|
|
"num_tokens": 623432.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"entropy": 8.172423458099365,
|
|
"epoch": 0.01753463089601964,
|
|
"grad_norm": 0.91015625,
|
|
"learning_rate": 0.0001495,
|
|
"loss": 7.72,
|
|
"mean_token_accuracy": 0.07163975387811661,
|
|
"num_tokens": 633890.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 8.120147132873536,
|
|
"epoch": 0.017826874744286632,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.000152,
|
|
"loss": 7.6996,
|
|
"mean_token_accuracy": 0.07336684912443162,
|
|
"num_tokens": 644020.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"entropy": 8.08779067993164,
|
|
"epoch": 0.018119118592553628,
|
|
"grad_norm": 0.8359375,
|
|
"learning_rate": 0.00015450000000000001,
|
|
"loss": 7.644,
|
|
"mean_token_accuracy": 0.07471407577395439,
|
|
"num_tokens": 654177.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 8.040860605239867,
|
|
"epoch": 0.01841136244082062,
|
|
"grad_norm": 0.75,
|
|
"learning_rate": 0.000157,
|
|
"loss": 7.5986,
|
|
"mean_token_accuracy": 0.08076093941926957,
|
|
"num_tokens": 665317.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"entropy": 8.058250904083252,
|
|
"epoch": 0.018703606289087616,
|
|
"grad_norm": 0.87109375,
|
|
"learning_rate": 0.0001595,
|
|
"loss": 7.7048,
|
|
"mean_token_accuracy": 0.07110530957579612,
|
|
"num_tokens": 676248.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 8.157335948944091,
|
|
"epoch": 0.01899585013735461,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.000162,
|
|
"loss": 7.7111,
|
|
"mean_token_accuracy": 0.06896476708352565,
|
|
"num_tokens": 687573.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"entropy": 8.00673680305481,
|
|
"epoch": 0.019288093985621604,
|
|
"grad_norm": 0.8125,
|
|
"learning_rate": 0.00016450000000000001,
|
|
"loss": 7.7276,
|
|
"mean_token_accuracy": 0.07006649971008301,
|
|
"num_tokens": 698919.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 8.120959758758545,
|
|
"epoch": 0.019580337833888597,
|
|
"grad_norm": 0.79296875,
|
|
"learning_rate": 0.00016700000000000002,
|
|
"loss": 7.6748,
|
|
"mean_token_accuracy": 0.06717267856001854,
|
|
"num_tokens": 709450.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"entropy": 8.110147094726562,
|
|
"epoch": 0.01987258168215559,
|
|
"grad_norm": 0.921875,
|
|
"learning_rate": 0.00016950000000000003,
|
|
"loss": 7.6476,
|
|
"mean_token_accuracy": 0.07495066076517105,
|
|
"num_tokens": 719909.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 8.031400108337403,
|
|
"epoch": 0.020164825530422585,
|
|
"grad_norm": 0.85546875,
|
|
"learning_rate": 0.00017199999999999998,
|
|
"loss": 7.5699,
|
|
"mean_token_accuracy": 0.07540844045579434,
|
|
"num_tokens": 729451.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"entropy": 8.043409729003907,
|
|
"epoch": 0.020457069378689578,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.00017449999999999999,
|
|
"loss": 7.5831,
|
|
"mean_token_accuracy": 0.07626536451280116,
|
|
"num_tokens": 739256.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 7.9351013660430905,
|
|
"epoch": 0.020749313226956573,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 0.000177,
|
|
"loss": 7.5958,
|
|
"mean_token_accuracy": 0.07205557823181152,
|
|
"num_tokens": 749690.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"entropy": 7.968925666809082,
|
|
"epoch": 0.021041557075223566,
|
|
"grad_norm": 0.8203125,
|
|
"learning_rate": 0.0001795,
|
|
"loss": 7.6478,
|
|
"mean_token_accuracy": 0.0736626997590065,
|
|
"num_tokens": 760665.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 8.029117727279663,
|
|
"epoch": 0.021333800923490562,
|
|
"grad_norm": 0.90234375,
|
|
"learning_rate": 0.000182,
|
|
"loss": 7.5629,
|
|
"mean_token_accuracy": 0.07812464088201523,
|
|
"num_tokens": 770506.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"entropy": 7.863681507110596,
|
|
"epoch": 0.021626044771757554,
|
|
"grad_norm": 0.9140625,
|
|
"learning_rate": 0.0001845,
|
|
"loss": 7.6357,
|
|
"mean_token_accuracy": 0.07435660734772682,
|
|
"num_tokens": 780939.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 8.057669687271119,
|
|
"epoch": 0.02191828862002455,
|
|
"grad_norm": 0.74609375,
|
|
"learning_rate": 0.000187,
|
|
"loss": 7.6061,
|
|
"mean_token_accuracy": 0.0721366185694933,
|
|
"num_tokens": 792336.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"entropy": 8.00227198600769,
|
|
"epoch": 0.022210532468291543,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.0001895,
|
|
"loss": 7.6259,
|
|
"mean_token_accuracy": 0.07205860875546932,
|
|
"num_tokens": 803357.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 8.049015951156616,
|
|
"epoch": 0.022502776316558535,
|
|
"grad_norm": 0.8046875,
|
|
"learning_rate": 0.000192,
|
|
"loss": 7.5829,
|
|
"mean_token_accuracy": 0.07036133743822574,
|
|
"num_tokens": 813964.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"entropy": 7.892692565917969,
|
|
"epoch": 0.02279502016482553,
|
|
"grad_norm": 0.859375,
|
|
"learning_rate": 0.0001945,
|
|
"loss": 7.5666,
|
|
"mean_token_accuracy": 0.07797688394784927,
|
|
"num_tokens": 824042.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 8.043365049362183,
|
|
"epoch": 0.023087264013092523,
|
|
"grad_norm": 0.86328125,
|
|
"learning_rate": 0.00019700000000000002,
|
|
"loss": 7.5108,
|
|
"mean_token_accuracy": 0.08256808444857597,
|
|
"num_tokens": 834080.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"entropy": 8.010589456558227,
|
|
"epoch": 0.02337950786135952,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.00019950000000000002,
|
|
"loss": 7.617,
|
|
"mean_token_accuracy": 0.06876562051475048,
|
|
"num_tokens": 844483.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 7.925695133209229,
|
|
"epoch": 0.02367175170962651,
|
|
"grad_norm": 0.83203125,
|
|
"learning_rate": 0.000202,
|
|
"loss": 7.6103,
|
|
"mean_token_accuracy": 0.07202962972223759,
|
|
"num_tokens": 855896.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"entropy": 7.904125165939331,
|
|
"epoch": 0.023963995557893508,
|
|
"grad_norm": 0.890625,
|
|
"learning_rate": 0.00020449999999999998,
|
|
"loss": 7.5299,
|
|
"mean_token_accuracy": 0.07748726978898049,
|
|
"num_tokens": 866631.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 7.941328859329223,
|
|
"epoch": 0.0242562394061605,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.000207,
|
|
"loss": 7.5184,
|
|
"mean_token_accuracy": 0.0782412201166153,
|
|
"num_tokens": 877779.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"entropy": 7.92393913269043,
|
|
"epoch": 0.024548483254427496,
|
|
"grad_norm": 0.8671875,
|
|
"learning_rate": 0.0002095,
|
|
"loss": 7.5434,
|
|
"mean_token_accuracy": 0.07804081477224827,
|
|
"num_tokens": 888478.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 7.941523599624634,
|
|
"epoch": 0.02484072710269449,
|
|
"grad_norm": 0.98828125,
|
|
"learning_rate": 0.000212,
|
|
"loss": 7.5063,
|
|
"mean_token_accuracy": 0.07843287661671638,
|
|
"num_tokens": 898164.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"entropy": 7.827020597457886,
|
|
"epoch": 0.02513297095096148,
|
|
"grad_norm": 0.90234375,
|
|
"learning_rate": 0.0002145,
|
|
"loss": 7.444,
|
|
"mean_token_accuracy": 0.08076250851154328,
|
|
"num_tokens": 908426.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 7.868816757202149,
|
|
"epoch": 0.025425214799228477,
|
|
"grad_norm": 0.7578125,
|
|
"learning_rate": 0.00021700000000000002,
|
|
"loss": 7.5119,
|
|
"mean_token_accuracy": 0.07991167530417442,
|
|
"num_tokens": 919164.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"entropy": 7.792380475997925,
|
|
"epoch": 0.02571745864749547,
|
|
"grad_norm": 0.859375,
|
|
"learning_rate": 0.0002195,
|
|
"loss": 7.4187,
|
|
"mean_token_accuracy": 0.08300766497850418,
|
|
"num_tokens": 931152.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 7.8588916778564455,
|
|
"epoch": 0.026009702495762465,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.000222,
|
|
"loss": 7.5027,
|
|
"mean_token_accuracy": 0.08102720528841019,
|
|
"num_tokens": 941629.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"entropy": 7.851666688919067,
|
|
"epoch": 0.026301946344029457,
|
|
"grad_norm": 0.88671875,
|
|
"learning_rate": 0.0002245,
|
|
"loss": 7.4577,
|
|
"mean_token_accuracy": 0.08078472912311555,
|
|
"num_tokens": 952665.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 7.90864748954773,
|
|
"epoch": 0.026594190192296453,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.00022700000000000002,
|
|
"loss": 7.4166,
|
|
"mean_token_accuracy": 0.07709830179810524,
|
|
"num_tokens": 963395.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"entropy": 7.783758735656738,
|
|
"epoch": 0.026886434040563446,
|
|
"grad_norm": 0.95703125,
|
|
"learning_rate": 0.00022950000000000002,
|
|
"loss": 7.402,
|
|
"mean_token_accuracy": 0.08202188611030578,
|
|
"num_tokens": 972789.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 7.851816082000733,
|
|
"epoch": 0.02717867788883044,
|
|
"grad_norm": 1.375,
|
|
"learning_rate": 0.00023200000000000003,
|
|
"loss": 7.4988,
|
|
"mean_token_accuracy": 0.07351949512958526,
|
|
"num_tokens": 984240.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"entropy": 7.863189268112182,
|
|
"epoch": 0.027470921737097434,
|
|
"grad_norm": 0.98046875,
|
|
"learning_rate": 0.00023449999999999998,
|
|
"loss": 7.4893,
|
|
"mean_token_accuracy": 0.07835234403610229,
|
|
"num_tokens": 994698.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 7.70669264793396,
|
|
"epoch": 0.027763165585364426,
|
|
"grad_norm": 0.8984375,
|
|
"learning_rate": 0.000237,
|
|
"loss": 7.4247,
|
|
"mean_token_accuracy": 0.08151589259505272,
|
|
"num_tokens": 1005324.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"entropy": 7.905373239517212,
|
|
"epoch": 0.028055409433631422,
|
|
"grad_norm": 0.828125,
|
|
"learning_rate": 0.0002395,
|
|
"loss": 7.4238,
|
|
"mean_token_accuracy": 0.07885461077094078,
|
|
"num_tokens": 1016587.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 7.845970726013183,
|
|
"epoch": 0.028347653281898415,
|
|
"grad_norm": 0.953125,
|
|
"learning_rate": 0.000242,
|
|
"loss": 7.4595,
|
|
"mean_token_accuracy": 0.07829558476805687,
|
|
"num_tokens": 1026998.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"entropy": 7.805553197860718,
|
|
"epoch": 0.02863989713016541,
|
|
"grad_norm": 0.98828125,
|
|
"learning_rate": 0.0002445,
|
|
"loss": 7.4385,
|
|
"mean_token_accuracy": 0.0780658096075058,
|
|
"num_tokens": 1038475.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 7.88462347984314,
|
|
"epoch": 0.028932140978432403,
|
|
"grad_norm": 0.9765625,
|
|
"learning_rate": 0.000247,
|
|
"loss": 7.3556,
|
|
"mean_token_accuracy": 0.08420942425727844,
|
|
"num_tokens": 1049631.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"entropy": 7.75970048904419,
|
|
"epoch": 0.0292243848266994,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0002495,
|
|
"loss": 7.4304,
|
|
"mean_token_accuracy": 0.07490956112742424,
|
|
"num_tokens": 1060052.0,
|
|
"step": 500
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 4000,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 227917410140160.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|