3068 lines
82 KiB
JSON
3068 lines
82 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.1755485893416928,
|
|
"eval_steps": 500,
|
|
"global_step": 1500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 10.742109775543213,
|
|
"epoch": 0.003918495297805642,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 10.7674,
|
|
"mean_token_accuracy": 9.596928721293807e-05,
|
|
"num_tokens": 9415.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"entropy": 10.742099189758301,
|
|
"epoch": 0.007836990595611285,
|
|
"grad_norm": 5.65625,
|
|
"learning_rate": 4.5e-06,
|
|
"loss": 10.7184,
|
|
"mean_token_accuracy": 0.0,
|
|
"num_tokens": 17669.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 10.742108917236328,
|
|
"epoch": 0.011755485893416929,
|
|
"grad_norm": 5.0625,
|
|
"learning_rate": 7e-06,
|
|
"loss": 10.7241,
|
|
"mean_token_accuracy": 0.00021240997593849898,
|
|
"num_tokens": 26558.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"entropy": 10.742095565795898,
|
|
"epoch": 0.01567398119122257,
|
|
"grad_norm": 5.40625,
|
|
"learning_rate": 9.5e-06,
|
|
"loss": 10.6562,
|
|
"mean_token_accuracy": 0.0,
|
|
"num_tokens": 34940.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 10.742070865631103,
|
|
"epoch": 0.019592476489028215,
|
|
"grad_norm": 4.40625,
|
|
"learning_rate": 1.2e-05,
|
|
"loss": 10.5966,
|
|
"mean_token_accuracy": 0.0032189773628488183,
|
|
"num_tokens": 44835.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"entropy": 10.741683006286621,
|
|
"epoch": 0.023510971786833857,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 1.4500000000000002e-05,
|
|
"loss": 10.4748,
|
|
"mean_token_accuracy": 0.02363332137465477,
|
|
"num_tokens": 53623.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 10.740711116790772,
|
|
"epoch": 0.0274294670846395,
|
|
"grad_norm": 3.5,
|
|
"learning_rate": 1.7000000000000003e-05,
|
|
"loss": 10.3625,
|
|
"mean_token_accuracy": 0.03914350755512715,
|
|
"num_tokens": 63198.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"entropy": 10.738054847717285,
|
|
"epoch": 0.03134796238244514,
|
|
"grad_norm": 2.828125,
|
|
"learning_rate": 1.95e-05,
|
|
"loss": 10.2445,
|
|
"mean_token_accuracy": 0.043806169740855695,
|
|
"num_tokens": 72379.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 10.73265199661255,
|
|
"epoch": 0.03526645768025078,
|
|
"grad_norm": 2.453125,
|
|
"learning_rate": 2.2e-05,
|
|
"loss": 10.1124,
|
|
"mean_token_accuracy": 0.04499626830220223,
|
|
"num_tokens": 81098.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"entropy": 10.725791835784912,
|
|
"epoch": 0.03918495297805643,
|
|
"grad_norm": 2.109375,
|
|
"learning_rate": 2.4500000000000003e-05,
|
|
"loss": 10.0549,
|
|
"mean_token_accuracy": 0.04616287313401699,
|
|
"num_tokens": 90637.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 10.720808124542236,
|
|
"epoch": 0.04310344827586207,
|
|
"grad_norm": 2.03125,
|
|
"learning_rate": 2.7e-05,
|
|
"loss": 9.9466,
|
|
"mean_token_accuracy": 0.04513598307967186,
|
|
"num_tokens": 98505.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"entropy": 10.717309474945068,
|
|
"epoch": 0.047021943573667714,
|
|
"grad_norm": 1.90625,
|
|
"learning_rate": 2.95e-05,
|
|
"loss": 9.8942,
|
|
"mean_token_accuracy": 0.048630540072917935,
|
|
"num_tokens": 107295.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 10.71382761001587,
|
|
"epoch": 0.050940438871473356,
|
|
"grad_norm": 2.0,
|
|
"learning_rate": 3.2e-05,
|
|
"loss": 9.8181,
|
|
"mean_token_accuracy": 0.04877537116408348,
|
|
"num_tokens": 116381.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"entropy": 10.70920639038086,
|
|
"epoch": 0.054858934169279,
|
|
"grad_norm": 1.984375,
|
|
"learning_rate": 3.4500000000000005e-05,
|
|
"loss": 9.7896,
|
|
"mean_token_accuracy": 0.04598020613193512,
|
|
"num_tokens": 124826.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 10.703414630889892,
|
|
"epoch": 0.05877742946708464,
|
|
"grad_norm": 1.9140625,
|
|
"learning_rate": 3.7e-05,
|
|
"loss": 9.7231,
|
|
"mean_token_accuracy": 0.04887317828834057,
|
|
"num_tokens": 133255.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"entropy": 10.69604263305664,
|
|
"epoch": 0.06269592476489028,
|
|
"grad_norm": 1.9609375,
|
|
"learning_rate": 3.95e-05,
|
|
"loss": 9.6411,
|
|
"mean_token_accuracy": 0.05134939663112163,
|
|
"num_tokens": 141626.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 10.6855975151062,
|
|
"epoch": 0.06661442006269593,
|
|
"grad_norm": 1.9375,
|
|
"learning_rate": 4.2000000000000004e-05,
|
|
"loss": 9.5459,
|
|
"mean_token_accuracy": 0.06545204371213913,
|
|
"num_tokens": 150051.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"entropy": 10.666928005218505,
|
|
"epoch": 0.07053291536050156,
|
|
"grad_norm": 1.953125,
|
|
"learning_rate": 4.45e-05,
|
|
"loss": 9.5417,
|
|
"mean_token_accuracy": 0.061743459478020665,
|
|
"num_tokens": 159325.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 10.645073413848877,
|
|
"epoch": 0.07445141065830721,
|
|
"grad_norm": 1.859375,
|
|
"learning_rate": 4.7000000000000004e-05,
|
|
"loss": 9.4416,
|
|
"mean_token_accuracy": 0.06291964910924434,
|
|
"num_tokens": 168142.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"entropy": 10.608468723297118,
|
|
"epoch": 0.07836990595611286,
|
|
"grad_norm": 1.765625,
|
|
"learning_rate": 4.9500000000000004e-05,
|
|
"loss": 9.3001,
|
|
"mean_token_accuracy": 0.06304978355765342,
|
|
"num_tokens": 177033.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 10.553625392913819,
|
|
"epoch": 0.0822884012539185,
|
|
"grad_norm": 1.8046875,
|
|
"learning_rate": 5.2e-05,
|
|
"loss": 9.229,
|
|
"mean_token_accuracy": 0.056359085440635684,
|
|
"num_tokens": 185605.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"entropy": 10.505783081054688,
|
|
"epoch": 0.08620689655172414,
|
|
"grad_norm": 1.703125,
|
|
"learning_rate": 5.45e-05,
|
|
"loss": 9.0952,
|
|
"mean_token_accuracy": 0.0655987836420536,
|
|
"num_tokens": 193863.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 10.448099517822266,
|
|
"epoch": 0.09012539184952978,
|
|
"grad_norm": 1.671875,
|
|
"learning_rate": 5.7e-05,
|
|
"loss": 8.9999,
|
|
"mean_token_accuracy": 0.0653056938201189,
|
|
"num_tokens": 202504.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"entropy": 10.432163047790528,
|
|
"epoch": 0.09404388714733543,
|
|
"grad_norm": 1.5390625,
|
|
"learning_rate": 5.9499999999999996e-05,
|
|
"loss": 8.9214,
|
|
"mean_token_accuracy": 0.06190854795277119,
|
|
"num_tokens": 211172.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 10.393641376495362,
|
|
"epoch": 0.09796238244514106,
|
|
"grad_norm": 1.46875,
|
|
"learning_rate": 6.2e-05,
|
|
"loss": 8.8333,
|
|
"mean_token_accuracy": 0.060395382717251776,
|
|
"num_tokens": 220345.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"entropy": 10.306912040710449,
|
|
"epoch": 0.10188087774294671,
|
|
"grad_norm": 1.421875,
|
|
"learning_rate": 6.450000000000001e-05,
|
|
"loss": 8.6192,
|
|
"mean_token_accuracy": 0.0663460198789835,
|
|
"num_tokens": 228396.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 10.185036849975585,
|
|
"epoch": 0.10579937304075235,
|
|
"grad_norm": 1.390625,
|
|
"learning_rate": 6.7e-05,
|
|
"loss": 8.5194,
|
|
"mean_token_accuracy": 0.06784758679568767,
|
|
"num_tokens": 236773.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"entropy": 10.10704460144043,
|
|
"epoch": 0.109717868338558,
|
|
"grad_norm": 1.4140625,
|
|
"learning_rate": 6.950000000000001e-05,
|
|
"loss": 8.4114,
|
|
"mean_token_accuracy": 0.06528282575309277,
|
|
"num_tokens": 245279.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 9.997784900665284,
|
|
"epoch": 0.11363636363636363,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 7.2e-05,
|
|
"loss": 8.2873,
|
|
"mean_token_accuracy": 0.06404702439904213,
|
|
"num_tokens": 253965.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"entropy": 9.862957191467284,
|
|
"epoch": 0.11755485893416928,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 7.45e-05,
|
|
"loss": 8.2398,
|
|
"mean_token_accuracy": 0.061683330684900284,
|
|
"num_tokens": 262859.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 9.73002061843872,
|
|
"epoch": 0.12147335423197492,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 7.7e-05,
|
|
"loss": 8.1396,
|
|
"mean_token_accuracy": 0.06386828869581222,
|
|
"num_tokens": 270757.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"entropy": 9.495413970947265,
|
|
"epoch": 0.12539184952978055,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 7.950000000000001e-05,
|
|
"loss": 8.0706,
|
|
"mean_token_accuracy": 0.060359453037381175,
|
|
"num_tokens": 279737.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 9.4040997505188,
|
|
"epoch": 0.12931034482758622,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 8.2e-05,
|
|
"loss": 7.895,
|
|
"mean_token_accuracy": 0.061074954271316526,
|
|
"num_tokens": 287788.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"entropy": 9.116162109375,
|
|
"epoch": 0.13322884012539185,
|
|
"grad_norm": 0.90625,
|
|
"learning_rate": 8.450000000000001e-05,
|
|
"loss": 7.8633,
|
|
"mean_token_accuracy": 0.06705749183893203,
|
|
"num_tokens": 296481.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 8.948561000823975,
|
|
"epoch": 0.1371473354231975,
|
|
"grad_norm": 0.8671875,
|
|
"learning_rate": 8.7e-05,
|
|
"loss": 7.76,
|
|
"mean_token_accuracy": 0.06502468548715115,
|
|
"num_tokens": 305412.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"entropy": 8.750289058685302,
|
|
"epoch": 0.14106583072100312,
|
|
"grad_norm": 0.8828125,
|
|
"learning_rate": 8.95e-05,
|
|
"loss": 7.7497,
|
|
"mean_token_accuracy": 0.06412209160625934,
|
|
"num_tokens": 314412.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 8.61333589553833,
|
|
"epoch": 0.14498432601880878,
|
|
"grad_norm": 0.80859375,
|
|
"learning_rate": 9.2e-05,
|
|
"loss": 7.6727,
|
|
"mean_token_accuracy": 0.06552823781967163,
|
|
"num_tokens": 322928.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"entropy": 8.46501522064209,
|
|
"epoch": 0.14890282131661442,
|
|
"grad_norm": 0.875,
|
|
"learning_rate": 9.45e-05,
|
|
"loss": 7.693,
|
|
"mean_token_accuracy": 0.0656155750155449,
|
|
"num_tokens": 331605.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 8.362890625,
|
|
"epoch": 0.15282131661442006,
|
|
"grad_norm": 0.86328125,
|
|
"learning_rate": 9.7e-05,
|
|
"loss": 7.6608,
|
|
"mean_token_accuracy": 0.06488309130072593,
|
|
"num_tokens": 340654.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"entropy": 8.327593994140624,
|
|
"epoch": 0.15673981191222572,
|
|
"grad_norm": 0.88671875,
|
|
"learning_rate": 9.95e-05,
|
|
"loss": 7.6721,
|
|
"mean_token_accuracy": 0.06592198349535465,
|
|
"num_tokens": 348905.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 8.273851680755616,
|
|
"epoch": 0.16065830721003135,
|
|
"grad_norm": 0.921875,
|
|
"learning_rate": 0.000102,
|
|
"loss": 7.5357,
|
|
"mean_token_accuracy": 0.06864950023591518,
|
|
"num_tokens": 357276.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"entropy": 8.201261711120605,
|
|
"epoch": 0.164576802507837,
|
|
"grad_norm": 0.79296875,
|
|
"learning_rate": 0.00010449999999999999,
|
|
"loss": 7.578,
|
|
"mean_token_accuracy": 0.06853164285421372,
|
|
"num_tokens": 365707.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 8.147554111480712,
|
|
"epoch": 0.16849529780564262,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.000107,
|
|
"loss": 7.5605,
|
|
"mean_token_accuracy": 0.06806492805480957,
|
|
"num_tokens": 374595.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"entropy": 8.051411581039428,
|
|
"epoch": 0.1724137931034483,
|
|
"grad_norm": 0.82421875,
|
|
"learning_rate": 0.0001095,
|
|
"loss": 7.5988,
|
|
"mean_token_accuracy": 0.06651411205530167,
|
|
"num_tokens": 383953.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 8.147121715545655,
|
|
"epoch": 0.17633228840125392,
|
|
"grad_norm": 0.9296875,
|
|
"learning_rate": 0.000112,
|
|
"loss": 7.5603,
|
|
"mean_token_accuracy": 0.06881510317325593,
|
|
"num_tokens": 393524.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"entropy": 8.113260126113891,
|
|
"epoch": 0.18025078369905956,
|
|
"grad_norm": 0.9140625,
|
|
"learning_rate": 0.0001145,
|
|
"loss": 7.439,
|
|
"mean_token_accuracy": 0.0729409821331501,
|
|
"num_tokens": 402079.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 8.0263240814209,
|
|
"epoch": 0.1841692789968652,
|
|
"grad_norm": 0.8828125,
|
|
"learning_rate": 0.00011700000000000001,
|
|
"loss": 7.4926,
|
|
"mean_token_accuracy": 0.06627442128956318,
|
|
"num_tokens": 410306.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"entropy": 8.06832423210144,
|
|
"epoch": 0.18808777429467086,
|
|
"grad_norm": 0.94140625,
|
|
"learning_rate": 0.00011949999999999999,
|
|
"loss": 7.508,
|
|
"mean_token_accuracy": 0.07089690454304218,
|
|
"num_tokens": 419157.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 8.074864530563355,
|
|
"epoch": 0.1920062695924765,
|
|
"grad_norm": 0.90234375,
|
|
"learning_rate": 0.000122,
|
|
"loss": 7.463,
|
|
"mean_token_accuracy": 0.06991211287677288,
|
|
"num_tokens": 428216.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"entropy": 7.9781323909759525,
|
|
"epoch": 0.19592476489028213,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.0001245,
|
|
"loss": 7.4362,
|
|
"mean_token_accuracy": 0.0743593417108059,
|
|
"num_tokens": 436695.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 8.013546228408813,
|
|
"epoch": 0.19984326018808776,
|
|
"grad_norm": 0.98046875,
|
|
"learning_rate": 0.000127,
|
|
"loss": 7.4641,
|
|
"mean_token_accuracy": 0.07037879787385463,
|
|
"num_tokens": 446801.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"entropy": 7.9741381168365475,
|
|
"epoch": 0.20376175548589343,
|
|
"grad_norm": 1.2890625,
|
|
"learning_rate": 0.0001295,
|
|
"loss": 7.4692,
|
|
"mean_token_accuracy": 0.0736465446650982,
|
|
"num_tokens": 454980.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 8.027658605575562,
|
|
"epoch": 0.20768025078369906,
|
|
"grad_norm": 0.88671875,
|
|
"learning_rate": 0.000132,
|
|
"loss": 7.4402,
|
|
"mean_token_accuracy": 0.07250457666814328,
|
|
"num_tokens": 463699.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"entropy": 8.018376350402832,
|
|
"epoch": 0.2115987460815047,
|
|
"grad_norm": 0.91015625,
|
|
"learning_rate": 0.00013450000000000002,
|
|
"loss": 7.334,
|
|
"mean_token_accuracy": 0.07564162723720073,
|
|
"num_tokens": 473005.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 7.891370153427124,
|
|
"epoch": 0.21551724137931033,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.00013700000000000002,
|
|
"loss": 7.3218,
|
|
"mean_token_accuracy": 0.07659829519689083,
|
|
"num_tokens": 481104.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"entropy": 7.955986738204956,
|
|
"epoch": 0.219435736677116,
|
|
"grad_norm": 0.890625,
|
|
"learning_rate": 0.0001395,
|
|
"loss": 7.4639,
|
|
"mean_token_accuracy": 0.07242306545376778,
|
|
"num_tokens": 490028.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 7.935779190063476,
|
|
"epoch": 0.22335423197492163,
|
|
"grad_norm": 0.91015625,
|
|
"learning_rate": 0.00014199999999999998,
|
|
"loss": 7.46,
|
|
"mean_token_accuracy": 0.07522198893129825,
|
|
"num_tokens": 499093.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"entropy": 8.021220302581787,
|
|
"epoch": 0.22727272727272727,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 0.0001445,
|
|
"loss": 7.4596,
|
|
"mean_token_accuracy": 0.07582265920937062,
|
|
"num_tokens": 508049.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 7.857068347930908,
|
|
"epoch": 0.23119122257053293,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 0.000147,
|
|
"loss": 7.276,
|
|
"mean_token_accuracy": 0.080800349265337,
|
|
"num_tokens": 516173.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"entropy": 7.936904716491699,
|
|
"epoch": 0.23510971786833856,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.0001495,
|
|
"loss": 7.3857,
|
|
"mean_token_accuracy": 0.07901114635169507,
|
|
"num_tokens": 524990.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 7.885950946807862,
|
|
"epoch": 0.2390282131661442,
|
|
"grad_norm": 0.9609375,
|
|
"learning_rate": 0.000152,
|
|
"loss": 7.4693,
|
|
"mean_token_accuracy": 0.07808374315500259,
|
|
"num_tokens": 533676.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"entropy": 7.922548103332519,
|
|
"epoch": 0.24294670846394983,
|
|
"grad_norm": 0.9921875,
|
|
"learning_rate": 0.00015450000000000001,
|
|
"loss": 7.4113,
|
|
"mean_token_accuracy": 0.0751194790005684,
|
|
"num_tokens": 542757.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 7.911523008346558,
|
|
"epoch": 0.2468652037617555,
|
|
"grad_norm": 1.3125,
|
|
"learning_rate": 0.000157,
|
|
"loss": 7.3816,
|
|
"mean_token_accuracy": 0.07483164295554161,
|
|
"num_tokens": 551958.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"entropy": 7.87831482887268,
|
|
"epoch": 0.2507836990595611,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0001595,
|
|
"loss": 7.297,
|
|
"mean_token_accuracy": 0.07816217541694641,
|
|
"num_tokens": 560850.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 7.742319774627686,
|
|
"epoch": 0.2547021943573668,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.000162,
|
|
"loss": 7.2442,
|
|
"mean_token_accuracy": 0.07955843061208726,
|
|
"num_tokens": 569226.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"entropy": 7.862335586547852,
|
|
"epoch": 0.25862068965517243,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00016450000000000001,
|
|
"loss": 7.2487,
|
|
"mean_token_accuracy": 0.08338543474674225,
|
|
"num_tokens": 577674.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 7.8038520336151125,
|
|
"epoch": 0.26253918495297807,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.00016700000000000002,
|
|
"loss": 7.3003,
|
|
"mean_token_accuracy": 0.08087649494409561,
|
|
"num_tokens": 587095.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"entropy": 7.815746021270752,
|
|
"epoch": 0.2664576802507837,
|
|
"grad_norm": 0.94921875,
|
|
"learning_rate": 0.00016950000000000003,
|
|
"loss": 7.2334,
|
|
"mean_token_accuracy": 0.08010689094662667,
|
|
"num_tokens": 595690.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 7.813080167770385,
|
|
"epoch": 0.27037617554858934,
|
|
"grad_norm": 1.0,
|
|
"learning_rate": 0.00017199999999999998,
|
|
"loss": 7.2638,
|
|
"mean_token_accuracy": 0.08633363470435143,
|
|
"num_tokens": 603962.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"entropy": 7.854545068740845,
|
|
"epoch": 0.274294670846395,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.00017449999999999999,
|
|
"loss": 7.2481,
|
|
"mean_token_accuracy": 0.08236029744148254,
|
|
"num_tokens": 612407.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 7.811877584457397,
|
|
"epoch": 0.2782131661442006,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.000177,
|
|
"loss": 7.3141,
|
|
"mean_token_accuracy": 0.08180430978536606,
|
|
"num_tokens": 620970.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"entropy": 7.737411594390869,
|
|
"epoch": 0.28213166144200624,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 0.0001795,
|
|
"loss": 7.2908,
|
|
"mean_token_accuracy": 0.07814904972910881,
|
|
"num_tokens": 630701.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 7.802158451080322,
|
|
"epoch": 0.28605015673981193,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.000182,
|
|
"loss": 7.2974,
|
|
"mean_token_accuracy": 0.08439014628529548,
|
|
"num_tokens": 640246.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"entropy": 7.708374309539795,
|
|
"epoch": 0.28996865203761757,
|
|
"grad_norm": 1.0078125,
|
|
"learning_rate": 0.0001845,
|
|
"loss": 7.2173,
|
|
"mean_token_accuracy": 0.08501755222678184,
|
|
"num_tokens": 648698.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 7.808612918853759,
|
|
"epoch": 0.2938871473354232,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.000187,
|
|
"loss": 7.2139,
|
|
"mean_token_accuracy": 0.09233642369508743,
|
|
"num_tokens": 658087.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"entropy": 7.6729991912841795,
|
|
"epoch": 0.29780564263322884,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.0001895,
|
|
"loss": 7.1842,
|
|
"mean_token_accuracy": 0.08876543864607811,
|
|
"num_tokens": 666664.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 7.760433673858643,
|
|
"epoch": 0.3017241379310345,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.000192,
|
|
"loss": 7.2527,
|
|
"mean_token_accuracy": 0.08252104483544827,
|
|
"num_tokens": 675738.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"entropy": 7.659246635437012,
|
|
"epoch": 0.3056426332288401,
|
|
"grad_norm": 0.93359375,
|
|
"learning_rate": 0.0001945,
|
|
"loss": 7.1341,
|
|
"mean_token_accuracy": 0.0883593812584877,
|
|
"num_tokens": 684751.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 7.6855284690856935,
|
|
"epoch": 0.30956112852664575,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.00019700000000000002,
|
|
"loss": 7.1392,
|
|
"mean_token_accuracy": 0.09001046866178512,
|
|
"num_tokens": 693819.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"entropy": 7.675692749023438,
|
|
"epoch": 0.31347962382445144,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.00019950000000000002,
|
|
"loss": 7.2002,
|
|
"mean_token_accuracy": 0.08597821295261383,
|
|
"num_tokens": 703082.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 7.6546083927154545,
|
|
"epoch": 0.31739811912225707,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.000202,
|
|
"loss": 7.1151,
|
|
"mean_token_accuracy": 0.08707506135106087,
|
|
"num_tokens": 711721.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"entropy": 7.646405744552612,
|
|
"epoch": 0.3213166144200627,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.00020449999999999998,
|
|
"loss": 7.1256,
|
|
"mean_token_accuracy": 0.08994124978780746,
|
|
"num_tokens": 719825.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 7.639235925674439,
|
|
"epoch": 0.32523510971786834,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.000207,
|
|
"loss": 7.053,
|
|
"mean_token_accuracy": 0.08337244391441345,
|
|
"num_tokens": 728575.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"entropy": 7.728874778747558,
|
|
"epoch": 0.329153605015674,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.0002095,
|
|
"loss": 7.2405,
|
|
"mean_token_accuracy": 0.08313387259840965,
|
|
"num_tokens": 738379.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 7.568649673461914,
|
|
"epoch": 0.3330721003134796,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.000212,
|
|
"loss": 7.0653,
|
|
"mean_token_accuracy": 0.0943116620182991,
|
|
"num_tokens": 747484.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"entropy": 7.662053632736206,
|
|
"epoch": 0.33699059561128525,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0002145,
|
|
"loss": 7.0976,
|
|
"mean_token_accuracy": 0.09167452827095986,
|
|
"num_tokens": 756250.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 7.631493139266968,
|
|
"epoch": 0.3409090909090909,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.00021700000000000002,
|
|
"loss": 7.1553,
|
|
"mean_token_accuracy": 0.09070844799280167,
|
|
"num_tokens": 765393.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"entropy": 7.533075618743896,
|
|
"epoch": 0.3448275862068966,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0002195,
|
|
"loss": 7.1282,
|
|
"mean_token_accuracy": 0.09012485444545745,
|
|
"num_tokens": 773726.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 7.584697198867798,
|
|
"epoch": 0.3487460815047022,
|
|
"grad_norm": 1.0078125,
|
|
"learning_rate": 0.000222,
|
|
"loss": 7.0359,
|
|
"mean_token_accuracy": 0.09066722467541695,
|
|
"num_tokens": 782335.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"entropy": 7.568591690063476,
|
|
"epoch": 0.35266457680250785,
|
|
"grad_norm": 1.0078125,
|
|
"learning_rate": 0.0002245,
|
|
"loss": 6.9887,
|
|
"mean_token_accuracy": 0.09229951500892639,
|
|
"num_tokens": 791546.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 7.518783092498779,
|
|
"epoch": 0.3565830721003135,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.00022700000000000002,
|
|
"loss": 6.964,
|
|
"mean_token_accuracy": 0.09283199757337571,
|
|
"num_tokens": 800650.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"entropy": 7.47516098022461,
|
|
"epoch": 0.3605015673981191,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.00022950000000000002,
|
|
"loss": 7.0328,
|
|
"mean_token_accuracy": 0.09423457458615303,
|
|
"num_tokens": 809968.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 7.544811105728149,
|
|
"epoch": 0.36442006269592475,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.00023200000000000003,
|
|
"loss": 7.0201,
|
|
"mean_token_accuracy": 0.09420157298445701,
|
|
"num_tokens": 818800.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"entropy": 7.581027984619141,
|
|
"epoch": 0.3683385579937304,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.00023449999999999998,
|
|
"loss": 6.9864,
|
|
"mean_token_accuracy": 0.09675867483019829,
|
|
"num_tokens": 827586.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 7.303757953643799,
|
|
"epoch": 0.3722570532915361,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.000237,
|
|
"loss": 6.9384,
|
|
"mean_token_accuracy": 0.09580465108156204,
|
|
"num_tokens": 836730.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"entropy": 7.696502780914306,
|
|
"epoch": 0.3761755485893417,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0002395,
|
|
"loss": 7.1422,
|
|
"mean_token_accuracy": 0.09025072157382966,
|
|
"num_tokens": 845105.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 7.521749973297119,
|
|
"epoch": 0.38009404388714735,
|
|
"grad_norm": 1.328125,
|
|
"learning_rate": 0.000242,
|
|
"loss": 7.0783,
|
|
"mean_token_accuracy": 0.09295645728707314,
|
|
"num_tokens": 853584.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"entropy": 7.556596755981445,
|
|
"epoch": 0.384012539184953,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.0002445,
|
|
"loss": 7.0668,
|
|
"mean_token_accuracy": 0.08822977468371392,
|
|
"num_tokens": 862442.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 7.4543486595153805,
|
|
"epoch": 0.3879310344827586,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 0.000247,
|
|
"loss": 6.9643,
|
|
"mean_token_accuracy": 0.0951703853905201,
|
|
"num_tokens": 870833.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"entropy": 7.596196413040161,
|
|
"epoch": 0.39184952978056425,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.0002495,
|
|
"loss": 7.1087,
|
|
"mean_token_accuracy": 0.09554079249501228,
|
|
"num_tokens": 879461.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.39184952978056425,
|
|
"eval_entropy": 7.3791314095489735,
|
|
"eval_loss": 7.098165035247803,
|
|
"eval_mean_token_accuracy": 0.09636555407051892,
|
|
"eval_num_tokens": 879461.0,
|
|
"eval_runtime": 2.2153,
|
|
"eval_samples_per_second": 1860.68,
|
|
"eval_steps_per_second": 232.924,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 7.323290538787842,
|
|
"epoch": 0.3957680250783699,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.000252,
|
|
"loss": 6.8468,
|
|
"mean_token_accuracy": 0.10375453308224677,
|
|
"num_tokens": 887893.0,
|
|
"step": 505
|
|
},
|
|
{
|
|
"entropy": 7.584144163131714,
|
|
"epoch": 0.3996865203761755,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.0002545,
|
|
"loss": 7.078,
|
|
"mean_token_accuracy": 0.09517347961664199,
|
|
"num_tokens": 896129.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"entropy": 7.514800357818603,
|
|
"epoch": 0.4036050156739812,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.000257,
|
|
"loss": 7.0467,
|
|
"mean_token_accuracy": 0.0908431887626648,
|
|
"num_tokens": 904448.0,
|
|
"step": 515
|
|
},
|
|
{
|
|
"entropy": 7.4618706703186035,
|
|
"epoch": 0.40752351097178685,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.0002595,
|
|
"loss": 6.9808,
|
|
"mean_token_accuracy": 0.09821045622229577,
|
|
"num_tokens": 912486.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"entropy": 7.48437385559082,
|
|
"epoch": 0.4114420062695925,
|
|
"grad_norm": 1.0,
|
|
"learning_rate": 0.000262,
|
|
"loss": 6.9342,
|
|
"mean_token_accuracy": 0.0975078672170639,
|
|
"num_tokens": 920913.0,
|
|
"step": 525
|
|
},
|
|
{
|
|
"entropy": 7.505733489990234,
|
|
"epoch": 0.4153605015673981,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.00026450000000000003,
|
|
"loss": 7.0757,
|
|
"mean_token_accuracy": 0.08893783465027809,
|
|
"num_tokens": 930227.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"entropy": 7.452349138259888,
|
|
"epoch": 0.41927899686520376,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.00026700000000000004,
|
|
"loss": 6.9462,
|
|
"mean_token_accuracy": 0.09549824744462967,
|
|
"num_tokens": 938835.0,
|
|
"step": 535
|
|
},
|
|
{
|
|
"entropy": 7.436382722854614,
|
|
"epoch": 0.4231974921630094,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.00026950000000000005,
|
|
"loss": 6.9256,
|
|
"mean_token_accuracy": 0.10100051760673523,
|
|
"num_tokens": 946930.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"entropy": 7.43581748008728,
|
|
"epoch": 0.427115987460815,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.00027200000000000005,
|
|
"loss": 6.9024,
|
|
"mean_token_accuracy": 0.09818659126758575,
|
|
"num_tokens": 955450.0,
|
|
"step": 545
|
|
},
|
|
{
|
|
"entropy": 7.412800025939942,
|
|
"epoch": 0.43103448275862066,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 0.0002745,
|
|
"loss": 6.9804,
|
|
"mean_token_accuracy": 0.09347106665372848,
|
|
"num_tokens": 964456.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"entropy": 7.413184976577758,
|
|
"epoch": 0.43495297805642635,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.000277,
|
|
"loss": 6.8786,
|
|
"mean_token_accuracy": 0.0999380074441433,
|
|
"num_tokens": 973088.0,
|
|
"step": 555
|
|
},
|
|
{
|
|
"entropy": 7.320905685424805,
|
|
"epoch": 0.438871473354232,
|
|
"grad_norm": 1.3046875,
|
|
"learning_rate": 0.0002795,
|
|
"loss": 6.8844,
|
|
"mean_token_accuracy": 0.09666496440768242,
|
|
"num_tokens": 982182.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"entropy": 7.376274633407593,
|
|
"epoch": 0.4427899686520376,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.00028199999999999997,
|
|
"loss": 6.8457,
|
|
"mean_token_accuracy": 0.10074941292405129,
|
|
"num_tokens": 990513.0,
|
|
"step": 565
|
|
},
|
|
{
|
|
"entropy": 7.300549602508545,
|
|
"epoch": 0.44670846394984326,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0002845,
|
|
"loss": 6.857,
|
|
"mean_token_accuracy": 0.09506906270980835,
|
|
"num_tokens": 999394.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"entropy": 7.375359916687012,
|
|
"epoch": 0.4506269592476489,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.000287,
|
|
"loss": 6.8453,
|
|
"mean_token_accuracy": 0.1013949416577816,
|
|
"num_tokens": 1007793.0,
|
|
"step": 575
|
|
},
|
|
{
|
|
"entropy": 7.34481520652771,
|
|
"epoch": 0.45454545454545453,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.0002895,
|
|
"loss": 6.9084,
|
|
"mean_token_accuracy": 0.09777404814958572,
|
|
"num_tokens": 1017198.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"entropy": 7.3425328731536865,
|
|
"epoch": 0.45846394984326017,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 0.000292,
|
|
"loss": 6.8047,
|
|
"mean_token_accuracy": 0.10280357003211975,
|
|
"num_tokens": 1025896.0,
|
|
"step": 585
|
|
},
|
|
{
|
|
"entropy": 7.445605945587158,
|
|
"epoch": 0.46238244514106586,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.0002945,
|
|
"loss": 7.0133,
|
|
"mean_token_accuracy": 0.09696159362792969,
|
|
"num_tokens": 1034540.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"entropy": 7.213116216659546,
|
|
"epoch": 0.4663009404388715,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.000297,
|
|
"loss": 6.8321,
|
|
"mean_token_accuracy": 0.10862131938338279,
|
|
"num_tokens": 1043006.0,
|
|
"step": 595
|
|
},
|
|
{
|
|
"entropy": 7.316764163970947,
|
|
"epoch": 0.4702194357366771,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.0002995,
|
|
"loss": 6.8099,
|
|
"mean_token_accuracy": 0.10350289791822434,
|
|
"num_tokens": 1052530.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"entropy": 7.323724317550659,
|
|
"epoch": 0.47413793103448276,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.000302,
|
|
"loss": 6.8137,
|
|
"mean_token_accuracy": 0.10041582360863685,
|
|
"num_tokens": 1060822.0,
|
|
"step": 605
|
|
},
|
|
{
|
|
"entropy": 7.30889310836792,
|
|
"epoch": 0.4780564263322884,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.0003045,
|
|
"loss": 6.8802,
|
|
"mean_token_accuracy": 0.10094830319285393,
|
|
"num_tokens": 1069962.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"entropy": 7.313723564147949,
|
|
"epoch": 0.48197492163009403,
|
|
"grad_norm": 0.9609375,
|
|
"learning_rate": 0.000307,
|
|
"loss": 6.806,
|
|
"mean_token_accuracy": 0.10286147370934487,
|
|
"num_tokens": 1078903.0,
|
|
"step": 615
|
|
},
|
|
{
|
|
"entropy": 7.307069587707519,
|
|
"epoch": 0.48589341692789967,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.0003095,
|
|
"loss": 6.8794,
|
|
"mean_token_accuracy": 0.09798636510968209,
|
|
"num_tokens": 1087921.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"entropy": 7.2093301773071286,
|
|
"epoch": 0.4898119122257053,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.000312,
|
|
"loss": 6.7742,
|
|
"mean_token_accuracy": 0.10318923816084861,
|
|
"num_tokens": 1096778.0,
|
|
"step": 625
|
|
},
|
|
{
|
|
"entropy": 7.321264219284058,
|
|
"epoch": 0.493730407523511,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.0003145,
|
|
"loss": 6.8046,
|
|
"mean_token_accuracy": 0.10321516692638397,
|
|
"num_tokens": 1105000.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"entropy": 7.175268077850342,
|
|
"epoch": 0.49764890282131663,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 0.000317,
|
|
"loss": 6.6925,
|
|
"mean_token_accuracy": 0.10435841083526612,
|
|
"num_tokens": 1113475.0,
|
|
"step": 635
|
|
},
|
|
{
|
|
"entropy": 7.1677467823028564,
|
|
"epoch": 0.5015673981191222,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.0003195,
|
|
"loss": 6.7436,
|
|
"mean_token_accuracy": 0.10689813792705535,
|
|
"num_tokens": 1122188.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"entropy": 7.284512710571289,
|
|
"epoch": 0.5054858934169278,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 0.000322,
|
|
"loss": 6.8446,
|
|
"mean_token_accuracy": 0.10488534867763519,
|
|
"num_tokens": 1130781.0,
|
|
"step": 645
|
|
},
|
|
{
|
|
"entropy": 7.213377618789673,
|
|
"epoch": 0.5094043887147336,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00032450000000000003,
|
|
"loss": 6.771,
|
|
"mean_token_accuracy": 0.09970749393105507,
|
|
"num_tokens": 1139889.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"entropy": 7.230893850326538,
|
|
"epoch": 0.5133228840125392,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 0.00032700000000000003,
|
|
"loss": 6.7344,
|
|
"mean_token_accuracy": 0.10886386185884475,
|
|
"num_tokens": 1147904.0,
|
|
"step": 655
|
|
},
|
|
{
|
|
"entropy": 7.088760614395142,
|
|
"epoch": 0.5172413793103449,
|
|
"grad_norm": 1.3203125,
|
|
"learning_rate": 0.00032950000000000004,
|
|
"loss": 6.696,
|
|
"mean_token_accuracy": 0.1124557226896286,
|
|
"num_tokens": 1157413.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"entropy": 7.306563091278076,
|
|
"epoch": 0.5211598746081505,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.00033200000000000005,
|
|
"loss": 6.8164,
|
|
"mean_token_accuracy": 0.10038655102252961,
|
|
"num_tokens": 1165635.0,
|
|
"step": 665
|
|
},
|
|
{
|
|
"entropy": 7.134089422225952,
|
|
"epoch": 0.5250783699059561,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00033450000000000005,
|
|
"loss": 6.8298,
|
|
"mean_token_accuracy": 0.09688482508063316,
|
|
"num_tokens": 1174681.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"entropy": 7.235197401046753,
|
|
"epoch": 0.5289968652037618,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.000337,
|
|
"loss": 6.8225,
|
|
"mean_token_accuracy": 0.10358083248138428,
|
|
"num_tokens": 1183454.0,
|
|
"step": 675
|
|
},
|
|
{
|
|
"entropy": 7.254007053375244,
|
|
"epoch": 0.5329153605015674,
|
|
"grad_norm": 1.3046875,
|
|
"learning_rate": 0.0003395,
|
|
"loss": 6.7803,
|
|
"mean_token_accuracy": 0.10250866636633874,
|
|
"num_tokens": 1191836.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"entropy": 7.161578273773193,
|
|
"epoch": 0.536833855799373,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.000342,
|
|
"loss": 6.7006,
|
|
"mean_token_accuracy": 0.10597401708364487,
|
|
"num_tokens": 1200280.0,
|
|
"step": 685
|
|
},
|
|
{
|
|
"entropy": 7.203849601745605,
|
|
"epoch": 0.5407523510971787,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.00034449999999999997,
|
|
"loss": 6.8066,
|
|
"mean_token_accuracy": 0.09864948689937592,
|
|
"num_tokens": 1208773.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"entropy": 7.236851644515991,
|
|
"epoch": 0.5446708463949843,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.000347,
|
|
"loss": 6.795,
|
|
"mean_token_accuracy": 0.10489028915762902,
|
|
"num_tokens": 1218171.0,
|
|
"step": 695
|
|
},
|
|
{
|
|
"entropy": 7.15051794052124,
|
|
"epoch": 0.54858934169279,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.0003495,
|
|
"loss": 6.7532,
|
|
"mean_token_accuracy": 0.10080546215176582,
|
|
"num_tokens": 1226549.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"entropy": 7.200709390640259,
|
|
"epoch": 0.5525078369905956,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.000352,
|
|
"loss": 6.7439,
|
|
"mean_token_accuracy": 0.11170587316155434,
|
|
"num_tokens": 1234995.0,
|
|
"step": 705
|
|
},
|
|
{
|
|
"entropy": 7.17590970993042,
|
|
"epoch": 0.5564263322884012,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0003545,
|
|
"loss": 6.787,
|
|
"mean_token_accuracy": 0.10626475065946579,
|
|
"num_tokens": 1243807.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"entropy": 7.112601041793823,
|
|
"epoch": 0.5603448275862069,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.000357,
|
|
"loss": 6.7665,
|
|
"mean_token_accuracy": 0.10710262060165406,
|
|
"num_tokens": 1252773.0,
|
|
"step": 715
|
|
},
|
|
{
|
|
"entropy": 7.164514064788818,
|
|
"epoch": 0.5642633228840125,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.0003595,
|
|
"loss": 6.7316,
|
|
"mean_token_accuracy": 0.09972823187708854,
|
|
"num_tokens": 1262514.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"entropy": 7.2263566017150875,
|
|
"epoch": 0.5681818181818182,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.000362,
|
|
"loss": 6.7304,
|
|
"mean_token_accuracy": 0.10351964756846428,
|
|
"num_tokens": 1271329.0,
|
|
"step": 725
|
|
},
|
|
{
|
|
"entropy": 7.020602416992188,
|
|
"epoch": 0.5721003134796239,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 0.0003645,
|
|
"loss": 6.6153,
|
|
"mean_token_accuracy": 0.1127767838537693,
|
|
"num_tokens": 1279563.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"entropy": 7.153738784790039,
|
|
"epoch": 0.5760188087774295,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.000367,
|
|
"loss": 6.7494,
|
|
"mean_token_accuracy": 0.10431532934308052,
|
|
"num_tokens": 1288906.0,
|
|
"step": 735
|
|
},
|
|
{
|
|
"entropy": 7.086837673187256,
|
|
"epoch": 0.5799373040752351,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.0003695,
|
|
"loss": 6.6582,
|
|
"mean_token_accuracy": 0.10919351503252983,
|
|
"num_tokens": 1297880.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"entropy": 7.083261871337891,
|
|
"epoch": 0.5838557993730408,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 0.000372,
|
|
"loss": 6.6923,
|
|
"mean_token_accuracy": 0.10958038419485092,
|
|
"num_tokens": 1306740.0,
|
|
"step": 745
|
|
},
|
|
{
|
|
"entropy": 7.1014440059661865,
|
|
"epoch": 0.5877742946708464,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.0003745,
|
|
"loss": 6.6471,
|
|
"mean_token_accuracy": 0.10642243772745133,
|
|
"num_tokens": 1315997.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"entropy": 7.118290996551513,
|
|
"epoch": 0.591692789968652,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.000377,
|
|
"loss": 6.634,
|
|
"mean_token_accuracy": 0.11355772763490676,
|
|
"num_tokens": 1324064.0,
|
|
"step": 755
|
|
},
|
|
{
|
|
"entropy": 7.116359090805053,
|
|
"epoch": 0.5956112852664577,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.0003795,
|
|
"loss": 6.7624,
|
|
"mean_token_accuracy": 0.10623705238103867,
|
|
"num_tokens": 1332560.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"entropy": 7.0753131866455075,
|
|
"epoch": 0.5995297805642633,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.000382,
|
|
"loss": 6.6853,
|
|
"mean_token_accuracy": 0.10491749420762062,
|
|
"num_tokens": 1341367.0,
|
|
"step": 765
|
|
},
|
|
{
|
|
"entropy": 7.208959865570068,
|
|
"epoch": 0.603448275862069,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0003845,
|
|
"loss": 6.8643,
|
|
"mean_token_accuracy": 0.09926253855228424,
|
|
"num_tokens": 1350602.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"entropy": 7.111758089065551,
|
|
"epoch": 0.6073667711598746,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.00038700000000000003,
|
|
"loss": 6.5959,
|
|
"mean_token_accuracy": 0.11473986878991127,
|
|
"num_tokens": 1358538.0,
|
|
"step": 775
|
|
},
|
|
{
|
|
"entropy": 6.9510876655578615,
|
|
"epoch": 0.6112852664576802,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00038950000000000003,
|
|
"loss": 6.5966,
|
|
"mean_token_accuracy": 0.11128686666488648,
|
|
"num_tokens": 1367089.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"entropy": 7.095315790176391,
|
|
"epoch": 0.6152037617554859,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.00039200000000000004,
|
|
"loss": 6.6987,
|
|
"mean_token_accuracy": 0.107613255828619,
|
|
"num_tokens": 1376095.0,
|
|
"step": 785
|
|
},
|
|
{
|
|
"entropy": 7.0291649341583256,
|
|
"epoch": 0.6191222570532915,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.00039450000000000005,
|
|
"loss": 6.7278,
|
|
"mean_token_accuracy": 0.10492047220468521,
|
|
"num_tokens": 1385132.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"entropy": 7.174719142913818,
|
|
"epoch": 0.6230407523510971,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 0.00039700000000000005,
|
|
"loss": 6.8042,
|
|
"mean_token_accuracy": 0.10808081552386284,
|
|
"num_tokens": 1394187.0,
|
|
"step": 795
|
|
},
|
|
{
|
|
"entropy": 7.009328317642212,
|
|
"epoch": 0.6269592476489029,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0003995,
|
|
"loss": 6.6543,
|
|
"mean_token_accuracy": 0.10319057255983352,
|
|
"num_tokens": 1403250.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"entropy": 7.114693927764892,
|
|
"epoch": 0.6308777429467085,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.000402,
|
|
"loss": 6.7523,
|
|
"mean_token_accuracy": 0.10417917445302009,
|
|
"num_tokens": 1412768.0,
|
|
"step": 805
|
|
},
|
|
{
|
|
"entropy": 7.1693981170654295,
|
|
"epoch": 0.6347962382445141,
|
|
"grad_norm": 0.984375,
|
|
"learning_rate": 0.0004045,
|
|
"loss": 6.7831,
|
|
"mean_token_accuracy": 0.10479006171226501,
|
|
"num_tokens": 1422437.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"entropy": 7.221331596374512,
|
|
"epoch": 0.6387147335423198,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.00040699999999999997,
|
|
"loss": 6.8352,
|
|
"mean_token_accuracy": 0.10820698514580726,
|
|
"num_tokens": 1432540.0,
|
|
"step": 815
|
|
},
|
|
{
|
|
"entropy": 6.996674919128418,
|
|
"epoch": 0.6426332288401254,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.0004095,
|
|
"loss": 6.6518,
|
|
"mean_token_accuracy": 0.10877189710736275,
|
|
"num_tokens": 1441403.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"entropy": 6.932708883285523,
|
|
"epoch": 0.646551724137931,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.000412,
|
|
"loss": 6.4891,
|
|
"mean_token_accuracy": 0.10969182550907135,
|
|
"num_tokens": 1450059.0,
|
|
"step": 825
|
|
},
|
|
{
|
|
"entropy": 7.004702997207642,
|
|
"epoch": 0.6504702194357367,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 0.0004145,
|
|
"loss": 6.6424,
|
|
"mean_token_accuracy": 0.11137095913290977,
|
|
"num_tokens": 1458413.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"entropy": 6.982443046569824,
|
|
"epoch": 0.6543887147335423,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.000417,
|
|
"loss": 6.622,
|
|
"mean_token_accuracy": 0.11147240921854973,
|
|
"num_tokens": 1467343.0,
|
|
"step": 835
|
|
},
|
|
{
|
|
"entropy": 6.956045198440552,
|
|
"epoch": 0.658307210031348,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.0004195,
|
|
"loss": 6.5607,
|
|
"mean_token_accuracy": 0.11432239189743995,
|
|
"num_tokens": 1476250.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"entropy": 7.017556047439575,
|
|
"epoch": 0.6622257053291536,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.000422,
|
|
"loss": 6.5268,
|
|
"mean_token_accuracy": 0.11572841182351112,
|
|
"num_tokens": 1484663.0,
|
|
"step": 845
|
|
},
|
|
{
|
|
"entropy": 7.062141704559326,
|
|
"epoch": 0.6661442006269592,
|
|
"grad_norm": 1.3203125,
|
|
"learning_rate": 0.0004245,
|
|
"loss": 6.6401,
|
|
"mean_token_accuracy": 0.10651951879262925,
|
|
"num_tokens": 1493898.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"entropy": 6.941916561126709,
|
|
"epoch": 0.6700626959247649,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 0.000427,
|
|
"loss": 6.6059,
|
|
"mean_token_accuracy": 0.1140934869647026,
|
|
"num_tokens": 1502429.0,
|
|
"step": 855
|
|
},
|
|
{
|
|
"entropy": 6.927605772018433,
|
|
"epoch": 0.6739811912225705,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 0.0004295,
|
|
"loss": 6.6293,
|
|
"mean_token_accuracy": 0.11264230906963349,
|
|
"num_tokens": 1511671.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"entropy": 6.936274194717408,
|
|
"epoch": 0.6778996865203761,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.000432,
|
|
"loss": 6.5421,
|
|
"mean_token_accuracy": 0.11239867955446244,
|
|
"num_tokens": 1520481.0,
|
|
"step": 865
|
|
},
|
|
{
|
|
"entropy": 6.910920095443726,
|
|
"epoch": 0.6818181818181818,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0004345,
|
|
"loss": 6.4874,
|
|
"mean_token_accuracy": 0.11565134301781654,
|
|
"num_tokens": 1528589.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"entropy": 6.936212968826294,
|
|
"epoch": 0.6857366771159875,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.000437,
|
|
"loss": 6.6584,
|
|
"mean_token_accuracy": 0.108811154961586,
|
|
"num_tokens": 1537555.0,
|
|
"step": 875
|
|
},
|
|
{
|
|
"entropy": 6.945295000076294,
|
|
"epoch": 0.6896551724137931,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.0004395,
|
|
"loss": 6.6032,
|
|
"mean_token_accuracy": 0.11123565956950188,
|
|
"num_tokens": 1546397.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"entropy": 7.034606647491455,
|
|
"epoch": 0.6935736677115988,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.000442,
|
|
"loss": 6.6331,
|
|
"mean_token_accuracy": 0.11215913444757461,
|
|
"num_tokens": 1555603.0,
|
|
"step": 885
|
|
},
|
|
{
|
|
"entropy": 6.92329592704773,
|
|
"epoch": 0.6974921630094044,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.0004445,
|
|
"loss": 6.5938,
|
|
"mean_token_accuracy": 0.11114713251590728,
|
|
"num_tokens": 1564688.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"entropy": 6.946131467819214,
|
|
"epoch": 0.70141065830721,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 0.000447,
|
|
"loss": 6.5727,
|
|
"mean_token_accuracy": 0.1131402924656868,
|
|
"num_tokens": 1573490.0,
|
|
"step": 895
|
|
},
|
|
{
|
|
"entropy": 6.9099291324615475,
|
|
"epoch": 0.7053291536050157,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.00044950000000000003,
|
|
"loss": 6.5425,
|
|
"mean_token_accuracy": 0.11272019520401955,
|
|
"num_tokens": 1582668.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"entropy": 6.931637620925903,
|
|
"epoch": 0.7092476489028213,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.00045200000000000004,
|
|
"loss": 6.6098,
|
|
"mean_token_accuracy": 0.11104563847184182,
|
|
"num_tokens": 1591667.0,
|
|
"step": 905
|
|
},
|
|
{
|
|
"entropy": 6.885412693023682,
|
|
"epoch": 0.713166144200627,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.00045450000000000004,
|
|
"loss": 6.5154,
|
|
"mean_token_accuracy": 0.11723392531275749,
|
|
"num_tokens": 1600248.0,
|
|
"step": 910
|
|
},
|
|
{
|
|
"entropy": 6.9636177062988285,
|
|
"epoch": 0.7170846394984326,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.00045700000000000005,
|
|
"loss": 6.5222,
|
|
"mean_token_accuracy": 0.11748122572898864,
|
|
"num_tokens": 1609094.0,
|
|
"step": 915
|
|
},
|
|
{
|
|
"entropy": 6.80591516494751,
|
|
"epoch": 0.7210031347962382,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 0.00045950000000000006,
|
|
"loss": 6.5523,
|
|
"mean_token_accuracy": 0.11386217251420021,
|
|
"num_tokens": 1617845.0,
|
|
"step": 920
|
|
},
|
|
{
|
|
"entropy": 7.011699295043945,
|
|
"epoch": 0.7249216300940439,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.000462,
|
|
"loss": 6.5865,
|
|
"mean_token_accuracy": 0.11231657788157463,
|
|
"num_tokens": 1626596.0,
|
|
"step": 925
|
|
},
|
|
{
|
|
"entropy": 6.840612173080444,
|
|
"epoch": 0.7288401253918495,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.0004645,
|
|
"loss": 6.5,
|
|
"mean_token_accuracy": 0.11078331917524338,
|
|
"num_tokens": 1635246.0,
|
|
"step": 930
|
|
},
|
|
{
|
|
"entropy": 6.838626718521118,
|
|
"epoch": 0.7327586206896551,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.000467,
|
|
"loss": 6.556,
|
|
"mean_token_accuracy": 0.11381039470434189,
|
|
"num_tokens": 1645041.0,
|
|
"step": 935
|
|
},
|
|
{
|
|
"entropy": 6.995179224014282,
|
|
"epoch": 0.7366771159874608,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.0004695,
|
|
"loss": 6.6265,
|
|
"mean_token_accuracy": 0.10761500224471092,
|
|
"num_tokens": 1653636.0,
|
|
"step": 940
|
|
},
|
|
{
|
|
"entropy": 6.835239219665527,
|
|
"epoch": 0.7405956112852664,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.000472,
|
|
"loss": 6.5227,
|
|
"mean_token_accuracy": 0.11463285759091377,
|
|
"num_tokens": 1662485.0,
|
|
"step": 945
|
|
},
|
|
{
|
|
"entropy": 6.926279592514038,
|
|
"epoch": 0.7445141065830722,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.0004745,
|
|
"loss": 6.5461,
|
|
"mean_token_accuracy": 0.12085802108049393,
|
|
"num_tokens": 1670414.0,
|
|
"step": 950
|
|
},
|
|
{
|
|
"entropy": 6.897070741653442,
|
|
"epoch": 0.7484326018808778,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 0.000477,
|
|
"loss": 6.5507,
|
|
"mean_token_accuracy": 0.11035623922944068,
|
|
"num_tokens": 1678853.0,
|
|
"step": 955
|
|
},
|
|
{
|
|
"entropy": 6.948398160934448,
|
|
"epoch": 0.7523510971786834,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0004795,
|
|
"loss": 6.4881,
|
|
"mean_token_accuracy": 0.11652906686067581,
|
|
"num_tokens": 1687286.0,
|
|
"step": 960
|
|
},
|
|
{
|
|
"entropy": 6.789101934432983,
|
|
"epoch": 0.7562695924764891,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.000482,
|
|
"loss": 6.5127,
|
|
"mean_token_accuracy": 0.1135128065943718,
|
|
"num_tokens": 1696302.0,
|
|
"step": 965
|
|
},
|
|
{
|
|
"entropy": 6.88850679397583,
|
|
"epoch": 0.7601880877742947,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.0004845,
|
|
"loss": 6.6328,
|
|
"mean_token_accuracy": 0.10952614322304725,
|
|
"num_tokens": 1704622.0,
|
|
"step": 970
|
|
},
|
|
{
|
|
"entropy": 6.836090850830078,
|
|
"epoch": 0.7641065830721003,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.000487,
|
|
"loss": 6.4682,
|
|
"mean_token_accuracy": 0.11953761354088784,
|
|
"num_tokens": 1713060.0,
|
|
"step": 975
|
|
},
|
|
{
|
|
"entropy": 6.769405508041382,
|
|
"epoch": 0.768025078369906,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0004895,
|
|
"loss": 6.4437,
|
|
"mean_token_accuracy": 0.1201899990439415,
|
|
"num_tokens": 1721654.0,
|
|
"step": 980
|
|
},
|
|
{
|
|
"entropy": 6.872002363204956,
|
|
"epoch": 0.7719435736677116,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.000492,
|
|
"loss": 6.4967,
|
|
"mean_token_accuracy": 0.11558816209435463,
|
|
"num_tokens": 1730243.0,
|
|
"step": 985
|
|
},
|
|
{
|
|
"entropy": 6.746385478973389,
|
|
"epoch": 0.7758620689655172,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.0004945,
|
|
"loss": 6.4224,
|
|
"mean_token_accuracy": 0.11981726884841919,
|
|
"num_tokens": 1739347.0,
|
|
"step": 990
|
|
},
|
|
{
|
|
"entropy": 6.771752119064331,
|
|
"epoch": 0.7797805642633229,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.000497,
|
|
"loss": 6.4607,
|
|
"mean_token_accuracy": 0.12106571421027183,
|
|
"num_tokens": 1747802.0,
|
|
"step": 995
|
|
},
|
|
{
|
|
"entropy": 6.778915500640869,
|
|
"epoch": 0.7836990595611285,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.0004995,
|
|
"loss": 6.4581,
|
|
"mean_token_accuracy": 0.11269956901669502,
|
|
"num_tokens": 1757407.0,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.7836990595611285,
|
|
"eval_entropy": 6.515588334364485,
|
|
"eval_loss": 6.588331699371338,
|
|
"eval_mean_token_accuracy": 0.1149799013077173,
|
|
"eval_num_tokens": 1757407.0,
|
|
"eval_runtime": 2.3882,
|
|
"eval_samples_per_second": 1725.989,
|
|
"eval_steps_per_second": 216.063,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"entropy": 6.8325563907623295,
|
|
"epoch": 0.7876175548589341,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.0004999998713243189,
|
|
"loss": 6.5047,
|
|
"mean_token_accuracy": 0.11974181681871414,
|
|
"num_tokens": 1766323.0,
|
|
"step": 1005
|
|
},
|
|
{
|
|
"entropy": 6.736519289016724,
|
|
"epoch": 0.7915360501567398,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.0004999993485796164,
|
|
"loss": 6.4707,
|
|
"mean_token_accuracy": 0.11981728523969651,
|
|
"num_tokens": 1776193.0,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"entropy": 6.85446834564209,
|
|
"epoch": 0.7954545454545454,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0004999984237245962,
|
|
"loss": 6.5192,
|
|
"mean_token_accuracy": 0.1184984378516674,
|
|
"num_tokens": 1785813.0,
|
|
"step": 1015
|
|
},
|
|
{
|
|
"entropy": 6.7683358669281,
|
|
"epoch": 0.799373040752351,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.0004999970967609109,
|
|
"loss": 6.4234,
|
|
"mean_token_accuracy": 0.11528945416212082,
|
|
"num_tokens": 1794448.0,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"entropy": 6.738554382324219,
|
|
"epoch": 0.8032915360501567,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0004999953676909322,
|
|
"loss": 6.4485,
|
|
"mean_token_accuracy": 0.11520285606384277,
|
|
"num_tokens": 1803091.0,
|
|
"step": 1025
|
|
},
|
|
{
|
|
"entropy": 6.897971105575562,
|
|
"epoch": 0.8072100313479624,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0004999932365177502,
|
|
"loss": 6.5432,
|
|
"mean_token_accuracy": 0.11414010152220726,
|
|
"num_tokens": 1811649.0,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"entropy": 6.877712345123291,
|
|
"epoch": 0.8111285266457681,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.0004999907032451735,
|
|
"loss": 6.5178,
|
|
"mean_token_accuracy": 0.11190005615353585,
|
|
"num_tokens": 1820060.0,
|
|
"step": 1035
|
|
},
|
|
{
|
|
"entropy": 6.638827466964722,
|
|
"epoch": 0.8150470219435737,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0004999877678777296,
|
|
"loss": 6.381,
|
|
"mean_token_accuracy": 0.12132669016718864,
|
|
"num_tokens": 1828917.0,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"entropy": 6.85034146308899,
|
|
"epoch": 0.8189655172413793,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004999844304206645,
|
|
"loss": 6.4692,
|
|
"mean_token_accuracy": 0.11515972465276718,
|
|
"num_tokens": 1838166.0,
|
|
"step": 1045
|
|
},
|
|
{
|
|
"entropy": 6.693251895904541,
|
|
"epoch": 0.822884012539185,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004999806908799428,
|
|
"loss": 6.3677,
|
|
"mean_token_accuracy": 0.12894104793667793,
|
|
"num_tokens": 1846752.0,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"entropy": 6.830075883865357,
|
|
"epoch": 0.8268025078369906,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0004999765492622474,
|
|
"loss": 6.4093,
|
|
"mean_token_accuracy": 0.11808309555053711,
|
|
"num_tokens": 1855250.0,
|
|
"step": 1055
|
|
},
|
|
{
|
|
"entropy": 6.702435922622681,
|
|
"epoch": 0.8307210031347962,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.0004999720055749804,
|
|
"loss": 6.4262,
|
|
"mean_token_accuracy": 0.11298789232969284,
|
|
"num_tokens": 1863809.0,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"entropy": 6.728408670425415,
|
|
"epoch": 0.8346394984326019,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 0.0004999670598262619,
|
|
"loss": 6.407,
|
|
"mean_token_accuracy": 0.11927128955721855,
|
|
"num_tokens": 1872308.0,
|
|
"step": 1065
|
|
},
|
|
{
|
|
"entropy": 6.6889567375183105,
|
|
"epoch": 0.8385579937304075,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0004999617120249308,
|
|
"loss": 6.3375,
|
|
"mean_token_accuracy": 0.1220373697578907,
|
|
"num_tokens": 1881107.0,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"entropy": 6.789124965667725,
|
|
"epoch": 0.8424764890282131,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.0004999559621805445,
|
|
"loss": 6.4623,
|
|
"mean_token_accuracy": 0.11761095523834228,
|
|
"num_tokens": 1889961.0,
|
|
"step": 1075
|
|
},
|
|
{
|
|
"entropy": 6.689903688430786,
|
|
"epoch": 0.8463949843260188,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0004999498103033788,
|
|
"loss": 6.3445,
|
|
"mean_token_accuracy": 0.11976620554924011,
|
|
"num_tokens": 1898432.0,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"entropy": 6.650704622268677,
|
|
"epoch": 0.8503134796238244,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.0004999432564044284,
|
|
"loss": 6.3578,
|
|
"mean_token_accuracy": 0.12074211835861207,
|
|
"num_tokens": 1907236.0,
|
|
"step": 1085
|
|
},
|
|
{
|
|
"entropy": 6.6976752281188965,
|
|
"epoch": 0.85423197492163,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.0004999363004954058,
|
|
"loss": 6.3497,
|
|
"mean_token_accuracy": 0.1198844201862812,
|
|
"num_tokens": 1916388.0,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"entropy": 6.725913238525391,
|
|
"epoch": 0.8581504702194357,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0004999289425887425,
|
|
"loss": 6.456,
|
|
"mean_token_accuracy": 0.11865570768713951,
|
|
"num_tokens": 1926150.0,
|
|
"step": 1095
|
|
},
|
|
{
|
|
"entropy": 6.795915842056274,
|
|
"epoch": 0.8620689655172413,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0004999211826975884,
|
|
"loss": 6.477,
|
|
"mean_token_accuracy": 0.11915030851960182,
|
|
"num_tokens": 1935113.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"entropy": 6.656408739089966,
|
|
"epoch": 0.8659874608150471,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0004999130208358114,
|
|
"loss": 6.4021,
|
|
"mean_token_accuracy": 0.11451392620801926,
|
|
"num_tokens": 1944470.0,
|
|
"step": 1105
|
|
},
|
|
{
|
|
"entropy": 6.738942337036133,
|
|
"epoch": 0.8699059561128527,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.0004999044570179983,
|
|
"loss": 6.4227,
|
|
"mean_token_accuracy": 0.11082111597061158,
|
|
"num_tokens": 1954327.0,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"entropy": 6.575215244293213,
|
|
"epoch": 0.8738244514106583,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0004998954912594541,
|
|
"loss": 6.3189,
|
|
"mean_token_accuracy": 0.1276112236082554,
|
|
"num_tokens": 1962860.0,
|
|
"step": 1115
|
|
},
|
|
{
|
|
"entropy": 6.6690638065338135,
|
|
"epoch": 0.877742946708464,
|
|
"grad_norm": 1.2890625,
|
|
"learning_rate": 0.0004998861235762018,
|
|
"loss": 6.3139,
|
|
"mean_token_accuracy": 0.12368955090641975,
|
|
"num_tokens": 1971212.0,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"entropy": 6.662813901901245,
|
|
"epoch": 0.8816614420062696,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0004998763539849832,
|
|
"loss": 6.2525,
|
|
"mean_token_accuracy": 0.11725214570760727,
|
|
"num_tokens": 1980489.0,
|
|
"step": 1125
|
|
},
|
|
{
|
|
"entropy": 6.6482361316680905,
|
|
"epoch": 0.8855799373040752,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0004998661825032579,
|
|
"loss": 6.4021,
|
|
"mean_token_accuracy": 0.1187636524438858,
|
|
"num_tokens": 1989326.0,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"entropy": 6.722862958908081,
|
|
"epoch": 0.8894984326018809,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0004998556091492043,
|
|
"loss": 6.459,
|
|
"mean_token_accuracy": 0.11343813464045524,
|
|
"num_tokens": 1997888.0,
|
|
"step": 1135
|
|
},
|
|
{
|
|
"entropy": 6.655751085281372,
|
|
"epoch": 0.8934169278996865,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.0004998446339417184,
|
|
"loss": 6.3462,
|
|
"mean_token_accuracy": 0.12144071459770203,
|
|
"num_tokens": 2006815.0,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"entropy": 6.685620832443237,
|
|
"epoch": 0.8973354231974922,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.0004998332569004149,
|
|
"loss": 6.3128,
|
|
"mean_token_accuracy": 0.12164490222930908,
|
|
"num_tokens": 2015684.0,
|
|
"step": 1145
|
|
},
|
|
{
|
|
"entropy": 6.632381391525269,
|
|
"epoch": 0.9012539184952978,
|
|
"grad_norm": 1.2890625,
|
|
"learning_rate": 0.0004998214780456263,
|
|
"loss": 6.2757,
|
|
"mean_token_accuracy": 0.12474701330065727,
|
|
"num_tokens": 2024148.0,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"entropy": 6.573228025436402,
|
|
"epoch": 0.9051724137931034,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 0.0004998092973984033,
|
|
"loss": 6.3772,
|
|
"mean_token_accuracy": 0.12780468836426734,
|
|
"num_tokens": 2032287.0,
|
|
"step": 1155
|
|
},
|
|
{
|
|
"entropy": 6.785788917541504,
|
|
"epoch": 0.9090909090909091,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.0004997967149805147,
|
|
"loss": 6.4894,
|
|
"mean_token_accuracy": 0.11504697501659393,
|
|
"num_tokens": 2041663.0,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"entropy": 6.793114900588989,
|
|
"epoch": 0.9130094043887147,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0004997837308144474,
|
|
"loss": 6.4433,
|
|
"mean_token_accuracy": 0.11913444623351097,
|
|
"num_tokens": 2049756.0,
|
|
"step": 1165
|
|
},
|
|
{
|
|
"entropy": 6.65000319480896,
|
|
"epoch": 0.9169278996865203,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.0004997703449234062,
|
|
"loss": 6.454,
|
|
"mean_token_accuracy": 0.11475523188710213,
|
|
"num_tokens": 2058626.0,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"entropy": 6.735292673110962,
|
|
"epoch": 0.920846394984326,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0004997565573313139,
|
|
"loss": 6.366,
|
|
"mean_token_accuracy": 0.11419134736061096,
|
|
"num_tokens": 2067395.0,
|
|
"step": 1175
|
|
},
|
|
{
|
|
"entropy": 6.603668308258056,
|
|
"epoch": 0.9247648902821317,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.0004997423680628109,
|
|
"loss": 6.3787,
|
|
"mean_token_accuracy": 0.12259217575192452,
|
|
"num_tokens": 2075651.0,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"entropy": 6.688165903091431,
|
|
"epoch": 0.9286833855799373,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.000499727777143256,
|
|
"loss": 6.4043,
|
|
"mean_token_accuracy": 0.11774512678384781,
|
|
"num_tokens": 2084458.0,
|
|
"step": 1185
|
|
},
|
|
{
|
|
"entropy": 6.804714918136597,
|
|
"epoch": 0.932601880877743,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.0004997127845987255,
|
|
"loss": 6.4884,
|
|
"mean_token_accuracy": 0.11993765234947204,
|
|
"num_tokens": 2092413.0,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"entropy": 6.611707592010498,
|
|
"epoch": 0.9365203761755486,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.0004996973904560135,
|
|
"loss": 6.3697,
|
|
"mean_token_accuracy": 0.1176217883825302,
|
|
"num_tokens": 2100799.0,
|
|
"step": 1195
|
|
},
|
|
{
|
|
"entropy": 6.671212148666382,
|
|
"epoch": 0.9404388714733543,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 0.0004996815947426317,
|
|
"loss": 6.2887,
|
|
"mean_token_accuracy": 0.12218108549714088,
|
|
"num_tokens": 2109357.0,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"entropy": 6.574631977081299,
|
|
"epoch": 0.9443573667711599,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0004996653974868097,
|
|
"loss": 6.2794,
|
|
"mean_token_accuracy": 0.11755874156951904,
|
|
"num_tokens": 2117642.0,
|
|
"step": 1205
|
|
},
|
|
{
|
|
"entropy": 6.538684892654419,
|
|
"epoch": 0.9482758620689655,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0004996487987174946,
|
|
"loss": 6.3098,
|
|
"mean_token_accuracy": 0.12178566977381707,
|
|
"num_tokens": 2126512.0,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"entropy": 6.6026019096374515,
|
|
"epoch": 0.9521943573667712,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.0004996317984643512,
|
|
"loss": 6.3228,
|
|
"mean_token_accuracy": 0.12325456142425537,
|
|
"num_tokens": 2135559.0,
|
|
"step": 1215
|
|
},
|
|
{
|
|
"entropy": 6.681765937805176,
|
|
"epoch": 0.9561128526645768,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0004996143967577615,
|
|
"loss": 6.3066,
|
|
"mean_token_accuracy": 0.12556551843881608,
|
|
"num_tokens": 2144436.0,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"entropy": 6.583171606063843,
|
|
"epoch": 0.9600313479623824,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004995965936288254,
|
|
"loss": 6.2765,
|
|
"mean_token_accuracy": 0.12223444804549218,
|
|
"num_tokens": 2152625.0,
|
|
"step": 1225
|
|
},
|
|
{
|
|
"entropy": 6.714479923248291,
|
|
"epoch": 0.9639498432601881,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 0.0004995783891093597,
|
|
"loss": 6.4299,
|
|
"mean_token_accuracy": 0.1136121116578579,
|
|
"num_tokens": 2162405.0,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"entropy": 6.595222473144531,
|
|
"epoch": 0.9678683385579937,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.000499559783231899,
|
|
"loss": 6.3848,
|
|
"mean_token_accuracy": 0.11950541213154793,
|
|
"num_tokens": 2171376.0,
|
|
"step": 1235
|
|
},
|
|
{
|
|
"entropy": 6.623385429382324,
|
|
"epoch": 0.9717868338557993,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 0.0004995407760296952,
|
|
"loss": 6.2444,
|
|
"mean_token_accuracy": 0.125845268368721,
|
|
"num_tokens": 2180343.0,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"entropy": 6.456481504440307,
|
|
"epoch": 0.975705329153605,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0004995213675367169,
|
|
"loss": 6.2042,
|
|
"mean_token_accuracy": 0.129156381636858,
|
|
"num_tokens": 2189277.0,
|
|
"step": 1245
|
|
},
|
|
{
|
|
"entropy": 6.577784681320191,
|
|
"epoch": 0.9796238244514106,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.0004995015577876504,
|
|
"loss": 6.1903,
|
|
"mean_token_accuracy": 0.12381277903914452,
|
|
"num_tokens": 2197614.0,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"entropy": 6.570421743392944,
|
|
"epoch": 0.9835423197492164,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 0.000499481346817899,
|
|
"loss": 6.2956,
|
|
"mean_token_accuracy": 0.12848778143525125,
|
|
"num_tokens": 2205589.0,
|
|
"step": 1255
|
|
},
|
|
{
|
|
"entropy": 6.496778583526611,
|
|
"epoch": 0.987460815047022,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0004994607346635829,
|
|
"loss": 6.2882,
|
|
"mean_token_accuracy": 0.1286270134150982,
|
|
"num_tokens": 2214409.0,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"entropy": 6.625440311431885,
|
|
"epoch": 0.9913793103448276,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0004994397213615394,
|
|
"loss": 6.3662,
|
|
"mean_token_accuracy": 0.11870254352688789,
|
|
"num_tokens": 2223624.0,
|
|
"step": 1265
|
|
},
|
|
{
|
|
"entropy": 6.606072521209716,
|
|
"epoch": 0.9952978056426333,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0004994183069493225,
|
|
"loss": 6.3187,
|
|
"mean_token_accuracy": 0.11965643540024758,
|
|
"num_tokens": 2231878.0,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"entropy": 6.662061548233032,
|
|
"epoch": 0.9992163009404389,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0004993964914652034,
|
|
"loss": 6.2584,
|
|
"mean_token_accuracy": 0.1252451278269291,
|
|
"num_tokens": 2240684.0,
|
|
"step": 1275
|
|
},
|
|
{
|
|
"entropy": 6.574692583084106,
|
|
"epoch": 1.0031347962382444,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.0004993742749481699,
|
|
"loss": 6.1326,
|
|
"mean_token_accuracy": 0.13472574055194855,
|
|
"num_tokens": 2248776.0,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"entropy": 6.412195444107056,
|
|
"epoch": 1.0070532915360502,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0004993516574379265,
|
|
"loss": 5.9829,
|
|
"mean_token_accuracy": 0.13876268342137338,
|
|
"num_tokens": 2257637.0,
|
|
"step": 1285
|
|
},
|
|
{
|
|
"entropy": 6.596172285079956,
|
|
"epoch": 1.0109717868338557,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.0004993286389748943,
|
|
"loss": 6.183,
|
|
"mean_token_accuracy": 0.1223673589527607,
|
|
"num_tokens": 2266602.0,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"entropy": 6.4536356925964355,
|
|
"epoch": 1.0148902821316614,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.0004993052196002112,
|
|
"loss": 6.0342,
|
|
"mean_token_accuracy": 0.13276253119111062,
|
|
"num_tokens": 2275323.0,
|
|
"step": 1295
|
|
},
|
|
{
|
|
"entropy": 6.496698045730591,
|
|
"epoch": 1.0188087774294672,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.0004992813993557312,
|
|
"loss": 6.0591,
|
|
"mean_token_accuracy": 0.12783461958169937,
|
|
"num_tokens": 2284455.0,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"entropy": 6.431064796447754,
|
|
"epoch": 1.0227272727272727,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.000499257178284025,
|
|
"loss": 6.0945,
|
|
"mean_token_accuracy": 0.1290090449154377,
|
|
"num_tokens": 2293210.0,
|
|
"step": 1305
|
|
},
|
|
{
|
|
"entropy": 6.506451511383057,
|
|
"epoch": 1.0266457680250785,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.0004992325564283797,
|
|
"loss": 6.1456,
|
|
"mean_token_accuracy": 0.12814865931868552,
|
|
"num_tokens": 2302645.0,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"entropy": 6.554348945617676,
|
|
"epoch": 1.030564263322884,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.0004992075338327984,
|
|
"loss": 6.1251,
|
|
"mean_token_accuracy": 0.13093000575900077,
|
|
"num_tokens": 2310590.0,
|
|
"step": 1315
|
|
},
|
|
{
|
|
"entropy": 6.577063179016113,
|
|
"epoch": 1.0344827586206897,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0004991821105420006,
|
|
"loss": 6.16,
|
|
"mean_token_accuracy": 0.12589022517204285,
|
|
"num_tokens": 2319539.0,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"entropy": 6.535883665084839,
|
|
"epoch": 1.0384012539184952,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.0004991562866014219,
|
|
"loss": 6.1524,
|
|
"mean_token_accuracy": 0.126813293248415,
|
|
"num_tokens": 2328934.0,
|
|
"step": 1325
|
|
},
|
|
{
|
|
"entropy": 6.542779064178466,
|
|
"epoch": 1.042319749216301,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.0004991300620572138,
|
|
"loss": 6.1033,
|
|
"mean_token_accuracy": 0.13062299489974977,
|
|
"num_tokens": 2338123.0,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"entropy": 6.488020515441894,
|
|
"epoch": 1.0462382445141065,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.0004991034369562438,
|
|
"loss": 6.1583,
|
|
"mean_token_accuracy": 0.13101462051272392,
|
|
"num_tokens": 2346535.0,
|
|
"step": 1335
|
|
},
|
|
{
|
|
"entropy": 6.596034717559815,
|
|
"epoch": 1.0501567398119123,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004990764113460953,
|
|
"loss": 6.0472,
|
|
"mean_token_accuracy": 0.13994293436408042,
|
|
"num_tokens": 2355391.0,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"entropy": 6.387597560882568,
|
|
"epoch": 1.0540752351097178,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0004990489852750675,
|
|
"loss": 6.089,
|
|
"mean_token_accuracy": 0.13415274173021316,
|
|
"num_tokens": 2364325.0,
|
|
"step": 1345
|
|
},
|
|
{
|
|
"entropy": 6.458016014099121,
|
|
"epoch": 1.0579937304075235,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0004990211587921751,
|
|
"loss": 6.0354,
|
|
"mean_token_accuracy": 0.13103872239589692,
|
|
"num_tokens": 2373615.0,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"entropy": 6.614066696166992,
|
|
"epoch": 1.061912225705329,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004989929319471487,
|
|
"loss": 6.1897,
|
|
"mean_token_accuracy": 0.1241883710026741,
|
|
"num_tokens": 2382335.0,
|
|
"step": 1355
|
|
},
|
|
{
|
|
"entropy": 6.386125135421753,
|
|
"epoch": 1.0658307210031348,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0004989643047904341,
|
|
"loss": 6.0857,
|
|
"mean_token_accuracy": 0.12894076779484748,
|
|
"num_tokens": 2391448.0,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"entropy": 6.49520092010498,
|
|
"epoch": 1.0697492163009406,
|
|
"grad_norm": 1.2734375,
|
|
"learning_rate": 0.0004989352773731928,
|
|
"loss": 6.051,
|
|
"mean_token_accuracy": 0.13576387241482735,
|
|
"num_tokens": 2399697.0,
|
|
"step": 1365
|
|
},
|
|
{
|
|
"entropy": 6.450667142868042,
|
|
"epoch": 1.073667711598746,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 0.0004989058497473014,
|
|
"loss": 6.0102,
|
|
"mean_token_accuracy": 0.14315034449100494,
|
|
"num_tokens": 2408704.0,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"entropy": 6.297135686874389,
|
|
"epoch": 1.0775862068965518,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 0.0004988760219653518,
|
|
"loss": 5.9451,
|
|
"mean_token_accuracy": 0.13236987814307213,
|
|
"num_tokens": 2417035.0,
|
|
"step": 1375
|
|
},
|
|
{
|
|
"entropy": 6.4906352996826175,
|
|
"epoch": 1.0815047021943573,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.0004988457940806513,
|
|
"loss": 6.0613,
|
|
"mean_token_accuracy": 0.1321863353252411,
|
|
"num_tokens": 2426226.0,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"entropy": 6.316649675369263,
|
|
"epoch": 1.085423197492163,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0004988151661472218,
|
|
"loss": 6.0177,
|
|
"mean_token_accuracy": 0.1341511443257332,
|
|
"num_tokens": 2435226.0,
|
|
"step": 1385
|
|
},
|
|
{
|
|
"entropy": 6.468252182006836,
|
|
"epoch": 1.0893416927899686,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 0.0004987841382198006,
|
|
"loss": 6.1007,
|
|
"mean_token_accuracy": 0.13451739475131036,
|
|
"num_tokens": 2444344.0,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"entropy": 6.4722333431243895,
|
|
"epoch": 1.0932601880877744,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004987527103538394,
|
|
"loss": 6.0977,
|
|
"mean_token_accuracy": 0.1346490688621998,
|
|
"num_tokens": 2453965.0,
|
|
"step": 1395
|
|
},
|
|
{
|
|
"entropy": 6.517949199676513,
|
|
"epoch": 1.09717868338558,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.000498720882605505,
|
|
"loss": 6.1152,
|
|
"mean_token_accuracy": 0.1274041309952736,
|
|
"num_tokens": 2462929.0,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"entropy": 6.403267765045166,
|
|
"epoch": 1.1010971786833856,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.0004986886550316788,
|
|
"loss": 6.0646,
|
|
"mean_token_accuracy": 0.13348740786314012,
|
|
"num_tokens": 2471050.0,
|
|
"step": 1405
|
|
},
|
|
{
|
|
"entropy": 6.464239931106567,
|
|
"epoch": 1.1050156739811912,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0004986560276899565,
|
|
"loss": 6.0249,
|
|
"mean_token_accuracy": 0.12662593945860862,
|
|
"num_tokens": 2479963.0,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"entropy": 6.395254039764405,
|
|
"epoch": 1.108934169278997,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0004986230006386488,
|
|
"loss": 6.0502,
|
|
"mean_token_accuracy": 0.13782202005386351,
|
|
"num_tokens": 2488525.0,
|
|
"step": 1415
|
|
},
|
|
{
|
|
"entropy": 6.461355304718017,
|
|
"epoch": 1.1128526645768024,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0004985895739367802,
|
|
"loss": 6.1372,
|
|
"mean_token_accuracy": 0.13265417739748955,
|
|
"num_tokens": 2496892.0,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"entropy": 6.468255186080933,
|
|
"epoch": 1.1167711598746082,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.0004985557476440895,
|
|
"loss": 6.1618,
|
|
"mean_token_accuracy": 0.12906021475791932,
|
|
"num_tokens": 2506004.0,
|
|
"step": 1425
|
|
},
|
|
{
|
|
"entropy": 6.465244293212891,
|
|
"epoch": 1.1206896551724137,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.00049852152182103,
|
|
"loss": 6.0595,
|
|
"mean_token_accuracy": 0.13433735817670822,
|
|
"num_tokens": 2515068.0,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"entropy": 6.416254186630249,
|
|
"epoch": 1.1246081504702194,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.0004984868965287686,
|
|
"loss": 6.0067,
|
|
"mean_token_accuracy": 0.13896693661808968,
|
|
"num_tokens": 2523908.0,
|
|
"step": 1435
|
|
},
|
|
{
|
|
"entropy": 6.395039653778076,
|
|
"epoch": 1.1285266457680252,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.0004984518718291864,
|
|
"loss": 6.0145,
|
|
"mean_token_accuracy": 0.13567812219262124,
|
|
"num_tokens": 2532187.0,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"entropy": 6.39695463180542,
|
|
"epoch": 1.1324451410658307,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.0004984164477848783,
|
|
"loss": 5.9564,
|
|
"mean_token_accuracy": 0.14016151651740075,
|
|
"num_tokens": 2540615.0,
|
|
"step": 1445
|
|
},
|
|
{
|
|
"entropy": 6.3852643966674805,
|
|
"epoch": 1.1363636363636362,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0004983806244591528,
|
|
"loss": 6.0761,
|
|
"mean_token_accuracy": 0.13532559722661971,
|
|
"num_tokens": 2549414.0,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"entropy": 6.389466953277588,
|
|
"epoch": 1.140282131661442,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004983444019160318,
|
|
"loss": 6.0829,
|
|
"mean_token_accuracy": 0.13113784193992614,
|
|
"num_tokens": 2558496.0,
|
|
"step": 1455
|
|
},
|
|
{
|
|
"entropy": 6.3495019435882565,
|
|
"epoch": 1.1442006269592477,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.0004983077802202511,
|
|
"loss": 5.9905,
|
|
"mean_token_accuracy": 0.13659327030181884,
|
|
"num_tokens": 2566810.0,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"entropy": 6.481125736236573,
|
|
"epoch": 1.1481191222570533,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0004982707594372595,
|
|
"loss": 6.1072,
|
|
"mean_token_accuracy": 0.1357683263719082,
|
|
"num_tokens": 2576166.0,
|
|
"step": 1465
|
|
},
|
|
{
|
|
"entropy": 6.357020330429077,
|
|
"epoch": 1.152037617554859,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.000498233339633219,
|
|
"loss": 5.9931,
|
|
"mean_token_accuracy": 0.14374880045652388,
|
|
"num_tokens": 2583954.0,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"entropy": 6.3597229480743405,
|
|
"epoch": 1.1559561128526645,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.000498195520875005,
|
|
"loss": 6.0535,
|
|
"mean_token_accuracy": 0.13409585878252983,
|
|
"num_tokens": 2593216.0,
|
|
"step": 1475
|
|
},
|
|
{
|
|
"entropy": 6.390771007537841,
|
|
"epoch": 1.1598746081504703,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 0.0004981573032302059,
|
|
"loss": 6.0011,
|
|
"mean_token_accuracy": 0.13834288418293,
|
|
"num_tokens": 2601294.0,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"entropy": 6.356062984466552,
|
|
"epoch": 1.1637931034482758,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.0004981186867671225,
|
|
"loss": 6.0337,
|
|
"mean_token_accuracy": 0.1364253617823124,
|
|
"num_tokens": 2609932.0,
|
|
"step": 1485
|
|
},
|
|
{
|
|
"entropy": 6.325963640213013,
|
|
"epoch": 1.1677115987460815,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.0004980796715547691,
|
|
"loss": 5.9638,
|
|
"mean_token_accuracy": 0.1368013933300972,
|
|
"num_tokens": 2618287.0,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"entropy": 6.45850830078125,
|
|
"epoch": 1.171630094043887,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.0004980402576628717,
|
|
"loss": 6.0361,
|
|
"mean_token_accuracy": 0.1395344130694866,
|
|
"num_tokens": 2626778.0,
|
|
"step": 1495
|
|
},
|
|
{
|
|
"entropy": 6.269816255569458,
|
|
"epoch": 1.1755485893416928,
|
|
"grad_norm": 1.015625,
|
|
"learning_rate": 0.0004980004451618697,
|
|
"loss": 5.995,
|
|
"mean_token_accuracy": 0.13766313791275026,
|
|
"num_tokens": 2636166.0,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.1755485893416928,
|
|
"eval_entropy": 6.173720941987148,
|
|
"eval_loss": 6.259021759033203,
|
|
"eval_mean_token_accuracy": 0.13140278101660485,
|
|
"eval_num_tokens": 2636166.0,
|
|
"eval_runtime": 2.187,
|
|
"eval_samples_per_second": 1884.745,
|
|
"eval_steps_per_second": 235.936,
|
|
"step": 1500
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 12750,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 10,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 524951786373120.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|