1035 lines
27 KiB
JSON
1035 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.038181054560726964,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 10.69164161682129,
|
|
"epoch": 0.00038181054560726966,
|
|
"grad_norm": 12.625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 10.8218,
|
|
"mean_token_accuracy": 8.658008882775903e-05,
|
|
"num_tokens": 12361.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"entropy": 10.691694355010986,
|
|
"epoch": 0.0007636210912145393,
|
|
"grad_norm": 12.125,
|
|
"learning_rate": 4.5e-06,
|
|
"loss": 10.7677,
|
|
"mean_token_accuracy": 7.163324044086039e-05,
|
|
"num_tokens": 24868.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 10.691064453125,
|
|
"epoch": 0.001145431636821809,
|
|
"grad_norm": 8.625,
|
|
"learning_rate": 7e-06,
|
|
"loss": 10.5505,
|
|
"mean_token_accuracy": 0.025445129600120707,
|
|
"num_tokens": 37349.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"entropy": 10.675649642944336,
|
|
"epoch": 0.0015272421824290786,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 9.5e-06,
|
|
"loss": 10.2462,
|
|
"mean_token_accuracy": 0.04803365767002106,
|
|
"num_tokens": 50032.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 10.536279582977295,
|
|
"epoch": 0.0019090527280363483,
|
|
"grad_norm": 3.9375,
|
|
"learning_rate": 1.2e-05,
|
|
"loss": 9.9405,
|
|
"mean_token_accuracy": 0.05315057449042797,
|
|
"num_tokens": 61773.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"entropy": 10.429403972625732,
|
|
"epoch": 0.002290863273643618,
|
|
"grad_norm": 2.96875,
|
|
"learning_rate": 1.4500000000000002e-05,
|
|
"loss": 9.7807,
|
|
"mean_token_accuracy": 0.06437831856310368,
|
|
"num_tokens": 74017.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 10.479450607299805,
|
|
"epoch": 0.0026726738192508876,
|
|
"grad_norm": 2.859375,
|
|
"learning_rate": 1.7000000000000003e-05,
|
|
"loss": 9.7981,
|
|
"mean_token_accuracy": 0.05096841156482697,
|
|
"num_tokens": 87243.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"entropy": 10.484258651733398,
|
|
"epoch": 0.0030544843648581572,
|
|
"grad_norm": 2.5,
|
|
"learning_rate": 1.95e-05,
|
|
"loss": 9.7624,
|
|
"mean_token_accuracy": 0.05546987727284432,
|
|
"num_tokens": 99487.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 10.404310131072998,
|
|
"epoch": 0.003436294910465427,
|
|
"grad_norm": 2.171875,
|
|
"learning_rate": 2.2e-05,
|
|
"loss": 9.6868,
|
|
"mean_token_accuracy": 0.05369072034955025,
|
|
"num_tokens": 112020.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"entropy": 10.335991859436035,
|
|
"epoch": 0.0038181054560726966,
|
|
"grad_norm": 2.5625,
|
|
"learning_rate": 2.4500000000000003e-05,
|
|
"loss": 9.6778,
|
|
"mean_token_accuracy": 0.06317304000258446,
|
|
"num_tokens": 124703.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 10.376905536651611,
|
|
"epoch": 0.004199916001679967,
|
|
"grad_norm": 2.421875,
|
|
"learning_rate": 2.7e-05,
|
|
"loss": 9.6359,
|
|
"mean_token_accuracy": 0.06049793548882008,
|
|
"num_tokens": 137288.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"entropy": 10.288232707977295,
|
|
"epoch": 0.004581726547287236,
|
|
"grad_norm": 2.09375,
|
|
"learning_rate": 2.95e-05,
|
|
"loss": 9.5869,
|
|
"mean_token_accuracy": 0.06641441285610199,
|
|
"num_tokens": 150954.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 10.287444877624512,
|
|
"epoch": 0.004963537092894506,
|
|
"grad_norm": 2.171875,
|
|
"learning_rate": 3.2e-05,
|
|
"loss": 9.5108,
|
|
"mean_token_accuracy": 0.0745735965669155,
|
|
"num_tokens": 163545.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"entropy": 10.178703498840331,
|
|
"epoch": 0.005345347638501775,
|
|
"grad_norm": 2.703125,
|
|
"learning_rate": 3.4500000000000005e-05,
|
|
"loss": 9.3584,
|
|
"mean_token_accuracy": 0.08311713635921478,
|
|
"num_tokens": 175089.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 10.129299736022949,
|
|
"epoch": 0.005727158184109045,
|
|
"grad_norm": 1.8515625,
|
|
"learning_rate": 3.7e-05,
|
|
"loss": 9.3743,
|
|
"mean_token_accuracy": 0.08062887638807296,
|
|
"num_tokens": 187355.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"entropy": 10.13417091369629,
|
|
"epoch": 0.0061089687297163145,
|
|
"grad_norm": 1.9140625,
|
|
"learning_rate": 3.95e-05,
|
|
"loss": 9.3354,
|
|
"mean_token_accuracy": 0.08319172337651252,
|
|
"num_tokens": 200925.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 10.144186305999757,
|
|
"epoch": 0.006490779275323585,
|
|
"grad_norm": 1.875,
|
|
"learning_rate": 4.2000000000000004e-05,
|
|
"loss": 9.2585,
|
|
"mean_token_accuracy": 0.08622478619217873,
|
|
"num_tokens": 213393.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"entropy": 10.041219711303711,
|
|
"epoch": 0.006872589820930854,
|
|
"grad_norm": 2.203125,
|
|
"learning_rate": 4.45e-05,
|
|
"loss": 9.2302,
|
|
"mean_token_accuracy": 0.08239276111125945,
|
|
"num_tokens": 226261.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 10.069394493103028,
|
|
"epoch": 0.007254400366538124,
|
|
"grad_norm": 1.9375,
|
|
"learning_rate": 4.7000000000000004e-05,
|
|
"loss": 9.1435,
|
|
"mean_token_accuracy": 0.0902495726943016,
|
|
"num_tokens": 238277.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"entropy": 9.870719623565673,
|
|
"epoch": 0.007636210912145393,
|
|
"grad_norm": 2.109375,
|
|
"learning_rate": 4.9500000000000004e-05,
|
|
"loss": 8.9964,
|
|
"mean_token_accuracy": 0.10341855883598328,
|
|
"num_tokens": 250664.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 10.039793014526367,
|
|
"epoch": 0.008018021457752662,
|
|
"grad_norm": 1.78125,
|
|
"learning_rate": 5.2e-05,
|
|
"loss": 9.0284,
|
|
"mean_token_accuracy": 0.086080052703619,
|
|
"num_tokens": 262403.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"entropy": 9.880093574523926,
|
|
"epoch": 0.008399832003359933,
|
|
"grad_norm": 1.671875,
|
|
"learning_rate": 5.45e-05,
|
|
"loss": 8.9692,
|
|
"mean_token_accuracy": 0.09072378277778625,
|
|
"num_tokens": 276036.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 9.872721576690674,
|
|
"epoch": 0.008781642548967203,
|
|
"grad_norm": 1.5703125,
|
|
"learning_rate": 5.7e-05,
|
|
"loss": 8.8476,
|
|
"mean_token_accuracy": 0.0959131620824337,
|
|
"num_tokens": 287556.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"entropy": 9.759550285339355,
|
|
"epoch": 0.009163453094574472,
|
|
"grad_norm": 2.03125,
|
|
"learning_rate": 5.9499999999999996e-05,
|
|
"loss": 8.8335,
|
|
"mean_token_accuracy": 0.09087418392300606,
|
|
"num_tokens": 299285.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 9.744350814819336,
|
|
"epoch": 0.009545263640181741,
|
|
"grad_norm": 1.609375,
|
|
"learning_rate": 6.2e-05,
|
|
"loss": 8.7962,
|
|
"mean_token_accuracy": 0.08946920111775399,
|
|
"num_tokens": 312172.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"entropy": 9.738001823425293,
|
|
"epoch": 0.009927074185789012,
|
|
"grad_norm": 1.5390625,
|
|
"learning_rate": 6.450000000000001e-05,
|
|
"loss": 8.707,
|
|
"mean_token_accuracy": 0.09405562952160836,
|
|
"num_tokens": 325689.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 9.49710578918457,
|
|
"epoch": 0.010308884731396281,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 6.7e-05,
|
|
"loss": 8.645,
|
|
"mean_token_accuracy": 0.08777745962142944,
|
|
"num_tokens": 337893.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"entropy": 9.492643737792969,
|
|
"epoch": 0.01069069527700355,
|
|
"grad_norm": 1.6015625,
|
|
"learning_rate": 6.950000000000001e-05,
|
|
"loss": 8.576,
|
|
"mean_token_accuracy": 0.09355450496077537,
|
|
"num_tokens": 349733.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 9.370576190948487,
|
|
"epoch": 0.011072505822610821,
|
|
"grad_norm": 1.4375,
|
|
"learning_rate": 7.2e-05,
|
|
"loss": 8.5334,
|
|
"mean_token_accuracy": 0.0905280888080597,
|
|
"num_tokens": 362624.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"entropy": 9.226265716552735,
|
|
"epoch": 0.01145431636821809,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 7.45e-05,
|
|
"loss": 8.4527,
|
|
"mean_token_accuracy": 0.0901684120297432,
|
|
"num_tokens": 374564.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 9.136695861816406,
|
|
"epoch": 0.01183612691382536,
|
|
"grad_norm": 1.9296875,
|
|
"learning_rate": 7.7e-05,
|
|
"loss": 8.4298,
|
|
"mean_token_accuracy": 0.09475162327289581,
|
|
"num_tokens": 387813.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"entropy": 9.067316818237305,
|
|
"epoch": 0.012217937459432629,
|
|
"grad_norm": 1.421875,
|
|
"learning_rate": 7.950000000000001e-05,
|
|
"loss": 8.4477,
|
|
"mean_token_accuracy": 0.09096779003739357,
|
|
"num_tokens": 400037.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 8.901337909698487,
|
|
"epoch": 0.0125997480050399,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 8.2e-05,
|
|
"loss": 8.3423,
|
|
"mean_token_accuracy": 0.09573070406913757,
|
|
"num_tokens": 412753.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"entropy": 8.815200519561767,
|
|
"epoch": 0.01298155855064717,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 8.450000000000001e-05,
|
|
"loss": 8.2726,
|
|
"mean_token_accuracy": 0.09817055985331535,
|
|
"num_tokens": 425067.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 8.797827339172363,
|
|
"epoch": 0.013363369096254438,
|
|
"grad_norm": 1.2734375,
|
|
"learning_rate": 8.7e-05,
|
|
"loss": 8.2692,
|
|
"mean_token_accuracy": 0.09679182544350624,
|
|
"num_tokens": 437866.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"entropy": 8.725690650939942,
|
|
"epoch": 0.013745179641861708,
|
|
"grad_norm": 1.3125,
|
|
"learning_rate": 8.95e-05,
|
|
"loss": 8.2826,
|
|
"mean_token_accuracy": 0.09829293191432953,
|
|
"num_tokens": 449242.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 8.697874736785888,
|
|
"epoch": 0.014126990187468979,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 9.2e-05,
|
|
"loss": 8.3005,
|
|
"mean_token_accuracy": 0.09152775704860687,
|
|
"num_tokens": 461696.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"entropy": 8.68546266555786,
|
|
"epoch": 0.014508800733076248,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 9.45e-05,
|
|
"loss": 8.3056,
|
|
"mean_token_accuracy": 0.09098555892705917,
|
|
"num_tokens": 473875.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 8.710109996795655,
|
|
"epoch": 0.014890611278683517,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 9.7e-05,
|
|
"loss": 8.2518,
|
|
"mean_token_accuracy": 0.09681897088885308,
|
|
"num_tokens": 486124.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"entropy": 8.575210094451904,
|
|
"epoch": 0.015272421824290786,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 9.95e-05,
|
|
"loss": 8.1633,
|
|
"mean_token_accuracy": 0.09988361150026322,
|
|
"num_tokens": 498782.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 8.50928020477295,
|
|
"epoch": 0.015654232369898057,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.000102,
|
|
"loss": 8.2107,
|
|
"mean_token_accuracy": 0.09708656743168831,
|
|
"num_tokens": 511164.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"entropy": 8.605385684967041,
|
|
"epoch": 0.016036042915505325,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.00010449999999999999,
|
|
"loss": 8.1851,
|
|
"mean_token_accuracy": 0.09802236407995224,
|
|
"num_tokens": 522323.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 8.443045234680175,
|
|
"epoch": 0.016417853461112596,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.000107,
|
|
"loss": 8.1946,
|
|
"mean_token_accuracy": 0.09736634492874145,
|
|
"num_tokens": 535007.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"entropy": 8.477715015411377,
|
|
"epoch": 0.016799664006719867,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.0001095,
|
|
"loss": 8.1695,
|
|
"mean_token_accuracy": 0.09888660684227943,
|
|
"num_tokens": 547715.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 8.503206348419189,
|
|
"epoch": 0.017181474552327134,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.000112,
|
|
"loss": 8.1855,
|
|
"mean_token_accuracy": 0.09887165278196335,
|
|
"num_tokens": 559301.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"entropy": 8.340103912353516,
|
|
"epoch": 0.017563285097934405,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.0001145,
|
|
"loss": 8.0996,
|
|
"mean_token_accuracy": 0.10817644372582436,
|
|
"num_tokens": 570741.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 8.480874156951904,
|
|
"epoch": 0.017945095643541676,
|
|
"grad_norm": 1.3203125,
|
|
"learning_rate": 0.00011700000000000001,
|
|
"loss": 8.1212,
|
|
"mean_token_accuracy": 0.10553076937794685,
|
|
"num_tokens": 582263.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"entropy": 8.344198608398438,
|
|
"epoch": 0.018326906189148943,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.00011949999999999999,
|
|
"loss": 8.106,
|
|
"mean_token_accuracy": 0.10434014797210693,
|
|
"num_tokens": 593081.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 8.449238491058349,
|
|
"epoch": 0.018708716734756214,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.000122,
|
|
"loss": 8.2145,
|
|
"mean_token_accuracy": 0.09799340069293976,
|
|
"num_tokens": 605725.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"entropy": 8.343080520629883,
|
|
"epoch": 0.019090527280363482,
|
|
"grad_norm": 1.3125,
|
|
"learning_rate": 0.0001245,
|
|
"loss": 8.1114,
|
|
"mean_token_accuracy": 0.10767603665590286,
|
|
"num_tokens": 617740.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 8.363426685333252,
|
|
"epoch": 0.019472337825970753,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.000127,
|
|
"loss": 8.0848,
|
|
"mean_token_accuracy": 0.10610114932060241,
|
|
"num_tokens": 630186.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"entropy": 8.39581470489502,
|
|
"epoch": 0.019854148371578024,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.0001295,
|
|
"loss": 8.0586,
|
|
"mean_token_accuracy": 0.10069277137517929,
|
|
"num_tokens": 642304.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 8.270940399169922,
|
|
"epoch": 0.02023595891718529,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.000132,
|
|
"loss": 8.1002,
|
|
"mean_token_accuracy": 0.10599680617451668,
|
|
"num_tokens": 655029.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"entropy": 8.393358612060547,
|
|
"epoch": 0.020617769462792562,
|
|
"grad_norm": 1.453125,
|
|
"learning_rate": 0.00013450000000000002,
|
|
"loss": 8.1602,
|
|
"mean_token_accuracy": 0.10279513671994209,
|
|
"num_tokens": 667193.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 8.328470611572266,
|
|
"epoch": 0.020999580008399833,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00013700000000000002,
|
|
"loss": 8.0464,
|
|
"mean_token_accuracy": 0.11250988617539406,
|
|
"num_tokens": 681382.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"entropy": 8.321224308013916,
|
|
"epoch": 0.0213813905540071,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.0001395,
|
|
"loss": 8.155,
|
|
"mean_token_accuracy": 0.1049920804798603,
|
|
"num_tokens": 694280.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 8.34145975112915,
|
|
"epoch": 0.02176320109961437,
|
|
"grad_norm": 1.28125,
|
|
"learning_rate": 0.00014199999999999998,
|
|
"loss": 7.9959,
|
|
"mean_token_accuracy": 0.11552856862545013,
|
|
"num_tokens": 707682.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"entropy": 8.31129264831543,
|
|
"epoch": 0.022145011645221643,
|
|
"grad_norm": 1.2734375,
|
|
"learning_rate": 0.0001445,
|
|
"loss": 8.1054,
|
|
"mean_token_accuracy": 0.10616647526621818,
|
|
"num_tokens": 720237.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 8.282443714141845,
|
|
"epoch": 0.02252682219082891,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 0.000147,
|
|
"loss": 8.0555,
|
|
"mean_token_accuracy": 0.11427121162414551,
|
|
"num_tokens": 732816.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"entropy": 8.297718620300293,
|
|
"epoch": 0.02290863273643618,
|
|
"grad_norm": 1.453125,
|
|
"learning_rate": 0.0001495,
|
|
"loss": 8.0507,
|
|
"mean_token_accuracy": 0.10974556654691696,
|
|
"num_tokens": 744107.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 8.396364116668702,
|
|
"epoch": 0.02329044328204345,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.000152,
|
|
"loss": 8.0398,
|
|
"mean_token_accuracy": 0.10700830891728401,
|
|
"num_tokens": 757498.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"entropy": 8.064451313018798,
|
|
"epoch": 0.02367225382765072,
|
|
"grad_norm": 2.09375,
|
|
"learning_rate": 0.00015450000000000001,
|
|
"loss": 7.9454,
|
|
"mean_token_accuracy": 0.12401409074664116,
|
|
"num_tokens": 769429.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 8.283315277099609,
|
|
"epoch": 0.02405406437325799,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 0.000157,
|
|
"loss": 7.9851,
|
|
"mean_token_accuracy": 0.113057342171669,
|
|
"num_tokens": 781982.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"entropy": 8.28858814239502,
|
|
"epoch": 0.024435874918865258,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0001595,
|
|
"loss": 8.0389,
|
|
"mean_token_accuracy": 0.11170414686203003,
|
|
"num_tokens": 795936.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 8.194317626953126,
|
|
"epoch": 0.02481768546447253,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 0.000162,
|
|
"loss": 7.9885,
|
|
"mean_token_accuracy": 0.11449503526091576,
|
|
"num_tokens": 808509.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"entropy": 8.179093265533448,
|
|
"epoch": 0.0251994960100798,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 0.00016450000000000001,
|
|
"loss": 7.9899,
|
|
"mean_token_accuracy": 0.11678558364510536,
|
|
"num_tokens": 822457.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 8.29484577178955,
|
|
"epoch": 0.025581306555687067,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 0.00016700000000000002,
|
|
"loss": 7.9877,
|
|
"mean_token_accuracy": 0.11315491572022437,
|
|
"num_tokens": 835539.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"entropy": 8.240567207336426,
|
|
"epoch": 0.02596311710129434,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.00016950000000000003,
|
|
"loss": 7.9996,
|
|
"mean_token_accuracy": 0.11061925888061523,
|
|
"num_tokens": 848255.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 8.150354766845703,
|
|
"epoch": 0.026344927646901606,
|
|
"grad_norm": 0.9375,
|
|
"learning_rate": 0.00017199999999999998,
|
|
"loss": 7.9971,
|
|
"mean_token_accuracy": 0.11511600390076637,
|
|
"num_tokens": 860940.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"entropy": 8.30101490020752,
|
|
"epoch": 0.026726738192508877,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.00017449999999999999,
|
|
"loss": 8.0025,
|
|
"mean_token_accuracy": 0.11156911998987198,
|
|
"num_tokens": 873757.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 8.16147975921631,
|
|
"epoch": 0.027108548738116148,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.000177,
|
|
"loss": 7.9879,
|
|
"mean_token_accuracy": 0.11351457163691521,
|
|
"num_tokens": 885751.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"entropy": 8.138264703750611,
|
|
"epoch": 0.027490359283723415,
|
|
"grad_norm": 1.4609375,
|
|
"learning_rate": 0.0001795,
|
|
"loss": 7.8642,
|
|
"mean_token_accuracy": 0.11878235787153243,
|
|
"num_tokens": 899203.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 8.131235504150391,
|
|
"epoch": 0.027872169829330686,
|
|
"grad_norm": 1.3984375,
|
|
"learning_rate": 0.000182,
|
|
"loss": 7.8713,
|
|
"mean_token_accuracy": 0.12657914236187934,
|
|
"num_tokens": 911384.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"entropy": 8.151749897003175,
|
|
"epoch": 0.028253980374937957,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 0.0001845,
|
|
"loss": 7.9322,
|
|
"mean_token_accuracy": 0.11955902799963951,
|
|
"num_tokens": 923558.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 8.194352340698241,
|
|
"epoch": 0.028635790920545225,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.000187,
|
|
"loss": 7.9254,
|
|
"mean_token_accuracy": 0.11569129005074501,
|
|
"num_tokens": 936309.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"entropy": 8.09564666748047,
|
|
"epoch": 0.029017601466152496,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.0001895,
|
|
"loss": 7.9299,
|
|
"mean_token_accuracy": 0.11621968820691109,
|
|
"num_tokens": 947882.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 8.115512371063232,
|
|
"epoch": 0.029399412011759767,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.000192,
|
|
"loss": 7.9385,
|
|
"mean_token_accuracy": 0.11436433717608452,
|
|
"num_tokens": 960242.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"entropy": 8.190027332305908,
|
|
"epoch": 0.029781222557367034,
|
|
"grad_norm": 0.98046875,
|
|
"learning_rate": 0.0001945,
|
|
"loss": 7.9643,
|
|
"mean_token_accuracy": 0.11476908028125762,
|
|
"num_tokens": 973814.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 8.06707820892334,
|
|
"epoch": 0.030163033102974305,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.00019700000000000002,
|
|
"loss": 7.8704,
|
|
"mean_token_accuracy": 0.12641484588384627,
|
|
"num_tokens": 985935.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"entropy": 8.11848578453064,
|
|
"epoch": 0.030544843648581572,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00019950000000000002,
|
|
"loss": 7.9411,
|
|
"mean_token_accuracy": 0.11482023000717163,
|
|
"num_tokens": 997618.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 8.1826322555542,
|
|
"epoch": 0.030926654194188843,
|
|
"grad_norm": 0.91015625,
|
|
"learning_rate": 0.000202,
|
|
"loss": 7.8844,
|
|
"mean_token_accuracy": 0.12198737263679504,
|
|
"num_tokens": 1009745.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"entropy": 8.074276161193847,
|
|
"epoch": 0.031308464739796114,
|
|
"grad_norm": 1.46875,
|
|
"learning_rate": 0.00020449999999999998,
|
|
"loss": 7.8455,
|
|
"mean_token_accuracy": 0.12325679659843444,
|
|
"num_tokens": 1022401.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 8.10068097114563,
|
|
"epoch": 0.031690275285403385,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.000207,
|
|
"loss": 7.8965,
|
|
"mean_token_accuracy": 0.11033826768398285,
|
|
"num_tokens": 1034844.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"entropy": 7.962525510787964,
|
|
"epoch": 0.03207208583101065,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.0002095,
|
|
"loss": 7.8646,
|
|
"mean_token_accuracy": 0.12236305102705955,
|
|
"num_tokens": 1046733.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 8.19951171875,
|
|
"epoch": 0.03245389637661792,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.000212,
|
|
"loss": 7.9132,
|
|
"mean_token_accuracy": 0.11312553584575653,
|
|
"num_tokens": 1060150.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"entropy": 8.00361738204956,
|
|
"epoch": 0.03283570692222519,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 0.0002145,
|
|
"loss": 7.8368,
|
|
"mean_token_accuracy": 0.1230610378086567,
|
|
"num_tokens": 1072178.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 8.090557527542114,
|
|
"epoch": 0.03321751746783246,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.00021700000000000002,
|
|
"loss": 7.8982,
|
|
"mean_token_accuracy": 0.11123135685920715,
|
|
"num_tokens": 1084404.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"entropy": 7.951789474487304,
|
|
"epoch": 0.03359932801343973,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.0002195,
|
|
"loss": 7.8519,
|
|
"mean_token_accuracy": 0.11904852241277694,
|
|
"num_tokens": 1096211.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 8.07854413986206,
|
|
"epoch": 0.033981138559047004,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.000222,
|
|
"loss": 7.8994,
|
|
"mean_token_accuracy": 0.12089689522981643,
|
|
"num_tokens": 1109118.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"entropy": 8.024729442596435,
|
|
"epoch": 0.03436294910465427,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 0.0002245,
|
|
"loss": 7.8407,
|
|
"mean_token_accuracy": 0.11579521223902703,
|
|
"num_tokens": 1121785.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 8.080478191375732,
|
|
"epoch": 0.03474475965026154,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 0.00022700000000000002,
|
|
"loss": 7.8267,
|
|
"mean_token_accuracy": 0.11973192021250725,
|
|
"num_tokens": 1134888.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"entropy": 7.96352367401123,
|
|
"epoch": 0.03512657019586881,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 0.00022950000000000002,
|
|
"loss": 7.7474,
|
|
"mean_token_accuracy": 0.1206622526049614,
|
|
"num_tokens": 1146777.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 8.051364040374756,
|
|
"epoch": 0.03550838074147608,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.00023200000000000003,
|
|
"loss": 7.8404,
|
|
"mean_token_accuracy": 0.11428562551736832,
|
|
"num_tokens": 1159489.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"entropy": 7.962832498550415,
|
|
"epoch": 0.03589019128708335,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.00023449999999999998,
|
|
"loss": 7.7844,
|
|
"mean_token_accuracy": 0.12907169461250306,
|
|
"num_tokens": 1171746.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 7.957472324371338,
|
|
"epoch": 0.036272001832690616,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.000237,
|
|
"loss": 7.8531,
|
|
"mean_token_accuracy": 0.1179780401289463,
|
|
"num_tokens": 1184965.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"entropy": 8.081069564819336,
|
|
"epoch": 0.03665381237829789,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.0002395,
|
|
"loss": 7.8668,
|
|
"mean_token_accuracy": 0.1181311696767807,
|
|
"num_tokens": 1198080.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 7.988467025756836,
|
|
"epoch": 0.03703562292390516,
|
|
"grad_norm": 1.0,
|
|
"learning_rate": 0.000242,
|
|
"loss": 7.8406,
|
|
"mean_token_accuracy": 0.12035353481769562,
|
|
"num_tokens": 1210484.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"entropy": 7.958967494964599,
|
|
"epoch": 0.03741743346951243,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0002445,
|
|
"loss": 7.7806,
|
|
"mean_token_accuracy": 0.12781597301363945,
|
|
"num_tokens": 1223524.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 7.937625741958618,
|
|
"epoch": 0.0377992440151197,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.000247,
|
|
"loss": 7.7644,
|
|
"mean_token_accuracy": 0.12598755583167076,
|
|
"num_tokens": 1234946.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"entropy": 7.959400177001953,
|
|
"epoch": 0.038181054560726964,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.0002495,
|
|
"loss": 7.7763,
|
|
"mean_token_accuracy": 0.12367670983076096,
|
|
"num_tokens": 1247187.0,
|
|
"step": 500
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 4000,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 1876297254912000.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|