1035 lines
27 KiB
JSON
1035 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.010868620119989566,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 10.692033958435058,
|
|
"epoch": 0.00010868620119989566,
|
|
"grad_norm": 14.125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 10.7755,
|
|
"mean_token_accuracy": 0.0,
|
|
"num_tokens": 7936.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"entropy": 10.691999912261963,
|
|
"epoch": 0.00021737240239979132,
|
|
"grad_norm": 15.1875,
|
|
"learning_rate": 4.5e-06,
|
|
"loss": 10.6915,
|
|
"mean_token_accuracy": 0.0,
|
|
"num_tokens": 15561.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 10.690576362609864,
|
|
"epoch": 0.000326058603599687,
|
|
"grad_norm": 10.0,
|
|
"learning_rate": 7e-06,
|
|
"loss": 10.4237,
|
|
"mean_token_accuracy": 0.03714140746742487,
|
|
"num_tokens": 23101.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"entropy": 10.662552452087402,
|
|
"epoch": 0.00043474480479958265,
|
|
"grad_norm": 6.4375,
|
|
"learning_rate": 9.5e-06,
|
|
"loss": 10.13,
|
|
"mean_token_accuracy": 0.053733503073453905,
|
|
"num_tokens": 31122.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 10.53134651184082,
|
|
"epoch": 0.0005434310059994783,
|
|
"grad_norm": 4.21875,
|
|
"learning_rate": 1.2e-05,
|
|
"loss": 9.8666,
|
|
"mean_token_accuracy": 0.049097494408488276,
|
|
"num_tokens": 39424.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"entropy": 10.499347591400147,
|
|
"epoch": 0.000652117207199374,
|
|
"grad_norm": 3.40625,
|
|
"learning_rate": 1.4500000000000002e-05,
|
|
"loss": 9.6573,
|
|
"mean_token_accuracy": 0.05333031602203846,
|
|
"num_tokens": 46113.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 10.45414752960205,
|
|
"epoch": 0.0007608034083992696,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 1.7000000000000003e-05,
|
|
"loss": 9.5644,
|
|
"mean_token_accuracy": 0.050568538904190066,
|
|
"num_tokens": 53459.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"entropy": 10.488454818725586,
|
|
"epoch": 0.0008694896095991653,
|
|
"grad_norm": 2.875,
|
|
"learning_rate": 1.95e-05,
|
|
"loss": 9.5746,
|
|
"mean_token_accuracy": 0.054808919131755826,
|
|
"num_tokens": 61325.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 10.44335298538208,
|
|
"epoch": 0.0009781758107990609,
|
|
"grad_norm": 2.578125,
|
|
"learning_rate": 2.2e-05,
|
|
"loss": 9.5243,
|
|
"mean_token_accuracy": 0.05896916501224041,
|
|
"num_tokens": 68741.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"entropy": 10.389917469024658,
|
|
"epoch": 0.0010868620119989566,
|
|
"grad_norm": 2.703125,
|
|
"learning_rate": 2.4500000000000003e-05,
|
|
"loss": 9.4498,
|
|
"mean_token_accuracy": 0.07500801645219327,
|
|
"num_tokens": 76827.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 10.32972011566162,
|
|
"epoch": 0.0011955482131988523,
|
|
"grad_norm": 2.28125,
|
|
"learning_rate": 2.7e-05,
|
|
"loss": 9.4108,
|
|
"mean_token_accuracy": 0.06764259599149228,
|
|
"num_tokens": 84543.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"entropy": 10.223662376403809,
|
|
"epoch": 0.001304234414398748,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 2.95e-05,
|
|
"loss": 9.3763,
|
|
"mean_token_accuracy": 0.06775642447173595,
|
|
"num_tokens": 92505.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 10.253492069244384,
|
|
"epoch": 0.0014129206155986437,
|
|
"grad_norm": 2.046875,
|
|
"learning_rate": 3.2e-05,
|
|
"loss": 9.292,
|
|
"mean_token_accuracy": 0.07278058938682079,
|
|
"num_tokens": 100004.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"entropy": 10.179506492614745,
|
|
"epoch": 0.0015216068167985392,
|
|
"grad_norm": 2.125,
|
|
"learning_rate": 3.4500000000000005e-05,
|
|
"loss": 9.1464,
|
|
"mean_token_accuracy": 0.07609819024801254,
|
|
"num_tokens": 106865.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 10.167201709747314,
|
|
"epoch": 0.0016302930179984349,
|
|
"grad_norm": 1.828125,
|
|
"learning_rate": 3.7e-05,
|
|
"loss": 9.184,
|
|
"mean_token_accuracy": 0.07288527600467205,
|
|
"num_tokens": 115745.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"entropy": 10.253772735595703,
|
|
"epoch": 0.0017389792191983306,
|
|
"grad_norm": 1.9609375,
|
|
"learning_rate": 3.95e-05,
|
|
"loss": 9.1019,
|
|
"mean_token_accuracy": 0.07272576205432416,
|
|
"num_tokens": 124724.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 10.141531181335449,
|
|
"epoch": 0.0018476654203982263,
|
|
"grad_norm": 1.921875,
|
|
"learning_rate": 4.2000000000000004e-05,
|
|
"loss": 8.9728,
|
|
"mean_token_accuracy": 0.07559517920017242,
|
|
"num_tokens": 131933.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"entropy": 10.05212688446045,
|
|
"epoch": 0.0019563516215981218,
|
|
"grad_norm": 2.125,
|
|
"learning_rate": 4.45e-05,
|
|
"loss": 8.9554,
|
|
"mean_token_accuracy": 0.07529733926057816,
|
|
"num_tokens": 139726.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 10.118945980072022,
|
|
"epoch": 0.0020650378227980177,
|
|
"grad_norm": 1.9453125,
|
|
"learning_rate": 4.7000000000000004e-05,
|
|
"loss": 8.8937,
|
|
"mean_token_accuracy": 0.07433236017823219,
|
|
"num_tokens": 147276.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"entropy": 10.070981788635255,
|
|
"epoch": 0.002173724023997913,
|
|
"grad_norm": 1.671875,
|
|
"learning_rate": 4.9500000000000004e-05,
|
|
"loss": 8.8032,
|
|
"mean_token_accuracy": 0.07382776252925397,
|
|
"num_tokens": 155449.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 9.931837844848634,
|
|
"epoch": 0.002282410225197809,
|
|
"grad_norm": 1.828125,
|
|
"learning_rate": 5.2e-05,
|
|
"loss": 8.6788,
|
|
"mean_token_accuracy": 0.06880230605602264,
|
|
"num_tokens": 163547.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"entropy": 9.92315607070923,
|
|
"epoch": 0.0023910964263977046,
|
|
"grad_norm": 1.71875,
|
|
"learning_rate": 5.45e-05,
|
|
"loss": 8.5975,
|
|
"mean_token_accuracy": 0.0769424244761467,
|
|
"num_tokens": 171777.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 9.855748653411865,
|
|
"epoch": 0.0024997826275976,
|
|
"grad_norm": 1.671875,
|
|
"learning_rate": 5.7e-05,
|
|
"loss": 8.4659,
|
|
"mean_token_accuracy": 0.0743311032652855,
|
|
"num_tokens": 179337.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"entropy": 9.745928478240966,
|
|
"epoch": 0.002608468828797496,
|
|
"grad_norm": 1.796875,
|
|
"learning_rate": 5.9499999999999996e-05,
|
|
"loss": 8.3905,
|
|
"mean_token_accuracy": 0.07685245871543885,
|
|
"num_tokens": 186939.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 9.60959987640381,
|
|
"epoch": 0.0027171550299973915,
|
|
"grad_norm": 1.3125,
|
|
"learning_rate": 6.2e-05,
|
|
"loss": 8.2612,
|
|
"mean_token_accuracy": 0.07991581708192826,
|
|
"num_tokens": 194755.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"entropy": 9.507084274291993,
|
|
"epoch": 0.0028258412311972874,
|
|
"grad_norm": 1.6015625,
|
|
"learning_rate": 6.450000000000001e-05,
|
|
"loss": 8.1596,
|
|
"mean_token_accuracy": 0.07951611280441284,
|
|
"num_tokens": 202610.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 9.229420471191407,
|
|
"epoch": 0.002934527432397183,
|
|
"grad_norm": 1.546875,
|
|
"learning_rate": 6.7e-05,
|
|
"loss": 8.0589,
|
|
"mean_token_accuracy": 0.07813627794384956,
|
|
"num_tokens": 211305.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"entropy": 9.083441066741944,
|
|
"epoch": 0.0030432136335970784,
|
|
"grad_norm": 1.4296875,
|
|
"learning_rate": 6.950000000000001e-05,
|
|
"loss": 7.9516,
|
|
"mean_token_accuracy": 0.07972251251339912,
|
|
"num_tokens": 218447.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 8.977557182312012,
|
|
"epoch": 0.0031518998347969743,
|
|
"grad_norm": 1.328125,
|
|
"learning_rate": 7.2e-05,
|
|
"loss": 7.9735,
|
|
"mean_token_accuracy": 0.07708753384649754,
|
|
"num_tokens": 226108.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"entropy": 8.854303741455078,
|
|
"epoch": 0.0032605860359968698,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 7.45e-05,
|
|
"loss": 7.9018,
|
|
"mean_token_accuracy": 0.0749433733522892,
|
|
"num_tokens": 233213.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 8.737026977539063,
|
|
"epoch": 0.0033692722371967657,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 7.7e-05,
|
|
"loss": 7.9657,
|
|
"mean_token_accuracy": 0.07061286456882954,
|
|
"num_tokens": 241483.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"entropy": 8.660515594482423,
|
|
"epoch": 0.003477958438396661,
|
|
"grad_norm": 1.28125,
|
|
"learning_rate": 7.950000000000001e-05,
|
|
"loss": 7.8216,
|
|
"mean_token_accuracy": 0.08022509105503559,
|
|
"num_tokens": 249537.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 8.48324089050293,
|
|
"epoch": 0.0035866446395965567,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 8.2e-05,
|
|
"loss": 7.7574,
|
|
"mean_token_accuracy": 0.08265799134969712,
|
|
"num_tokens": 256775.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"entropy": 8.382307147979736,
|
|
"epoch": 0.0036953308407964526,
|
|
"grad_norm": 1.40625,
|
|
"learning_rate": 8.450000000000001e-05,
|
|
"loss": 7.7628,
|
|
"mean_token_accuracy": 0.08266141265630722,
|
|
"num_tokens": 263931.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 8.312588787078857,
|
|
"epoch": 0.003804017041996348,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 8.7e-05,
|
|
"loss": 7.7557,
|
|
"mean_token_accuracy": 0.08323334604501724,
|
|
"num_tokens": 272293.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"entropy": 8.368490505218507,
|
|
"epoch": 0.0039127032431962435,
|
|
"grad_norm": 1.390625,
|
|
"learning_rate": 8.95e-05,
|
|
"loss": 7.7964,
|
|
"mean_token_accuracy": 0.07600408792495728,
|
|
"num_tokens": 280599.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 8.208444499969483,
|
|
"epoch": 0.00402138944439614,
|
|
"grad_norm": 1.4921875,
|
|
"learning_rate": 9.2e-05,
|
|
"loss": 7.7604,
|
|
"mean_token_accuracy": 0.07719066366553307,
|
|
"num_tokens": 287961.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"entropy": 8.142381763458252,
|
|
"epoch": 0.004130075645596035,
|
|
"grad_norm": 1.8671875,
|
|
"learning_rate": 9.45e-05,
|
|
"loss": 7.684,
|
|
"mean_token_accuracy": 0.08424306884407998,
|
|
"num_tokens": 297280.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 8.162713718414306,
|
|
"epoch": 0.004238761846795931,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 9.7e-05,
|
|
"loss": 7.7526,
|
|
"mean_token_accuracy": 0.08235807195305825,
|
|
"num_tokens": 304699.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"entropy": 8.149117755889893,
|
|
"epoch": 0.004347448047995826,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 9.95e-05,
|
|
"loss": 7.6864,
|
|
"mean_token_accuracy": 0.08296664133667946,
|
|
"num_tokens": 313026.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 8.115727186203003,
|
|
"epoch": 0.004456134249195722,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.000102,
|
|
"loss": 7.7092,
|
|
"mean_token_accuracy": 0.08166395723819733,
|
|
"num_tokens": 320738.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"entropy": 8.076230096817017,
|
|
"epoch": 0.004564820450395618,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.00010449999999999999,
|
|
"loss": 7.6861,
|
|
"mean_token_accuracy": 0.07836256250739097,
|
|
"num_tokens": 328740.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 8.025853300094605,
|
|
"epoch": 0.004673506651595514,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 0.000107,
|
|
"loss": 7.6627,
|
|
"mean_token_accuracy": 0.08186743408441544,
|
|
"num_tokens": 336819.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"entropy": 8.132151794433593,
|
|
"epoch": 0.004782192852795409,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 0.0001095,
|
|
"loss": 7.5969,
|
|
"mean_token_accuracy": 0.08840056881308556,
|
|
"num_tokens": 343978.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 8.009299278259277,
|
|
"epoch": 0.004890879053995305,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.000112,
|
|
"loss": 7.6897,
|
|
"mean_token_accuracy": 0.07886088080704212,
|
|
"num_tokens": 353783.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"entropy": 7.969748306274414,
|
|
"epoch": 0.0049995652551952,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 0.0001145,
|
|
"loss": 7.5909,
|
|
"mean_token_accuracy": 0.08582306802272796,
|
|
"num_tokens": 361371.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 7.9828167915344235,
|
|
"epoch": 0.0051082514563950965,
|
|
"grad_norm": 1.3984375,
|
|
"learning_rate": 0.00011700000000000001,
|
|
"loss": 7.6185,
|
|
"mean_token_accuracy": 0.07745145447552204,
|
|
"num_tokens": 367717.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"entropy": 8.051587057113647,
|
|
"epoch": 0.005216937657594992,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 0.00011949999999999999,
|
|
"loss": 7.6177,
|
|
"mean_token_accuracy": 0.08022010251879692,
|
|
"num_tokens": 375529.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 7.905487108230591,
|
|
"epoch": 0.0053256238587948874,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.000122,
|
|
"loss": 7.5073,
|
|
"mean_token_accuracy": 0.08242316544055939,
|
|
"num_tokens": 383031.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"entropy": 7.966064023971557,
|
|
"epoch": 0.005434310059994783,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.0001245,
|
|
"loss": 7.6472,
|
|
"mean_token_accuracy": 0.0840654157102108,
|
|
"num_tokens": 391914.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 7.939682960510254,
|
|
"epoch": 0.005542996261194678,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 0.000127,
|
|
"loss": 7.6111,
|
|
"mean_token_accuracy": 0.08445862084627151,
|
|
"num_tokens": 399447.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"entropy": 7.95916223526001,
|
|
"epoch": 0.005651682462394575,
|
|
"grad_norm": 1.5078125,
|
|
"learning_rate": 0.0001295,
|
|
"loss": 7.6053,
|
|
"mean_token_accuracy": 0.08623345196247101,
|
|
"num_tokens": 407636.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 7.863192081451416,
|
|
"epoch": 0.00576036866359447,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 0.000132,
|
|
"loss": 7.5498,
|
|
"mean_token_accuracy": 0.08252564668655396,
|
|
"num_tokens": 416650.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"entropy": 7.940821886062622,
|
|
"epoch": 0.005869054864794366,
|
|
"grad_norm": 1.71875,
|
|
"learning_rate": 0.00013450000000000002,
|
|
"loss": 7.4851,
|
|
"mean_token_accuracy": 0.08535419404506683,
|
|
"num_tokens": 424619.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 7.923966264724731,
|
|
"epoch": 0.005977741065994261,
|
|
"grad_norm": 1.5546875,
|
|
"learning_rate": 0.00013700000000000002,
|
|
"loss": 7.5731,
|
|
"mean_token_accuracy": 0.08694756105542183,
|
|
"num_tokens": 432854.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"entropy": 7.861378288269043,
|
|
"epoch": 0.006086427267194157,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 0.0001395,
|
|
"loss": 7.5646,
|
|
"mean_token_accuracy": 0.08049703910946845,
|
|
"num_tokens": 440472.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 7.979065895080566,
|
|
"epoch": 0.006195113468394053,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.00014199999999999998,
|
|
"loss": 7.4921,
|
|
"mean_token_accuracy": 0.08637023121118545,
|
|
"num_tokens": 447843.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"entropy": 7.926018571853637,
|
|
"epoch": 0.0063037996695939486,
|
|
"grad_norm": 1.4921875,
|
|
"learning_rate": 0.0001445,
|
|
"loss": 7.6005,
|
|
"mean_token_accuracy": 0.08329573571681977,
|
|
"num_tokens": 455272.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 7.811395597457886,
|
|
"epoch": 0.006412485870793844,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 0.000147,
|
|
"loss": 7.4287,
|
|
"mean_token_accuracy": 0.08994799107313156,
|
|
"num_tokens": 462190.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"entropy": 7.937566328048706,
|
|
"epoch": 0.0065211720719937395,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 0.0001495,
|
|
"loss": 7.5291,
|
|
"mean_token_accuracy": 0.08383448868989944,
|
|
"num_tokens": 470562.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 7.770080280303955,
|
|
"epoch": 0.006629858273193635,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 0.000152,
|
|
"loss": 7.433,
|
|
"mean_token_accuracy": 0.09061657935380936,
|
|
"num_tokens": 478523.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"entropy": 7.844554090499878,
|
|
"epoch": 0.006738544474393531,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 0.00015450000000000001,
|
|
"loss": 7.517,
|
|
"mean_token_accuracy": 0.08662905022501946,
|
|
"num_tokens": 486759.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 7.828848934173584,
|
|
"epoch": 0.006847230675593427,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.000157,
|
|
"loss": 7.5283,
|
|
"mean_token_accuracy": 0.08165703043341636,
|
|
"num_tokens": 495168.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"entropy": 7.823043441772461,
|
|
"epoch": 0.006955916876793322,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.0001595,
|
|
"loss": 7.4207,
|
|
"mean_token_accuracy": 0.08956636488437653,
|
|
"num_tokens": 503490.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 7.771451330184936,
|
|
"epoch": 0.007064603077993218,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 0.000162,
|
|
"loss": 7.5539,
|
|
"mean_token_accuracy": 0.08244621530175208,
|
|
"num_tokens": 512522.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"entropy": 7.877453756332398,
|
|
"epoch": 0.007173289279193113,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 0.00016450000000000001,
|
|
"loss": 7.4913,
|
|
"mean_token_accuracy": 0.08120876513421535,
|
|
"num_tokens": 521100.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 7.771992826461792,
|
|
"epoch": 0.00728197548039301,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 0.00016700000000000002,
|
|
"loss": 7.396,
|
|
"mean_token_accuracy": 0.0948154278099537,
|
|
"num_tokens": 528027.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"entropy": 7.7505473613739015,
|
|
"epoch": 0.007390661681592905,
|
|
"grad_norm": 1.390625,
|
|
"learning_rate": 0.00016950000000000003,
|
|
"loss": 7.3994,
|
|
"mean_token_accuracy": 0.08797064200043678,
|
|
"num_tokens": 535541.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 7.820106267929077,
|
|
"epoch": 0.007499347882792801,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.00017199999999999998,
|
|
"loss": 7.4177,
|
|
"mean_token_accuracy": 0.08866829052567482,
|
|
"num_tokens": 543253.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"entropy": 7.8131262302398685,
|
|
"epoch": 0.007608034083992696,
|
|
"grad_norm": 1.515625,
|
|
"learning_rate": 0.00017449999999999999,
|
|
"loss": 7.4312,
|
|
"mean_token_accuracy": 0.08869538456201553,
|
|
"num_tokens": 550414.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 7.678452491760254,
|
|
"epoch": 0.007716720285192592,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 0.000177,
|
|
"loss": 7.4866,
|
|
"mean_token_accuracy": 0.08567069470882416,
|
|
"num_tokens": 558554.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"entropy": 7.860786962509155,
|
|
"epoch": 0.007825406486392487,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.0001795,
|
|
"loss": 7.4562,
|
|
"mean_token_accuracy": 0.0866042397916317,
|
|
"num_tokens": 566297.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 7.691314792633056,
|
|
"epoch": 0.007934092687592383,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 0.000182,
|
|
"loss": 7.399,
|
|
"mean_token_accuracy": 0.09093262627720833,
|
|
"num_tokens": 574600.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"entropy": 7.761751461029053,
|
|
"epoch": 0.00804277888879228,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.0001845,
|
|
"loss": 7.4278,
|
|
"mean_token_accuracy": 0.08792822286486626,
|
|
"num_tokens": 583349.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 7.841218614578247,
|
|
"epoch": 0.008151465089992175,
|
|
"grad_norm": 1.3671875,
|
|
"learning_rate": 0.000187,
|
|
"loss": 7.3269,
|
|
"mean_token_accuracy": 0.08868191167712211,
|
|
"num_tokens": 590169.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"entropy": 7.639244318008423,
|
|
"epoch": 0.00826015129119207,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 0.0001895,
|
|
"loss": 7.2957,
|
|
"mean_token_accuracy": 0.09457754641771317,
|
|
"num_tokens": 597508.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 7.685401201248169,
|
|
"epoch": 0.008368837492391966,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 0.000192,
|
|
"loss": 7.375,
|
|
"mean_token_accuracy": 0.09122001230716706,
|
|
"num_tokens": 605412.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"entropy": 7.795647668838501,
|
|
"epoch": 0.008477523693591862,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 0.0001945,
|
|
"loss": 7.3872,
|
|
"mean_token_accuracy": 0.09148699790239334,
|
|
"num_tokens": 613512.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 7.641701507568359,
|
|
"epoch": 0.008586209894791757,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 0.00019700000000000002,
|
|
"loss": 7.3275,
|
|
"mean_token_accuracy": 0.09618531838059426,
|
|
"num_tokens": 621117.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"entropy": 7.689551591873169,
|
|
"epoch": 0.008694896095991653,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.00019950000000000002,
|
|
"loss": 7.3832,
|
|
"mean_token_accuracy": 0.08407204449176789,
|
|
"num_tokens": 629970.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 7.637691450119019,
|
|
"epoch": 0.008803582297191548,
|
|
"grad_norm": 1.375,
|
|
"learning_rate": 0.000202,
|
|
"loss": 7.3219,
|
|
"mean_token_accuracy": 0.09131594821810722,
|
|
"num_tokens": 638184.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"entropy": 7.5977555274963375,
|
|
"epoch": 0.008912268498391444,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 0.00020449999999999998,
|
|
"loss": 7.2837,
|
|
"mean_token_accuracy": 0.08929966911673545,
|
|
"num_tokens": 645953.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 7.6464704990386965,
|
|
"epoch": 0.00902095469959134,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 0.000207,
|
|
"loss": 7.3305,
|
|
"mean_token_accuracy": 0.08722574934363365,
|
|
"num_tokens": 654291.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"entropy": 7.672793436050415,
|
|
"epoch": 0.009129640900791236,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 0.0002095,
|
|
"loss": 7.3299,
|
|
"mean_token_accuracy": 0.08851959705352783,
|
|
"num_tokens": 661393.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 7.531088209152221,
|
|
"epoch": 0.009238327101991132,
|
|
"grad_norm": 1.4296875,
|
|
"learning_rate": 0.000212,
|
|
"loss": 7.3139,
|
|
"mean_token_accuracy": 0.0963095597922802,
|
|
"num_tokens": 669913.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"entropy": 7.74607515335083,
|
|
"epoch": 0.009347013303191027,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.0002145,
|
|
"loss": 7.3663,
|
|
"mean_token_accuracy": 0.09074652194976807,
|
|
"num_tokens": 678745.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 7.417545986175537,
|
|
"epoch": 0.009455699504390923,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.00021700000000000002,
|
|
"loss": 7.2468,
|
|
"mean_token_accuracy": 0.09117906540632248,
|
|
"num_tokens": 686191.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"entropy": 7.690976524353028,
|
|
"epoch": 0.009564385705590818,
|
|
"grad_norm": 2.109375,
|
|
"learning_rate": 0.0002195,
|
|
"loss": 7.2921,
|
|
"mean_token_accuracy": 0.08830784261226654,
|
|
"num_tokens": 694776.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 7.471565675735474,
|
|
"epoch": 0.009673071906790714,
|
|
"grad_norm": 1.1953125,
|
|
"learning_rate": 0.000222,
|
|
"loss": 7.2498,
|
|
"mean_token_accuracy": 0.08958808556199074,
|
|
"num_tokens": 702481.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"entropy": 7.601398086547851,
|
|
"epoch": 0.00978175810799061,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.0002245,
|
|
"loss": 7.2892,
|
|
"mean_token_accuracy": 0.08943363651633263,
|
|
"num_tokens": 710304.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 7.6107948303222654,
|
|
"epoch": 0.009890444309190505,
|
|
"grad_norm": 1.2734375,
|
|
"learning_rate": 0.00022700000000000002,
|
|
"loss": 7.2655,
|
|
"mean_token_accuracy": 0.0950858935713768,
|
|
"num_tokens": 718462.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"entropy": 7.4872644424438475,
|
|
"epoch": 0.0099991305103904,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.00022950000000000002,
|
|
"loss": 7.2699,
|
|
"mean_token_accuracy": 0.09967489093542099,
|
|
"num_tokens": 726798.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 7.636318254470825,
|
|
"epoch": 0.010107816711590296,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 0.00023200000000000003,
|
|
"loss": 7.2289,
|
|
"mean_token_accuracy": 0.09415075033903123,
|
|
"num_tokens": 735054.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"entropy": 7.467661046981812,
|
|
"epoch": 0.010216502912790193,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 0.00023449999999999998,
|
|
"loss": 7.2689,
|
|
"mean_token_accuracy": 0.08854644820094108,
|
|
"num_tokens": 743701.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 7.5553264141082765,
|
|
"epoch": 0.010325189113990088,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 0.000237,
|
|
"loss": 7.3436,
|
|
"mean_token_accuracy": 0.08918146193027496,
|
|
"num_tokens": 751548.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"entropy": 7.606575393676758,
|
|
"epoch": 0.010433875315189984,
|
|
"grad_norm": 1.359375,
|
|
"learning_rate": 0.0002395,
|
|
"loss": 7.1245,
|
|
"mean_token_accuracy": 0.09939143806695938,
|
|
"num_tokens": 758580.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 7.376218938827515,
|
|
"epoch": 0.01054256151638988,
|
|
"grad_norm": 1.375,
|
|
"learning_rate": 0.000242,
|
|
"loss": 7.1472,
|
|
"mean_token_accuracy": 0.09824580624699593,
|
|
"num_tokens": 766240.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"entropy": 7.460130596160889,
|
|
"epoch": 0.010651247717589775,
|
|
"grad_norm": 1.3125,
|
|
"learning_rate": 0.0002445,
|
|
"loss": 7.1908,
|
|
"mean_token_accuracy": 0.09887742102146149,
|
|
"num_tokens": 774206.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 7.499665117263794,
|
|
"epoch": 0.01075993391878967,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 0.000247,
|
|
"loss": 7.135,
|
|
"mean_token_accuracy": 0.10001260414719582,
|
|
"num_tokens": 782005.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"entropy": 7.313957262039184,
|
|
"epoch": 0.010868620119989566,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.0002495,
|
|
"loss": 7.1614,
|
|
"mean_token_accuracy": 0.10134897753596306,
|
|
"num_tokens": 790043.0,
|
|
"step": 500
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 4000,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 1294440726528000.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|