1035 lines
27 KiB
JSON
1035 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.0657116572479958,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 10.691970920562744,
|
|
"epoch": 0.000657116572479958,
|
|
"grad_norm": 11.375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 10.9102,
|
|
"mean_token_accuracy": 0.0,
|
|
"num_tokens": 18160.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"entropy": 10.69195909500122,
|
|
"epoch": 0.001314233144959916,
|
|
"grad_norm": 11.8125,
|
|
"learning_rate": 4.5e-06,
|
|
"loss": 10.8426,
|
|
"mean_token_accuracy": 0.00034901105682365595,
|
|
"num_tokens": 34700.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 10.689266014099122,
|
|
"epoch": 0.001971349717439874,
|
|
"grad_norm": 8.375,
|
|
"learning_rate": 7e-06,
|
|
"loss": 10.5846,
|
|
"mean_token_accuracy": 0.0499508892185986,
|
|
"num_tokens": 52678.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"entropy": 10.562198734283447,
|
|
"epoch": 0.002628466289919832,
|
|
"grad_norm": 5.40625,
|
|
"learning_rate": 9.5e-06,
|
|
"loss": 10.3172,
|
|
"mean_token_accuracy": 0.0605622336268425,
|
|
"num_tokens": 69607.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 10.345282936096192,
|
|
"epoch": 0.0032855828623997896,
|
|
"grad_norm": 3.71875,
|
|
"learning_rate": 1.2e-05,
|
|
"loss": 10.0938,
|
|
"mean_token_accuracy": 0.06148592121899128,
|
|
"num_tokens": 88068.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"entropy": 10.521661567687989,
|
|
"epoch": 0.003942699434879748,
|
|
"grad_norm": 3.171875,
|
|
"learning_rate": 1.4500000000000002e-05,
|
|
"loss": 9.9441,
|
|
"mean_token_accuracy": 0.06816631071269512,
|
|
"num_tokens": 105839.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 10.302499389648437,
|
|
"epoch": 0.004599816007359705,
|
|
"grad_norm": 2.375,
|
|
"learning_rate": 1.7000000000000003e-05,
|
|
"loss": 9.899,
|
|
"mean_token_accuracy": 0.0670669324696064,
|
|
"num_tokens": 125068.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"entropy": 10.426428699493409,
|
|
"epoch": 0.005256932579839664,
|
|
"grad_norm": 2.765625,
|
|
"learning_rate": 1.95e-05,
|
|
"loss": 9.8561,
|
|
"mean_token_accuracy": 0.06749101914465427,
|
|
"num_tokens": 143271.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 10.273355770111085,
|
|
"epoch": 0.0059140491523196215,
|
|
"grad_norm": 2.0,
|
|
"learning_rate": 2.2e-05,
|
|
"loss": 9.8657,
|
|
"mean_token_accuracy": 0.06502391099929809,
|
|
"num_tokens": 160886.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"entropy": 10.348071479797364,
|
|
"epoch": 0.006571165724799579,
|
|
"grad_norm": 2.359375,
|
|
"learning_rate": 2.4500000000000003e-05,
|
|
"loss": 9.7845,
|
|
"mean_token_accuracy": 0.06918984390795231,
|
|
"num_tokens": 179036.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 10.29029483795166,
|
|
"epoch": 0.007228282297279538,
|
|
"grad_norm": 1.7578125,
|
|
"learning_rate": 2.7e-05,
|
|
"loss": 9.731,
|
|
"mean_token_accuracy": 0.07089065425097943,
|
|
"num_tokens": 195478.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"entropy": 10.25726251602173,
|
|
"epoch": 0.007885398869759495,
|
|
"grad_norm": 2.296875,
|
|
"learning_rate": 2.95e-05,
|
|
"loss": 9.7119,
|
|
"mean_token_accuracy": 0.0690944753587246,
|
|
"num_tokens": 213003.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 10.302559661865235,
|
|
"epoch": 0.008542515442239454,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 3.2e-05,
|
|
"loss": 9.6475,
|
|
"mean_token_accuracy": 0.07418905831873417,
|
|
"num_tokens": 230223.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"entropy": 10.087755107879639,
|
|
"epoch": 0.00919963201471941,
|
|
"grad_norm": 1.578125,
|
|
"learning_rate": 3.4500000000000005e-05,
|
|
"loss": 9.5049,
|
|
"mean_token_accuracy": 0.0840003713965416,
|
|
"num_tokens": 246555.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 10.160563468933105,
|
|
"epoch": 0.00985674858719937,
|
|
"grad_norm": 1.7109375,
|
|
"learning_rate": 3.7e-05,
|
|
"loss": 9.4854,
|
|
"mean_token_accuracy": 0.07695499211549758,
|
|
"num_tokens": 262985.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"entropy": 10.03568115234375,
|
|
"epoch": 0.010513865159679328,
|
|
"grad_norm": 2.140625,
|
|
"learning_rate": 3.95e-05,
|
|
"loss": 9.3528,
|
|
"mean_token_accuracy": 0.08797612711787224,
|
|
"num_tokens": 281701.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 10.050819873809814,
|
|
"epoch": 0.011170981732159285,
|
|
"grad_norm": 1.6171875,
|
|
"learning_rate": 4.2000000000000004e-05,
|
|
"loss": 9.3271,
|
|
"mean_token_accuracy": 0.08114923164248466,
|
|
"num_tokens": 298432.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"entropy": 10.067263984680176,
|
|
"epoch": 0.011828098304639243,
|
|
"grad_norm": 1.640625,
|
|
"learning_rate": 4.45e-05,
|
|
"loss": 9.2657,
|
|
"mean_token_accuracy": 0.08178326860070229,
|
|
"num_tokens": 315559.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 9.996760272979737,
|
|
"epoch": 0.012485214877119202,
|
|
"grad_norm": 1.59375,
|
|
"learning_rate": 4.7000000000000004e-05,
|
|
"loss": 9.174,
|
|
"mean_token_accuracy": 0.08735092431306839,
|
|
"num_tokens": 332959.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"entropy": 10.032902812957763,
|
|
"epoch": 0.013142331449599158,
|
|
"grad_norm": 1.9609375,
|
|
"learning_rate": 4.9500000000000004e-05,
|
|
"loss": 9.1565,
|
|
"mean_token_accuracy": 0.07826860286295415,
|
|
"num_tokens": 349872.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 9.87295093536377,
|
|
"epoch": 0.013799448022079117,
|
|
"grad_norm": 1.3203125,
|
|
"learning_rate": 5.2e-05,
|
|
"loss": 9.0779,
|
|
"mean_token_accuracy": 0.07845050990581512,
|
|
"num_tokens": 365952.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"entropy": 9.94616346359253,
|
|
"epoch": 0.014456564594559075,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 5.45e-05,
|
|
"loss": 8.9723,
|
|
"mean_token_accuracy": 0.0847774550318718,
|
|
"num_tokens": 382830.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 9.765446853637695,
|
|
"epoch": 0.015113681167039032,
|
|
"grad_norm": 1.6796875,
|
|
"learning_rate": 5.7e-05,
|
|
"loss": 8.8959,
|
|
"mean_token_accuracy": 0.08348627090454101,
|
|
"num_tokens": 401729.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"entropy": 9.696452522277832,
|
|
"epoch": 0.01577079773951899,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 5.9499999999999996e-05,
|
|
"loss": 8.7663,
|
|
"mean_token_accuracy": 0.0866998441517353,
|
|
"num_tokens": 419094.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 9.62384328842163,
|
|
"epoch": 0.01642791431199895,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 6.2e-05,
|
|
"loss": 8.7123,
|
|
"mean_token_accuracy": 0.08670503944158554,
|
|
"num_tokens": 438632.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"entropy": 9.458528423309327,
|
|
"epoch": 0.017085030884478908,
|
|
"grad_norm": 2.109375,
|
|
"learning_rate": 6.450000000000001e-05,
|
|
"loss": 8.5991,
|
|
"mean_token_accuracy": 0.08925274014472961,
|
|
"num_tokens": 456025.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 9.395387935638428,
|
|
"epoch": 0.017742147456958863,
|
|
"grad_norm": 0.98046875,
|
|
"learning_rate": 6.7e-05,
|
|
"loss": 8.5874,
|
|
"mean_token_accuracy": 0.08284339755773544,
|
|
"num_tokens": 473414.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"entropy": 9.288764190673827,
|
|
"epoch": 0.01839926402943882,
|
|
"grad_norm": 0.91796875,
|
|
"learning_rate": 6.950000000000001e-05,
|
|
"loss": 8.484,
|
|
"mean_token_accuracy": 0.08540036529302597,
|
|
"num_tokens": 490039.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 9.080249691009522,
|
|
"epoch": 0.01905638060191878,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 7.2e-05,
|
|
"loss": 8.4185,
|
|
"mean_token_accuracy": 0.09069929495453835,
|
|
"num_tokens": 506896.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"entropy": 8.990424156188965,
|
|
"epoch": 0.01971349717439874,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 7.45e-05,
|
|
"loss": 8.3134,
|
|
"mean_token_accuracy": 0.0939207799732685,
|
|
"num_tokens": 523544.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 8.895142555236816,
|
|
"epoch": 0.020370613746878697,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 7.7e-05,
|
|
"loss": 8.3193,
|
|
"mean_token_accuracy": 0.0861300639808178,
|
|
"num_tokens": 540240.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"entropy": 8.84927806854248,
|
|
"epoch": 0.021027730319358656,
|
|
"grad_norm": 0.94140625,
|
|
"learning_rate": 7.950000000000001e-05,
|
|
"loss": 8.3388,
|
|
"mean_token_accuracy": 0.08859094232320786,
|
|
"num_tokens": 558460.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 8.804749298095704,
|
|
"epoch": 0.02168484689183861,
|
|
"grad_norm": 1.671875,
|
|
"learning_rate": 8.2e-05,
|
|
"loss": 8.3132,
|
|
"mean_token_accuracy": 0.08978721722960473,
|
|
"num_tokens": 577356.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"entropy": 8.755771827697753,
|
|
"epoch": 0.02234196346431857,
|
|
"grad_norm": 0.8359375,
|
|
"learning_rate": 8.450000000000001e-05,
|
|
"loss": 8.3233,
|
|
"mean_token_accuracy": 0.08325318172574044,
|
|
"num_tokens": 594702.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 8.61918659210205,
|
|
"epoch": 0.022999080036798528,
|
|
"grad_norm": 0.7734375,
|
|
"learning_rate": 8.7e-05,
|
|
"loss": 8.2927,
|
|
"mean_token_accuracy": 0.08967687636613846,
|
|
"num_tokens": 612005.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"entropy": 8.692005062103272,
|
|
"epoch": 0.023656196609278486,
|
|
"grad_norm": 1.171875,
|
|
"learning_rate": 8.95e-05,
|
|
"loss": 8.2979,
|
|
"mean_token_accuracy": 0.08311463445425034,
|
|
"num_tokens": 629271.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 8.543895149230957,
|
|
"epoch": 0.024313313181758445,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 9.2e-05,
|
|
"loss": 8.2711,
|
|
"mean_token_accuracy": 0.09236689656972885,
|
|
"num_tokens": 646273.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"entropy": 8.623021125793457,
|
|
"epoch": 0.024970429754238403,
|
|
"grad_norm": 0.94140625,
|
|
"learning_rate": 9.45e-05,
|
|
"loss": 8.2622,
|
|
"mean_token_accuracy": 0.08237618803977967,
|
|
"num_tokens": 663263.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 8.503989696502686,
|
|
"epoch": 0.02562754632671836,
|
|
"grad_norm": 1.015625,
|
|
"learning_rate": 9.7e-05,
|
|
"loss": 8.1725,
|
|
"mean_token_accuracy": 0.09226469621062279,
|
|
"num_tokens": 681032.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"entropy": 8.496358585357665,
|
|
"epoch": 0.026284662899198317,
|
|
"grad_norm": 0.89453125,
|
|
"learning_rate": 9.95e-05,
|
|
"loss": 8.2157,
|
|
"mean_token_accuracy": 0.08867634385824204,
|
|
"num_tokens": 699353.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 8.597747421264648,
|
|
"epoch": 0.026941779471678275,
|
|
"grad_norm": 0.90625,
|
|
"learning_rate": 0.000102,
|
|
"loss": 8.2941,
|
|
"mean_token_accuracy": 0.08325276263058186,
|
|
"num_tokens": 717777.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"entropy": 8.493968677520751,
|
|
"epoch": 0.027598896044158234,
|
|
"grad_norm": 0.8671875,
|
|
"learning_rate": 0.00010449999999999999,
|
|
"loss": 8.2395,
|
|
"mean_token_accuracy": 0.08642884120345115,
|
|
"num_tokens": 733831.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 8.45910186767578,
|
|
"epoch": 0.028256012616638192,
|
|
"grad_norm": 0.7734375,
|
|
"learning_rate": 0.000107,
|
|
"loss": 8.202,
|
|
"mean_token_accuracy": 0.0875705562531948,
|
|
"num_tokens": 750057.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"entropy": 8.502654266357421,
|
|
"epoch": 0.02891312918911815,
|
|
"grad_norm": 1.5,
|
|
"learning_rate": 0.0001095,
|
|
"loss": 8.2335,
|
|
"mean_token_accuracy": 0.08846379294991494,
|
|
"num_tokens": 767590.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 8.471471691131592,
|
|
"epoch": 0.029570245761598106,
|
|
"grad_norm": 0.8359375,
|
|
"learning_rate": 0.000112,
|
|
"loss": 8.1997,
|
|
"mean_token_accuracy": 0.0836844079196453,
|
|
"num_tokens": 784496.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"entropy": 8.473236179351806,
|
|
"epoch": 0.030227362334078064,
|
|
"grad_norm": 0.8359375,
|
|
"learning_rate": 0.0001145,
|
|
"loss": 8.22,
|
|
"mean_token_accuracy": 0.08451615646481514,
|
|
"num_tokens": 802589.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 8.466453552246094,
|
|
"epoch": 0.030884478906558023,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 0.00011700000000000001,
|
|
"loss": 8.2459,
|
|
"mean_token_accuracy": 0.08304352685809135,
|
|
"num_tokens": 820375.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"entropy": 8.500118160247803,
|
|
"epoch": 0.03154159547903798,
|
|
"grad_norm": 0.9140625,
|
|
"learning_rate": 0.00011949999999999999,
|
|
"loss": 8.1952,
|
|
"mean_token_accuracy": 0.08304070755839348,
|
|
"num_tokens": 838658.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 8.381299304962159,
|
|
"epoch": 0.03219871205151794,
|
|
"grad_norm": 0.78515625,
|
|
"learning_rate": 0.000122,
|
|
"loss": 8.205,
|
|
"mean_token_accuracy": 0.0860319972038269,
|
|
"num_tokens": 856534.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"entropy": 8.390718936920166,
|
|
"epoch": 0.0328558286239979,
|
|
"grad_norm": 0.98828125,
|
|
"learning_rate": 0.0001245,
|
|
"loss": 8.1696,
|
|
"mean_token_accuracy": 0.09368039518594742,
|
|
"num_tokens": 872813.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 8.36027283668518,
|
|
"epoch": 0.03351294519647786,
|
|
"grad_norm": 0.83984375,
|
|
"learning_rate": 0.000127,
|
|
"loss": 8.1428,
|
|
"mean_token_accuracy": 0.09875724688172341,
|
|
"num_tokens": 891675.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"entropy": 8.41018009185791,
|
|
"epoch": 0.034170061768957816,
|
|
"grad_norm": 0.8515625,
|
|
"learning_rate": 0.0001295,
|
|
"loss": 8.1972,
|
|
"mean_token_accuracy": 0.08137553408741952,
|
|
"num_tokens": 907440.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 8.448321056365966,
|
|
"epoch": 0.034827178341437774,
|
|
"grad_norm": 0.9921875,
|
|
"learning_rate": 0.000132,
|
|
"loss": 8.2255,
|
|
"mean_token_accuracy": 0.08277432322502136,
|
|
"num_tokens": 923909.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"entropy": 8.386664962768554,
|
|
"epoch": 0.035484294913917726,
|
|
"grad_norm": 0.859375,
|
|
"learning_rate": 0.00013450000000000002,
|
|
"loss": 8.1077,
|
|
"mean_token_accuracy": 0.09150323122739792,
|
|
"num_tokens": 940658.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 8.34205665588379,
|
|
"epoch": 0.036141411486397684,
|
|
"grad_norm": 0.8203125,
|
|
"learning_rate": 0.00013700000000000002,
|
|
"loss": 8.1535,
|
|
"mean_token_accuracy": 0.08561521768569946,
|
|
"num_tokens": 959095.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"entropy": 8.375339221954345,
|
|
"epoch": 0.03679852805887764,
|
|
"grad_norm": 0.94921875,
|
|
"learning_rate": 0.0001395,
|
|
"loss": 8.1774,
|
|
"mean_token_accuracy": 0.08340813145041466,
|
|
"num_tokens": 976656.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 8.368755626678468,
|
|
"epoch": 0.0374556446313576,
|
|
"grad_norm": 0.8515625,
|
|
"learning_rate": 0.00014199999999999998,
|
|
"loss": 8.2048,
|
|
"mean_token_accuracy": 0.08358296155929565,
|
|
"num_tokens": 994326.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"entropy": 8.306324100494384,
|
|
"epoch": 0.03811276120383756,
|
|
"grad_norm": 0.90234375,
|
|
"learning_rate": 0.0001445,
|
|
"loss": 8.1147,
|
|
"mean_token_accuracy": 0.0932599276304245,
|
|
"num_tokens": 1011859.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 8.31154556274414,
|
|
"epoch": 0.03876987777631752,
|
|
"grad_norm": 0.87890625,
|
|
"learning_rate": 0.000147,
|
|
"loss": 8.142,
|
|
"mean_token_accuracy": 0.08438249453902244,
|
|
"num_tokens": 1028329.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"entropy": 8.38582706451416,
|
|
"epoch": 0.03942699434879748,
|
|
"grad_norm": 0.890625,
|
|
"learning_rate": 0.0001495,
|
|
"loss": 8.1226,
|
|
"mean_token_accuracy": 0.08788742870092392,
|
|
"num_tokens": 1045519.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 8.42501277923584,
|
|
"epoch": 0.040084110921277435,
|
|
"grad_norm": 1.265625,
|
|
"learning_rate": 0.000152,
|
|
"loss": 8.1104,
|
|
"mean_token_accuracy": 0.09161722511053086,
|
|
"num_tokens": 1063832.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"entropy": 8.180773544311524,
|
|
"epoch": 0.040741227493757394,
|
|
"grad_norm": 1.4921875,
|
|
"learning_rate": 0.00015450000000000001,
|
|
"loss": 8.0692,
|
|
"mean_token_accuracy": 0.09160381332039833,
|
|
"num_tokens": 1081062.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 8.319191455841064,
|
|
"epoch": 0.04139834406623735,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 0.000157,
|
|
"loss": 8.1206,
|
|
"mean_token_accuracy": 0.08650392666459084,
|
|
"num_tokens": 1100139.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"entropy": 8.339826774597167,
|
|
"epoch": 0.04205546063871731,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 0.0001595,
|
|
"loss": 8.0878,
|
|
"mean_token_accuracy": 0.08833744451403618,
|
|
"num_tokens": 1117100.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 8.358532238006593,
|
|
"epoch": 0.04271257721119727,
|
|
"grad_norm": 0.83984375,
|
|
"learning_rate": 0.000162,
|
|
"loss": 8.1339,
|
|
"mean_token_accuracy": 0.08414132967591285,
|
|
"num_tokens": 1136607.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"entropy": 8.360890769958496,
|
|
"epoch": 0.04336969378367722,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.00016450000000000001,
|
|
"loss": 8.1636,
|
|
"mean_token_accuracy": 0.08001711294054985,
|
|
"num_tokens": 1154463.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 8.302077484130859,
|
|
"epoch": 0.04402681035615718,
|
|
"grad_norm": 0.953125,
|
|
"learning_rate": 0.00016700000000000002,
|
|
"loss": 8.0492,
|
|
"mean_token_accuracy": 0.08888905569911003,
|
|
"num_tokens": 1172540.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"entropy": 8.284979152679444,
|
|
"epoch": 0.04468392692863714,
|
|
"grad_norm": 0.87890625,
|
|
"learning_rate": 0.00016950000000000003,
|
|
"loss": 8.089,
|
|
"mean_token_accuracy": 0.08480592146515846,
|
|
"num_tokens": 1189393.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 8.305260276794433,
|
|
"epoch": 0.0453410435011171,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.00017199999999999998,
|
|
"loss": 8.0804,
|
|
"mean_token_accuracy": 0.087804014980793,
|
|
"num_tokens": 1205232.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"entropy": 8.235641956329346,
|
|
"epoch": 0.045998160073597055,
|
|
"grad_norm": 0.87890625,
|
|
"learning_rate": 0.00017449999999999999,
|
|
"loss": 8.0386,
|
|
"mean_token_accuracy": 0.08915610313415527,
|
|
"num_tokens": 1222020.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 8.296236038208008,
|
|
"epoch": 0.046655276646077014,
|
|
"grad_norm": 0.9453125,
|
|
"learning_rate": 0.000177,
|
|
"loss": 8.0122,
|
|
"mean_token_accuracy": 0.09561690092086791,
|
|
"num_tokens": 1239201.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"entropy": 8.312324523925781,
|
|
"epoch": 0.04731239321855697,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.0001795,
|
|
"loss": 8.0528,
|
|
"mean_token_accuracy": 0.09137414395809174,
|
|
"num_tokens": 1256715.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 8.276749324798583,
|
|
"epoch": 0.04796950979103693,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.000182,
|
|
"loss": 8.0682,
|
|
"mean_token_accuracy": 0.08286751359701157,
|
|
"num_tokens": 1273800.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"entropy": 8.162324810028077,
|
|
"epoch": 0.04862662636351689,
|
|
"grad_norm": 0.99609375,
|
|
"learning_rate": 0.0001845,
|
|
"loss": 8.0052,
|
|
"mean_token_accuracy": 0.09242228120565414,
|
|
"num_tokens": 1291181.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 8.235886669158935,
|
|
"epoch": 0.04928374293599685,
|
|
"grad_norm": 0.90234375,
|
|
"learning_rate": 0.000187,
|
|
"loss": 8.088,
|
|
"mean_token_accuracy": 0.08434378504753112,
|
|
"num_tokens": 1309264.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"entropy": 8.252655410766602,
|
|
"epoch": 0.049940859508476806,
|
|
"grad_norm": 0.88671875,
|
|
"learning_rate": 0.0001895,
|
|
"loss": 7.9823,
|
|
"mean_token_accuracy": 0.08949511088430881,
|
|
"num_tokens": 1327117.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 8.148046970367432,
|
|
"epoch": 0.050597976080956765,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 0.000192,
|
|
"loss": 8.0288,
|
|
"mean_token_accuracy": 0.08982144668698311,
|
|
"num_tokens": 1343972.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"entropy": 8.245464324951172,
|
|
"epoch": 0.05125509265343672,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 0.0001945,
|
|
"loss": 7.9572,
|
|
"mean_token_accuracy": 0.08928473889827729,
|
|
"num_tokens": 1361432.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 8.094884061813355,
|
|
"epoch": 0.051912209225916675,
|
|
"grad_norm": 1.4296875,
|
|
"learning_rate": 0.00019700000000000002,
|
|
"loss": 7.9669,
|
|
"mean_token_accuracy": 0.09419442862272262,
|
|
"num_tokens": 1378191.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"entropy": 8.211234951019287,
|
|
"epoch": 0.052569325798396634,
|
|
"grad_norm": 0.875,
|
|
"learning_rate": 0.00019950000000000002,
|
|
"loss": 7.9813,
|
|
"mean_token_accuracy": 0.08906673565506935,
|
|
"num_tokens": 1395872.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 8.18825888633728,
|
|
"epoch": 0.05322644237087659,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 0.000202,
|
|
"loss": 7.9578,
|
|
"mean_token_accuracy": 0.09145129919052124,
|
|
"num_tokens": 1412816.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"entropy": 8.134449100494384,
|
|
"epoch": 0.05388355894335655,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 0.00020449999999999998,
|
|
"loss": 7.8935,
|
|
"mean_token_accuracy": 0.10044471621513366,
|
|
"num_tokens": 1430644.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 8.148541164398193,
|
|
"epoch": 0.05454067551583651,
|
|
"grad_norm": 1.015625,
|
|
"learning_rate": 0.000207,
|
|
"loss": 7.9432,
|
|
"mean_token_accuracy": 0.09160158559679984,
|
|
"num_tokens": 1448021.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"entropy": 8.23169984817505,
|
|
"epoch": 0.05519779208831647,
|
|
"grad_norm": 0.84765625,
|
|
"learning_rate": 0.0002095,
|
|
"loss": 7.9585,
|
|
"mean_token_accuracy": 0.09313838183879852,
|
|
"num_tokens": 1466428.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 8.167637252807618,
|
|
"epoch": 0.055854908660796426,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 0.000212,
|
|
"loss": 7.8798,
|
|
"mean_token_accuracy": 0.09942131265997886,
|
|
"num_tokens": 1484554.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"entropy": 8.104038047790528,
|
|
"epoch": 0.056512025233276385,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 0.0002145,
|
|
"loss": 7.8905,
|
|
"mean_token_accuracy": 0.09655392467975617,
|
|
"num_tokens": 1500452.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 8.07980990409851,
|
|
"epoch": 0.05716914180575634,
|
|
"grad_norm": 1.0,
|
|
"learning_rate": 0.00021700000000000002,
|
|
"loss": 7.9718,
|
|
"mean_token_accuracy": 0.0888563945889473,
|
|
"num_tokens": 1518482.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"entropy": 8.099183750152587,
|
|
"epoch": 0.0578262583782363,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 0.0002195,
|
|
"loss": 7.9304,
|
|
"mean_token_accuracy": 0.0907039225101471,
|
|
"num_tokens": 1536107.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 8.13019618988037,
|
|
"epoch": 0.05848337495071626,
|
|
"grad_norm": 1.015625,
|
|
"learning_rate": 0.000222,
|
|
"loss": 7.9209,
|
|
"mean_token_accuracy": 0.08752450793981552,
|
|
"num_tokens": 1553722.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"entropy": 8.061606645584106,
|
|
"epoch": 0.05914049152319621,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 0.0002245,
|
|
"loss": 7.878,
|
|
"mean_token_accuracy": 0.09788650721311569,
|
|
"num_tokens": 1571959.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 8.092618274688721,
|
|
"epoch": 0.05979760809567617,
|
|
"grad_norm": 0.98828125,
|
|
"learning_rate": 0.00022700000000000002,
|
|
"loss": 7.965,
|
|
"mean_token_accuracy": 0.08797019943594933,
|
|
"num_tokens": 1588595.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"entropy": 8.051505088806152,
|
|
"epoch": 0.06045472466815613,
|
|
"grad_norm": 0.828125,
|
|
"learning_rate": 0.00022950000000000002,
|
|
"loss": 7.8329,
|
|
"mean_token_accuracy": 0.08995860144495964,
|
|
"num_tokens": 1605185.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 8.061400985717773,
|
|
"epoch": 0.06111184124063609,
|
|
"grad_norm": 0.90625,
|
|
"learning_rate": 0.00023200000000000003,
|
|
"loss": 7.8334,
|
|
"mean_token_accuracy": 0.0945149227976799,
|
|
"num_tokens": 1622966.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"entropy": 8.077755498886109,
|
|
"epoch": 0.061768957813116046,
|
|
"grad_norm": 1.0390625,
|
|
"learning_rate": 0.00023449999999999998,
|
|
"loss": 7.8267,
|
|
"mean_token_accuracy": 0.09398441277444362,
|
|
"num_tokens": 1639697.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 8.02926688194275,
|
|
"epoch": 0.062426074385596005,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 0.000237,
|
|
"loss": 7.851,
|
|
"mean_token_accuracy": 0.09462198317050934,
|
|
"num_tokens": 1656342.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"entropy": 8.030053997039795,
|
|
"epoch": 0.06308319095807596,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 0.0002395,
|
|
"loss": 7.9004,
|
|
"mean_token_accuracy": 0.09348834380507469,
|
|
"num_tokens": 1673829.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 8.142562198638917,
|
|
"epoch": 0.06374030753055591,
|
|
"grad_norm": 0.84765625,
|
|
"learning_rate": 0.000242,
|
|
"loss": 7.9383,
|
|
"mean_token_accuracy": 0.08797306269407272,
|
|
"num_tokens": 1691938.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"entropy": 7.987461423873901,
|
|
"epoch": 0.06439742410303588,
|
|
"grad_norm": 1.0,
|
|
"learning_rate": 0.0002445,
|
|
"loss": 7.8913,
|
|
"mean_token_accuracy": 0.09134467840194702,
|
|
"num_tokens": 1708545.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 7.951012325286865,
|
|
"epoch": 0.06505454067551583,
|
|
"grad_norm": 0.83203125,
|
|
"learning_rate": 0.000247,
|
|
"loss": 7.8445,
|
|
"mean_token_accuracy": 0.09492667466402054,
|
|
"num_tokens": 1725199.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"entropy": 8.079988193511962,
|
|
"epoch": 0.0657116572479958,
|
|
"grad_norm": 1.03125,
|
|
"learning_rate": 0.0002495,
|
|
"loss": 7.8607,
|
|
"mean_token_accuracy": 0.09226195812225342,
|
|
"num_tokens": 1742893.0,
|
|
"step": 500
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 4000,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 2319097430016000.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|