3165 lines
80 KiB
JSON
3165 lines
80 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 1953,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.002560163850486431,
|
|
"grad_norm": 2.5300508630709384,
|
|
"learning_rate": 2.040816326530612e-06,
|
|
"loss": 0.8795,
|
|
"num_tokens": 1480979.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.005120327700972862,
|
|
"grad_norm": 1.8393469607334099,
|
|
"learning_rate": 4.591836734693878e-06,
|
|
"loss": 0.8214,
|
|
"num_tokens": 2810063.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.007680491551459293,
|
|
"grad_norm": 1.3599779572924813,
|
|
"learning_rate": 7.142857142857143e-06,
|
|
"loss": 0.7645,
|
|
"num_tokens": 4202971.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.010240655401945725,
|
|
"grad_norm": 0.9512403762321455,
|
|
"learning_rate": 9.693877551020408e-06,
|
|
"loss": 0.7523,
|
|
"num_tokens": 5644524.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.012800819252432157,
|
|
"grad_norm": 0.6762938509105254,
|
|
"learning_rate": 1.2244897959183674e-05,
|
|
"loss": 0.707,
|
|
"num_tokens": 7178713.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.015360983102918587,
|
|
"grad_norm": 0.5524432248998126,
|
|
"learning_rate": 1.479591836734694e-05,
|
|
"loss": 0.6706,
|
|
"num_tokens": 8560803.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.017921146953405017,
|
|
"grad_norm": 0.5187145700243703,
|
|
"learning_rate": 1.7346938775510206e-05,
|
|
"loss": 0.6606,
|
|
"num_tokens": 9959461.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.02048131080389145,
|
|
"grad_norm": 0.4790463822798176,
|
|
"learning_rate": 1.989795918367347e-05,
|
|
"loss": 0.632,
|
|
"num_tokens": 11368873.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.02304147465437788,
|
|
"grad_norm": 0.47176011413134555,
|
|
"learning_rate": 2.2448979591836737e-05,
|
|
"loss": 0.6383,
|
|
"num_tokens": 12733944.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.025601638504864313,
|
|
"grad_norm": 0.5173620263189799,
|
|
"learning_rate": 2.5e-05,
|
|
"loss": 0.6149,
|
|
"num_tokens": 14130423.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.02816180235535074,
|
|
"grad_norm": 0.44858255417087073,
|
|
"learning_rate": 2.7551020408163265e-05,
|
|
"loss": 0.6207,
|
|
"num_tokens": 15588155.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.030721966205837174,
|
|
"grad_norm": 0.4967975291995471,
|
|
"learning_rate": 3.0102040816326533e-05,
|
|
"loss": 0.6164,
|
|
"num_tokens": 17042190.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.033282130056323606,
|
|
"grad_norm": 0.49054966743225126,
|
|
"learning_rate": 3.265306122448979e-05,
|
|
"loss": 0.615,
|
|
"num_tokens": 18452237.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.035842293906810034,
|
|
"grad_norm": 0.5191397187176672,
|
|
"learning_rate": 3.520408163265306e-05,
|
|
"loss": 0.5991,
|
|
"num_tokens": 19889730.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.03840245775729647,
|
|
"grad_norm": 0.5278605833297623,
|
|
"learning_rate": 3.775510204081633e-05,
|
|
"loss": 0.5846,
|
|
"num_tokens": 21256777.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.0409626216077829,
|
|
"grad_norm": 0.5136035796748111,
|
|
"learning_rate": 4.0306122448979596e-05,
|
|
"loss": 0.6055,
|
|
"num_tokens": 22641755.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.04352278545826933,
|
|
"grad_norm": 0.654732021792767,
|
|
"learning_rate": 4.2857142857142856e-05,
|
|
"loss": 0.6089,
|
|
"num_tokens": 24118220.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.04608294930875576,
|
|
"grad_norm": 0.5224587767919349,
|
|
"learning_rate": 4.5408163265306124e-05,
|
|
"loss": 0.6022,
|
|
"num_tokens": 25614380.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.04864311315924219,
|
|
"grad_norm": 0.4813232523267767,
|
|
"learning_rate": 4.795918367346939e-05,
|
|
"loss": 0.5822,
|
|
"num_tokens": 27092899.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.051203277009728626,
|
|
"grad_norm": 0.5494936328949089,
|
|
"learning_rate": 4.9999967732573004e-05,
|
|
"loss": 0.5738,
|
|
"num_tokens": 28441260.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.053763440860215055,
|
|
"grad_norm": 0.5480785112155125,
|
|
"learning_rate": 4.999883838234591e-05,
|
|
"loss": 0.5891,
|
|
"num_tokens": 29961915.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.05632360471070148,
|
|
"grad_norm": 0.597366705286243,
|
|
"learning_rate": 4.999609575331786e-05,
|
|
"loss": 0.5854,
|
|
"num_tokens": 31367061.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.05888376856118792,
|
|
"grad_norm": 0.5560459787986106,
|
|
"learning_rate": 4.999174004214899e-05,
|
|
"loss": 0.5971,
|
|
"num_tokens": 32837613.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.06144393241167435,
|
|
"grad_norm": 0.555439269400926,
|
|
"learning_rate": 4.998577156116551e-05,
|
|
"loss": 0.5863,
|
|
"num_tokens": 34231333.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.06400409626216078,
|
|
"grad_norm": 0.5837305787129228,
|
|
"learning_rate": 4.9978190738337236e-05,
|
|
"loss": 0.5878,
|
|
"num_tokens": 35595055.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.06656426011264721,
|
|
"grad_norm": 0.5267965361040731,
|
|
"learning_rate": 4.9968998117246934e-05,
|
|
"loss": 0.606,
|
|
"num_tokens": 37049317.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.06912442396313365,
|
|
"grad_norm": 0.5671476743530952,
|
|
"learning_rate": 4.995819435705133e-05,
|
|
"loss": 0.5814,
|
|
"num_tokens": 38449337.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.07168458781362007,
|
|
"grad_norm": 0.4963014383637305,
|
|
"learning_rate": 4.994578023243386e-05,
|
|
"loss": 0.5928,
|
|
"num_tokens": 39846738.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.0742447516641065,
|
|
"grad_norm": 0.4941157587756355,
|
|
"learning_rate": 4.99317566335491e-05,
|
|
"loss": 0.5849,
|
|
"num_tokens": 41244260.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.07680491551459294,
|
|
"grad_norm": 0.45786773012580717,
|
|
"learning_rate": 4.991612456595897e-05,
|
|
"loss": 0.5839,
|
|
"num_tokens": 42737662.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.07936507936507936,
|
|
"grad_norm": 0.5165068638674144,
|
|
"learning_rate": 4.989888515056057e-05,
|
|
"loss": 0.5973,
|
|
"num_tokens": 44122925.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.0819252432155658,
|
|
"grad_norm": 0.5289485439406936,
|
|
"learning_rate": 4.988003962350589e-05,
|
|
"loss": 0.5704,
|
|
"num_tokens": 45543403.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.08448540706605223,
|
|
"grad_norm": 0.43223276878556893,
|
|
"learning_rate": 4.98595893361131e-05,
|
|
"loss": 0.5893,
|
|
"num_tokens": 46939676.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.08704557091653865,
|
|
"grad_norm": 0.4913198147129614,
|
|
"learning_rate": 4.9837535754769695e-05,
|
|
"loss": 0.5873,
|
|
"num_tokens": 48417309.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.08960573476702509,
|
|
"grad_norm": 0.5121557712264927,
|
|
"learning_rate": 4.981388046082731e-05,
|
|
"loss": 0.5885,
|
|
"num_tokens": 49823574.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.09216589861751152,
|
|
"grad_norm": 0.4983423841902187,
|
|
"learning_rate": 4.97886251504884e-05,
|
|
"loss": 0.5685,
|
|
"num_tokens": 51193029.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.09472606246799795,
|
|
"grad_norm": 0.4579200152395981,
|
|
"learning_rate": 4.976177163468452e-05,
|
|
"loss": 0.5786,
|
|
"num_tokens": 52565059.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.09728622631848438,
|
|
"grad_norm": 0.4992785537966212,
|
|
"learning_rate": 4.973332183894654e-05,
|
|
"loss": 0.5727,
|
|
"num_tokens": 54012592.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.09984639016897082,
|
|
"grad_norm": 0.4709642840412311,
|
|
"learning_rate": 4.970327780326655e-05,
|
|
"loss": 0.5631,
|
|
"num_tokens": 55377389.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.10240655401945725,
|
|
"grad_norm": 0.442086352017686,
|
|
"learning_rate": 4.967164168195159e-05,
|
|
"loss": 0.6005,
|
|
"num_tokens": 56909889.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.10496671786994367,
|
|
"grad_norm": 0.4613928878272275,
|
|
"learning_rate": 4.9638415743469166e-05,
|
|
"loss": 0.5554,
|
|
"num_tokens": 58331874.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.10752688172043011,
|
|
"grad_norm": 0.4762325458125078,
|
|
"learning_rate": 4.9603602370284606e-05,
|
|
"loss": 0.5612,
|
|
"num_tokens": 59676292.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.11008704557091654,
|
|
"grad_norm": 0.4490076387815698,
|
|
"learning_rate": 4.9567204058690194e-05,
|
|
"loss": 0.5638,
|
|
"num_tokens": 61094637.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.11264720942140297,
|
|
"grad_norm": 0.45553375821573616,
|
|
"learning_rate": 4.9529223418626223e-05,
|
|
"loss": 0.5535,
|
|
"num_tokens": 62662896.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.1152073732718894,
|
|
"grad_norm": 0.45681192919046726,
|
|
"learning_rate": 4.9489663173493795e-05,
|
|
"loss": 0.5582,
|
|
"num_tokens": 64108252.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.11776753712237584,
|
|
"grad_norm": 0.46612174997324396,
|
|
"learning_rate": 4.944852615995956e-05,
|
|
"loss": 0.5688,
|
|
"num_tokens": 65561717.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.12032770097286226,
|
|
"grad_norm": 0.43628808082695975,
|
|
"learning_rate": 4.940581532775237e-05,
|
|
"loss": 0.5746,
|
|
"num_tokens": 66953512.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.1228878648233487,
|
|
"grad_norm": 0.4212974917250847,
|
|
"learning_rate": 4.936153373945165e-05,
|
|
"loss": 0.577,
|
|
"num_tokens": 68426882.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.12544802867383512,
|
|
"grad_norm": 0.45286773138071107,
|
|
"learning_rate": 4.9315684570267895e-05,
|
|
"loss": 0.5722,
|
|
"num_tokens": 69913688.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.12800819252432155,
|
|
"grad_norm": 0.4375860811266958,
|
|
"learning_rate": 4.926827110781496e-05,
|
|
"loss": 0.5778,
|
|
"num_tokens": 71299972.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.130568356374808,
|
|
"grad_norm": 0.4205633203097725,
|
|
"learning_rate": 4.9219296751874326e-05,
|
|
"loss": 0.5477,
|
|
"num_tokens": 72771440.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.13312852022529442,
|
|
"grad_norm": 0.409070631530362,
|
|
"learning_rate": 4.9168765014151285e-05,
|
|
"loss": 0.5461,
|
|
"num_tokens": 74221358.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.13568868407578086,
|
|
"grad_norm": 0.486088735719261,
|
|
"learning_rate": 4.9116679518023185e-05,
|
|
"loss": 0.5741,
|
|
"num_tokens": 75684137.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.1382488479262673,
|
|
"grad_norm": 0.4690291905328147,
|
|
"learning_rate": 4.90630439982796e-05,
|
|
"loss": 0.5528,
|
|
"num_tokens": 77124379.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.1408090117767537,
|
|
"grad_norm": 0.466147056512378,
|
|
"learning_rate": 4.900786230085449e-05,
|
|
"loss": 0.5654,
|
|
"num_tokens": 78667993.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.14336917562724014,
|
|
"grad_norm": 0.39507232005205956,
|
|
"learning_rate": 4.8951138382550494e-05,
|
|
"loss": 0.5546,
|
|
"num_tokens": 80152955.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.14592933947772657,
|
|
"grad_norm": 0.5008477536472351,
|
|
"learning_rate": 4.889287631075514e-05,
|
|
"loss": 0.5554,
|
|
"num_tokens": 81535402.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.148489503328213,
|
|
"grad_norm": 0.4372427275060086,
|
|
"learning_rate": 4.883308026314924e-05,
|
|
"loss": 0.5725,
|
|
"num_tokens": 82980445.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.15104966717869944,
|
|
"grad_norm": 0.41410485591451773,
|
|
"learning_rate": 4.87717545274073e-05,
|
|
"loss": 0.5474,
|
|
"num_tokens": 84319720.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.15360983102918588,
|
|
"grad_norm": 0.45862658956169994,
|
|
"learning_rate": 4.870890350089011e-05,
|
|
"loss": 0.5475,
|
|
"num_tokens": 85675897.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.15616999487967229,
|
|
"grad_norm": 0.4487979731969304,
|
|
"learning_rate": 4.8644531690329395e-05,
|
|
"loss": 0.5898,
|
|
"num_tokens": 87175045.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.15873015873015872,
|
|
"grad_norm": 0.4360864293961,
|
|
"learning_rate": 4.857864371150467e-05,
|
|
"loss": 0.5512,
|
|
"num_tokens": 88637962.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.16129032258064516,
|
|
"grad_norm": 0.4353017431808826,
|
|
"learning_rate": 4.851124428891226e-05,
|
|
"loss": 0.556,
|
|
"num_tokens": 90060334.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.1638504864311316,
|
|
"grad_norm": 0.47023171702942174,
|
|
"learning_rate": 4.844233825542656e-05,
|
|
"loss": 0.5465,
|
|
"num_tokens": 91542428.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.16641065028161803,
|
|
"grad_norm": 0.43917403172913827,
|
|
"learning_rate": 4.837193055195344e-05,
|
|
"loss": 0.5525,
|
|
"num_tokens": 93031983.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.16897081413210446,
|
|
"grad_norm": 0.468642550537799,
|
|
"learning_rate": 4.830002622707605e-05,
|
|
"loss": 0.5649,
|
|
"num_tokens": 94474291.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.1715309779825909,
|
|
"grad_norm": 0.3880940106612931,
|
|
"learning_rate": 4.822663043669267e-05,
|
|
"loss": 0.5519,
|
|
"num_tokens": 95859458.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.1740911418330773,
|
|
"grad_norm": 0.40564434459390325,
|
|
"learning_rate": 4.8151748443647146e-05,
|
|
"loss": 0.5599,
|
|
"num_tokens": 97303144.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.17665130568356374,
|
|
"grad_norm": 0.4544115961078073,
|
|
"learning_rate": 4.807538561735143e-05,
|
|
"loss": 0.5507,
|
|
"num_tokens": 98828485.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.17921146953405018,
|
|
"grad_norm": 0.4139210229468292,
|
|
"learning_rate": 4.799754743340062e-05,
|
|
"loss": 0.5597,
|
|
"num_tokens": 100403220.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.1817716333845366,
|
|
"grad_norm": 0.3901273720281205,
|
|
"learning_rate": 4.791823947318029e-05,
|
|
"loss": 0.5456,
|
|
"num_tokens": 101853954.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.18433179723502305,
|
|
"grad_norm": 0.446523107540595,
|
|
"learning_rate": 4.783746742346629e-05,
|
|
"loss": 0.5449,
|
|
"num_tokens": 103223537.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.18689196108550948,
|
|
"grad_norm": 0.43291124305935974,
|
|
"learning_rate": 4.7755237076017e-05,
|
|
"loss": 0.5652,
|
|
"num_tokens": 104589567.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.1894521249359959,
|
|
"grad_norm": 0.4358186840014953,
|
|
"learning_rate": 4.7671554327158e-05,
|
|
"loss": 0.5561,
|
|
"num_tokens": 105902522.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.19201228878648233,
|
|
"grad_norm": 0.39449293473235647,
|
|
"learning_rate": 4.7586425177359284e-05,
|
|
"loss": 0.552,
|
|
"num_tokens": 107379980.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.19457245263696876,
|
|
"grad_norm": 0.39544662671369296,
|
|
"learning_rate": 4.749985573080501e-05,
|
|
"loss": 0.5312,
|
|
"num_tokens": 108801293.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.1971326164874552,
|
|
"grad_norm": 0.41877815630888177,
|
|
"learning_rate": 4.7411852194955795e-05,
|
|
"loss": 0.5504,
|
|
"num_tokens": 110322879.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.19969278033794163,
|
|
"grad_norm": 0.4747472060422058,
|
|
"learning_rate": 4.732242088010357e-05,
|
|
"loss": 0.5527,
|
|
"num_tokens": 111770152.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.20225294418842807,
|
|
"grad_norm": 0.4003919718503912,
|
|
"learning_rate": 4.723156819891917e-05,
|
|
"loss": 0.5539,
|
|
"num_tokens": 113170346.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.2048131080389145,
|
|
"grad_norm": 0.48941720328887883,
|
|
"learning_rate": 4.713930066599246e-05,
|
|
"loss": 0.547,
|
|
"num_tokens": 114521504.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.2073732718894009,
|
|
"grad_norm": 0.42098673145028487,
|
|
"learning_rate": 4.704562489736522e-05,
|
|
"loss": 0.5368,
|
|
"num_tokens": 115935889.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.20993343573988735,
|
|
"grad_norm": 0.3743853899710457,
|
|
"learning_rate": 4.695054761005676e-05,
|
|
"loss": 0.5397,
|
|
"num_tokens": 117394038.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.21249359959037378,
|
|
"grad_norm": 0.4432867373644286,
|
|
"learning_rate": 4.685407562158226e-05,
|
|
"loss": 0.5474,
|
|
"num_tokens": 118961688.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.21505376344086022,
|
|
"grad_norm": 0.3885693691556606,
|
|
"learning_rate": 4.675621584946391e-05,
|
|
"loss": 0.5344,
|
|
"num_tokens": 120303683.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.21761392729134665,
|
|
"grad_norm": 0.43949459217718756,
|
|
"learning_rate": 4.665697531073489e-05,
|
|
"loss": 0.5532,
|
|
"num_tokens": 121845705.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.2201740911418331,
|
|
"grad_norm": 0.5086802933229837,
|
|
"learning_rate": 4.655636112143627e-05,
|
|
"loss": 0.5416,
|
|
"num_tokens": 123256293.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.2227342549923195,
|
|
"grad_norm": 0.38300640047152024,
|
|
"learning_rate": 4.645438049610667e-05,
|
|
"loss": 0.5639,
|
|
"num_tokens": 124670009.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.22529441884280593,
|
|
"grad_norm": 0.35447003296997764,
|
|
"learning_rate": 4.635104074726498e-05,
|
|
"loss": 0.5349,
|
|
"num_tokens": 126094524.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.22785458269329237,
|
|
"grad_norm": 0.3755640943519397,
|
|
"learning_rate": 4.624634928488605e-05,
|
|
"loss": 0.5288,
|
|
"num_tokens": 127590732.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.2304147465437788,
|
|
"grad_norm": 0.36064430665188524,
|
|
"learning_rate": 4.614031361586929e-05,
|
|
"loss": 0.5399,
|
|
"num_tokens": 129049487.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.23297491039426524,
|
|
"grad_norm": 0.3669491377684556,
|
|
"learning_rate": 4.6032941343500446e-05,
|
|
"loss": 0.5578,
|
|
"num_tokens": 130454114.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.23553507424475167,
|
|
"grad_norm": 0.4527979204127126,
|
|
"learning_rate": 4.592424016690637e-05,
|
|
"loss": 0.5553,
|
|
"num_tokens": 131847305.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.23809523809523808,
|
|
"grad_norm": 0.4138734102681689,
|
|
"learning_rate": 4.581421788050297e-05,
|
|
"loss": 0.5536,
|
|
"num_tokens": 133312972.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.24065540194572452,
|
|
"grad_norm": 0.4834711650819541,
|
|
"learning_rate": 4.570288237343632e-05,
|
|
"loss": 0.5463,
|
|
"num_tokens": 134643087.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.24321556579621095,
|
|
"grad_norm": 0.45413608237054304,
|
|
"learning_rate": 4.5590241629016934e-05,
|
|
"loss": 0.5322,
|
|
"num_tokens": 135981616.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.2457757296466974,
|
|
"grad_norm": 0.43455496497258744,
|
|
"learning_rate": 4.547630372414737e-05,
|
|
"loss": 0.5469,
|
|
"num_tokens": 137522281.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.24833589349718382,
|
|
"grad_norm": 0.3672461952328971,
|
|
"learning_rate": 4.5361076828743056e-05,
|
|
"loss": 0.5326,
|
|
"num_tokens": 138920117.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.25089605734767023,
|
|
"grad_norm": 0.42193382569425947,
|
|
"learning_rate": 4.5244569205146425e-05,
|
|
"loss": 0.5314,
|
|
"num_tokens": 140291163.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.2534562211981567,
|
|
"grad_norm": 0.3350889556588213,
|
|
"learning_rate": 4.512678920753455e-05,
|
|
"loss": 0.5262,
|
|
"num_tokens": 141776979.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.2560163850486431,
|
|
"grad_norm": 0.40227783564480246,
|
|
"learning_rate": 4.5007745281320034e-05,
|
|
"loss": 0.5674,
|
|
"num_tokens": 143277757.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.25857654889912957,
|
|
"grad_norm": 0.4342210414853687,
|
|
"learning_rate": 4.488744596254545e-05,
|
|
"loss": 0.529,
|
|
"num_tokens": 144640269.0,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.261136712749616,
|
|
"grad_norm": 0.397021534746738,
|
|
"learning_rate": 4.476589987727131e-05,
|
|
"loss": 0.5329,
|
|
"num_tokens": 146097501.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.2636968766001024,
|
|
"grad_norm": 0.40582353428712326,
|
|
"learning_rate": 4.464311574095743e-05,
|
|
"loss": 0.5252,
|
|
"num_tokens": 147469788.0,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.26625704045058884,
|
|
"grad_norm": 0.3912884953981449,
|
|
"learning_rate": 4.451910235783814e-05,
|
|
"loss": 0.5378,
|
|
"num_tokens": 148940122.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.26881720430107525,
|
|
"grad_norm": 0.38697280777982906,
|
|
"learning_rate": 4.439386862029081e-05,
|
|
"loss": 0.5481,
|
|
"num_tokens": 150470710.0,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.2713773681515617,
|
|
"grad_norm": 0.3837520231871204,
|
|
"learning_rate": 4.426742350819834e-05,
|
|
"loss": 0.5404,
|
|
"num_tokens": 151835104.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.2739375320020481,
|
|
"grad_norm": 0.406054324311566,
|
|
"learning_rate": 4.413977608830522e-05,
|
|
"loss": 0.5206,
|
|
"num_tokens": 153284379.0,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.2764976958525346,
|
|
"grad_norm": 0.34706080946915546,
|
|
"learning_rate": 4.401093551356739e-05,
|
|
"loss": 0.523,
|
|
"num_tokens": 154713852.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.279057859703021,
|
|
"grad_norm": 0.3889060778304188,
|
|
"learning_rate": 4.388091102249596e-05,
|
|
"loss": 0.5589,
|
|
"num_tokens": 156194055.0,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.2816180235535074,
|
|
"grad_norm": 0.3653717635863124,
|
|
"learning_rate": 4.374971193849469e-05,
|
|
"loss": 0.5271,
|
|
"num_tokens": 157579072.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.28417818740399386,
|
|
"grad_norm": 0.41898480591447024,
|
|
"learning_rate": 4.361734766919156e-05,
|
|
"loss": 0.5295,
|
|
"num_tokens": 158993240.0,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.2867383512544803,
|
|
"grad_norm": 0.3725192690688962,
|
|
"learning_rate": 4.3483827705764094e-05,
|
|
"loss": 0.5427,
|
|
"num_tokens": 160433326.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.28929851510496674,
|
|
"grad_norm": 0.40941972486510936,
|
|
"learning_rate": 4.3349161622258885e-05,
|
|
"loss": 0.5251,
|
|
"num_tokens": 161877664.0,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.29185867895545314,
|
|
"grad_norm": 0.4096837717468568,
|
|
"learning_rate": 4.321335907490502e-05,
|
|
"loss": 0.5497,
|
|
"num_tokens": 163305990.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.2944188428059396,
|
|
"grad_norm": 0.37691830402973187,
|
|
"learning_rate": 4.3076429801421706e-05,
|
|
"loss": 0.5296,
|
|
"num_tokens": 164843463.0,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.296979006656426,
|
|
"grad_norm": 0.3563555690840735,
|
|
"learning_rate": 4.2938383620320034e-05,
|
|
"loss": 0.5131,
|
|
"num_tokens": 166249544.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.2995391705069124,
|
|
"grad_norm": 0.3885863366967285,
|
|
"learning_rate": 4.279923043019893e-05,
|
|
"loss": 0.5166,
|
|
"num_tokens": 167717750.0,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.3020993343573989,
|
|
"grad_norm": 0.35983955088654485,
|
|
"learning_rate": 4.265898020903539e-05,
|
|
"loss": 0.5353,
|
|
"num_tokens": 169217712.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.3046594982078853,
|
|
"grad_norm": 0.34837734988389174,
|
|
"learning_rate": 4.2517643013469e-05,
|
|
"loss": 0.5444,
|
|
"num_tokens": 170673117.0,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.30721966205837176,
|
|
"grad_norm": 0.33174509514566075,
|
|
"learning_rate": 4.237522897808081e-05,
|
|
"loss": 0.5138,
|
|
"num_tokens": 172132000.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.30977982590885816,
|
|
"grad_norm": 0.3636223922655082,
|
|
"learning_rate": 4.2231748314666684e-05,
|
|
"loss": 0.5224,
|
|
"num_tokens": 173616801.0,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.31233998975934457,
|
|
"grad_norm": 0.36812793348796435,
|
|
"learning_rate": 4.208721131150499e-05,
|
|
"loss": 0.5421,
|
|
"num_tokens": 175123127.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.31490015360983103,
|
|
"grad_norm": 0.3635308268435295,
|
|
"learning_rate": 4.194162833261893e-05,
|
|
"loss": 0.5108,
|
|
"num_tokens": 176593606.0,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.31746031746031744,
|
|
"grad_norm": 0.3716178330349266,
|
|
"learning_rate": 4.1795009817033405e-05,
|
|
"loss": 0.4996,
|
|
"num_tokens": 178019887.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.3200204813108039,
|
|
"grad_norm": 0.4593909214341713,
|
|
"learning_rate": 4.164736627802642e-05,
|
|
"loss": 0.536,
|
|
"num_tokens": 179438824.0,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.3225806451612903,
|
|
"grad_norm": 0.37100308583638236,
|
|
"learning_rate": 4.1498708302375277e-05,
|
|
"loss": 0.5318,
|
|
"num_tokens": 180865573.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.3251408090117768,
|
|
"grad_norm": 0.3816706005261259,
|
|
"learning_rate": 4.134904654959745e-05,
|
|
"loss": 0.5247,
|
|
"num_tokens": 182320409.0,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.3277009728622632,
|
|
"grad_norm": 0.3615973929611168,
|
|
"learning_rate": 4.1198391751186227e-05,
|
|
"loss": 0.5308,
|
|
"num_tokens": 183746129.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.3302611367127496,
|
|
"grad_norm": 0.37017697423260804,
|
|
"learning_rate": 4.104675470984121e-05,
|
|
"loss": 0.5335,
|
|
"num_tokens": 185213495.0,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.33282130056323606,
|
|
"grad_norm": 0.36360252959695066,
|
|
"learning_rate": 4.089414629869373e-05,
|
|
"loss": 0.5445,
|
|
"num_tokens": 186708443.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.33538146441372246,
|
|
"grad_norm": 0.4000400218919281,
|
|
"learning_rate": 4.0740577460527136e-05,
|
|
"loss": 0.5343,
|
|
"num_tokens": 188157669.0,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.3379416282642089,
|
|
"grad_norm": 0.36500608380632915,
|
|
"learning_rate": 4.0586059206992215e-05,
|
|
"loss": 0.5177,
|
|
"num_tokens": 189580008.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.34050179211469533,
|
|
"grad_norm": 0.39983646639575704,
|
|
"learning_rate": 4.043060261781756e-05,
|
|
"loss": 0.5227,
|
|
"num_tokens": 191045650.0,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.3430619559651818,
|
|
"grad_norm": 0.3734390230199649,
|
|
"learning_rate": 4.02742188400151e-05,
|
|
"loss": 0.5167,
|
|
"num_tokens": 192504425.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.3456221198156682,
|
|
"grad_norm": 0.37319999637961876,
|
|
"learning_rate": 4.011691908708079e-05,
|
|
"loss": 0.5265,
|
|
"num_tokens": 193911623.0,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.3481822836661546,
|
|
"grad_norm": 0.37651097648209897,
|
|
"learning_rate": 3.995871463819059e-05,
|
|
"loss": 0.5244,
|
|
"num_tokens": 195367749.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.3507424475166411,
|
|
"grad_norm": 0.3543241861639982,
|
|
"learning_rate": 3.9799616837391654e-05,
|
|
"loss": 0.5039,
|
|
"num_tokens": 196798616.0,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.3533026113671275,
|
|
"grad_norm": 0.36016558697740264,
|
|
"learning_rate": 3.9639637092788936e-05,
|
|
"loss": 0.5276,
|
|
"num_tokens": 198231053.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.35586277521761395,
|
|
"grad_norm": 0.3544365991223893,
|
|
"learning_rate": 3.947878687572712e-05,
|
|
"loss": 0.5098,
|
|
"num_tokens": 199699326.0,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.35842293906810035,
|
|
"grad_norm": 0.3837987899904068,
|
|
"learning_rate": 3.9317077719968146e-05,
|
|
"loss": 0.5452,
|
|
"num_tokens": 201102127.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.36098310291858676,
|
|
"grad_norm": 0.41621003825962916,
|
|
"learning_rate": 3.9154521220864095e-05,
|
|
"loss": 0.5296,
|
|
"num_tokens": 202475799.0,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.3635432667690732,
|
|
"grad_norm": 0.35879726336225265,
|
|
"learning_rate": 3.899112903452582e-05,
|
|
"loss": 0.5044,
|
|
"num_tokens": 203775449.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.36610343061955963,
|
|
"grad_norm": 0.3990769821383768,
|
|
"learning_rate": 3.8826912876987094e-05,
|
|
"loss": 0.5277,
|
|
"num_tokens": 205270172.0,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.3686635944700461,
|
|
"grad_norm": 0.34721481700475765,
|
|
"learning_rate": 3.866188452336454e-05,
|
|
"loss": 0.5163,
|
|
"num_tokens": 206761213.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.3712237583205325,
|
|
"grad_norm": 0.3492201310638506,
|
|
"learning_rate": 3.849605580701327e-05,
|
|
"loss": 0.5194,
|
|
"num_tokens": 208239844.0,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.37378392217101897,
|
|
"grad_norm": 0.32682932421177585,
|
|
"learning_rate": 3.8329438618678425e-05,
|
|
"loss": 0.5098,
|
|
"num_tokens": 209671527.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.3763440860215054,
|
|
"grad_norm": 0.3282377869047365,
|
|
"learning_rate": 3.816204490564247e-05,
|
|
"loss": 0.5174,
|
|
"num_tokens": 211033102.0,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.3789042498719918,
|
|
"grad_norm": 0.39929588311220043,
|
|
"learning_rate": 3.79938866708686e-05,
|
|
"loss": 0.5278,
|
|
"num_tokens": 212446145.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.38146441372247825,
|
|
"grad_norm": 0.35602714397584617,
|
|
"learning_rate": 3.782497597214e-05,
|
|
"loss": 0.511,
|
|
"num_tokens": 213808174.0,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.38402457757296465,
|
|
"grad_norm": 0.3771251981999951,
|
|
"learning_rate": 3.7655324921195294e-05,
|
|
"loss": 0.5316,
|
|
"num_tokens": 215271195.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.3865847414234511,
|
|
"grad_norm": 0.39440755338940503,
|
|
"learning_rate": 3.748494568286002e-05,
|
|
"loss": 0.5323,
|
|
"num_tokens": 216725313.0,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.3891449052739375,
|
|
"grad_norm": 0.3500877771353314,
|
|
"learning_rate": 3.7313850474174396e-05,
|
|
"loss": 0.5182,
|
|
"num_tokens": 218144024.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.391705069124424,
|
|
"grad_norm": 0.35378795501468535,
|
|
"learning_rate": 3.714205156351729e-05,
|
|
"loss": 0.5226,
|
|
"num_tokens": 219640135.0,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.3942652329749104,
|
|
"grad_norm": 0.3278555645253241,
|
|
"learning_rate": 3.696956126972648e-05,
|
|
"loss": 0.5148,
|
|
"num_tokens": 221095532.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.3968253968253968,
|
|
"grad_norm": 0.339664329389688,
|
|
"learning_rate": 3.6796391961215395e-05,
|
|
"loss": 0.5291,
|
|
"num_tokens": 222548049.0,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.39938556067588327,
|
|
"grad_norm": 0.34000563116050575,
|
|
"learning_rate": 3.662255605508617e-05,
|
|
"loss": 0.5012,
|
|
"num_tokens": 223892351.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.4019457245263697,
|
|
"grad_norm": 0.32313400641464557,
|
|
"learning_rate": 3.644806601623932e-05,
|
|
"loss": 0.5191,
|
|
"num_tokens": 225310024.0,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.40450588837685614,
|
|
"grad_norm": 0.31743221548414213,
|
|
"learning_rate": 3.627293435647994e-05,
|
|
"loss": 0.4996,
|
|
"num_tokens": 226712054.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.40706605222734255,
|
|
"grad_norm": 0.33409744382517903,
|
|
"learning_rate": 3.609717363362054e-05,
|
|
"loss": 0.518,
|
|
"num_tokens": 228214245.0,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.409626216077829,
|
|
"grad_norm": 0.31812008883160964,
|
|
"learning_rate": 3.59207964505806e-05,
|
|
"loss": 0.5156,
|
|
"num_tokens": 229710487.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.4121863799283154,
|
|
"grad_norm": 0.34727764385831694,
|
|
"learning_rate": 3.5743815454482835e-05,
|
|
"loss": 0.5168,
|
|
"num_tokens": 231162439.0,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.4147465437788018,
|
|
"grad_norm": 0.3298989605094125,
|
|
"learning_rate": 3.556624333574639e-05,
|
|
"loss": 0.5245,
|
|
"num_tokens": 232659926.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.4173067076292883,
|
|
"grad_norm": 0.3439524425429194,
|
|
"learning_rate": 3.5388092827176834e-05,
|
|
"loss": 0.5324,
|
|
"num_tokens": 234109138.0,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.4198668714797747,
|
|
"grad_norm": 0.3717699302323384,
|
|
"learning_rate": 3.520937670305318e-05,
|
|
"loss": 0.5241,
|
|
"num_tokens": 235464027.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.42242703533026116,
|
|
"grad_norm": 0.35395801687334566,
|
|
"learning_rate": 3.5030107778211894e-05,
|
|
"loss": 0.5233,
|
|
"num_tokens": 236975716.0,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.42498719918074757,
|
|
"grad_norm": 0.3315275770907257,
|
|
"learning_rate": 3.4850298907128e-05,
|
|
"loss": 0.514,
|
|
"num_tokens": 238339259.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.427547363031234,
|
|
"grad_norm": 0.3924875343629337,
|
|
"learning_rate": 3.466996298299336e-05,
|
|
"loss": 0.5232,
|
|
"num_tokens": 239771501.0,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.43010752688172044,
|
|
"grad_norm": 0.35873280382329653,
|
|
"learning_rate": 3.4489112936792176e-05,
|
|
"loss": 0.524,
|
|
"num_tokens": 241114261.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.43266769073220684,
|
|
"grad_norm": 0.33628365229190205,
|
|
"learning_rate": 3.430776173637372e-05,
|
|
"loss": 0.5367,
|
|
"num_tokens": 242532270.0,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.4352278545826933,
|
|
"grad_norm": 0.32729602596480545,
|
|
"learning_rate": 3.412592238552259e-05,
|
|
"loss": 0.5216,
|
|
"num_tokens": 244012374.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.4377880184331797,
|
|
"grad_norm": 0.36753201882428965,
|
|
"learning_rate": 3.394360792302614e-05,
|
|
"loss": 0.5017,
|
|
"num_tokens": 245526117.0,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.4403481822836662,
|
|
"grad_norm": 0.3575294299962963,
|
|
"learning_rate": 3.3760831421739644e-05,
|
|
"loss": 0.5445,
|
|
"num_tokens": 246976387.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.4429083461341526,
|
|
"grad_norm": 0.35982251774954194,
|
|
"learning_rate": 3.35776059876488e-05,
|
|
"loss": 0.5054,
|
|
"num_tokens": 248334101.0,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.445468509984639,
|
|
"grad_norm": 0.34561318189843604,
|
|
"learning_rate": 3.339394475893008e-05,
|
|
"loss": 0.5174,
|
|
"num_tokens": 249695184.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.44802867383512546,
|
|
"grad_norm": 0.3831459355264965,
|
|
"learning_rate": 3.3209860905008584e-05,
|
|
"loss": 0.517,
|
|
"num_tokens": 251110352.0,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.45058883768561186,
|
|
"grad_norm": 0.3407664273554093,
|
|
"learning_rate": 3.302536762561374e-05,
|
|
"loss": 0.5045,
|
|
"num_tokens": 252607265.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.45314900153609833,
|
|
"grad_norm": 0.3271851932583608,
|
|
"learning_rate": 3.2840478149832815e-05,
|
|
"loss": 0.522,
|
|
"num_tokens": 254005194.0,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.45570916538658474,
|
|
"grad_norm": 0.3318362999795959,
|
|
"learning_rate": 3.265520573516235e-05,
|
|
"loss": 0.5132,
|
|
"num_tokens": 255433548.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.4582693292370712,
|
|
"grad_norm": 0.3206257789542918,
|
|
"learning_rate": 3.24695636665575e-05,
|
|
"loss": 0.5128,
|
|
"num_tokens": 256884308.0,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.4608294930875576,
|
|
"grad_norm": 0.3067741802742858,
|
|
"learning_rate": 3.228356525547944e-05,
|
|
"loss": 0.5028,
|
|
"num_tokens": 258396985.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.463389656938044,
|
|
"grad_norm": 0.3234640842724153,
|
|
"learning_rate": 3.20972238389409e-05,
|
|
"loss": 0.5132,
|
|
"num_tokens": 259903241.0,
|
|
"step": 905
|
|
},
|
|
{
|
|
"epoch": 0.4659498207885305,
|
|
"grad_norm": 0.3179488285662874,
|
|
"learning_rate": 3.191055277854979e-05,
|
|
"loss": 0.5147,
|
|
"num_tokens": 261399686.0,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.4685099846390169,
|
|
"grad_norm": 0.32545955872592386,
|
|
"learning_rate": 3.172356545955113e-05,
|
|
"loss": 0.5002,
|
|
"num_tokens": 262792408.0,
|
|
"step": 915
|
|
},
|
|
{
|
|
"epoch": 0.47107014848950335,
|
|
"grad_norm": 0.3521143288987651,
|
|
"learning_rate": 3.153627528986727e-05,
|
|
"loss": 0.5122,
|
|
"num_tokens": 264287905.0,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.47363031233998976,
|
|
"grad_norm": 0.32376661882443836,
|
|
"learning_rate": 3.134869569913645e-05,
|
|
"loss": 0.5099,
|
|
"num_tokens": 265682500.0,
|
|
"step": 925
|
|
},
|
|
{
|
|
"epoch": 0.47619047619047616,
|
|
"grad_norm": 0.3191083554096279,
|
|
"learning_rate": 3.116084013774984e-05,
|
|
"loss": 0.5163,
|
|
"num_tokens": 267147235.0,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.4787506400409626,
|
|
"grad_norm": 0.32980849411874075,
|
|
"learning_rate": 3.09727220758871e-05,
|
|
"loss": 0.5177,
|
|
"num_tokens": 268586746.0,
|
|
"step": 935
|
|
},
|
|
{
|
|
"epoch": 0.48131080389144903,
|
|
"grad_norm": 0.3206498658390865,
|
|
"learning_rate": 3.078435500255044e-05,
|
|
"loss": 0.5219,
|
|
"num_tokens": 270033989.0,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.4838709677419355,
|
|
"grad_norm": 0.3666925279403229,
|
|
"learning_rate": 3.059575242459749e-05,
|
|
"loss": 0.5274,
|
|
"num_tokens": 271512097.0,
|
|
"step": 945
|
|
},
|
|
{
|
|
"epoch": 0.4864311315924219,
|
|
"grad_norm": 0.3508651661356095,
|
|
"learning_rate": 3.040692786577269e-05,
|
|
"loss": 0.5146,
|
|
"num_tokens": 273006318.0,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.48899129544290837,
|
|
"grad_norm": 0.3474386441840942,
|
|
"learning_rate": 3.0217894865737628e-05,
|
|
"loss": 0.5338,
|
|
"num_tokens": 274477598.0,
|
|
"step": 955
|
|
},
|
|
{
|
|
"epoch": 0.4915514592933948,
|
|
"grad_norm": 0.346132856025321,
|
|
"learning_rate": 3.002866697910018e-05,
|
|
"loss": 0.5205,
|
|
"num_tokens": 275852045.0,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.4941116231438812,
|
|
"grad_norm": 0.31098729529769065,
|
|
"learning_rate": 2.9839257774442535e-05,
|
|
"loss": 0.49,
|
|
"num_tokens": 277166569.0,
|
|
"step": 965
|
|
},
|
|
{
|
|
"epoch": 0.49667178699436765,
|
|
"grad_norm": 0.3066485254287687,
|
|
"learning_rate": 2.9649680833348304e-05,
|
|
"loss": 0.5129,
|
|
"num_tokens": 278609285.0,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.49923195084485406,
|
|
"grad_norm": 0.3410424065006403,
|
|
"learning_rate": 2.9459949749428646e-05,
|
|
"loss": 0.5076,
|
|
"num_tokens": 279994947.0,
|
|
"step": 975
|
|
},
|
|
{
|
|
"epoch": 0.5017921146953405,
|
|
"grad_norm": 0.32238531708864987,
|
|
"learning_rate": 2.92700781273475e-05,
|
|
"loss": 0.4997,
|
|
"num_tokens": 281456293.0,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.504352278545827,
|
|
"grad_norm": 0.375723416219021,
|
|
"learning_rate": 2.908007958184611e-05,
|
|
"loss": 0.5256,
|
|
"num_tokens": 282971910.0,
|
|
"step": 985
|
|
},
|
|
{
|
|
"epoch": 0.5069124423963134,
|
|
"grad_norm": 0.33727932139243444,
|
|
"learning_rate": 2.888996773676677e-05,
|
|
"loss": 0.4812,
|
|
"num_tokens": 284383940.0,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.5094726062467998,
|
|
"grad_norm": 0.3204308728769862,
|
|
"learning_rate": 2.8699756224075897e-05,
|
|
"loss": 0.5246,
|
|
"num_tokens": 285937767.0,
|
|
"step": 995
|
|
},
|
|
{
|
|
"epoch": 0.5120327700972862,
|
|
"grad_norm": 0.34420963723916087,
|
|
"learning_rate": 2.8509458682886592e-05,
|
|
"loss": 0.518,
|
|
"num_tokens": 287371918.0,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.5145929339477726,
|
|
"grad_norm": 0.3281576650939101,
|
|
"learning_rate": 2.83190887584806e-05,
|
|
"loss": 0.4844,
|
|
"num_tokens": 288748407.0,
|
|
"step": 1005
|
|
},
|
|
{
|
|
"epoch": 0.5171530977982591,
|
|
"grad_norm": 0.332521773525128,
|
|
"learning_rate": 2.812866010132995e-05,
|
|
"loss": 0.5094,
|
|
"num_tokens": 290188857.0,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.5197132616487455,
|
|
"grad_norm": 0.3415251603678356,
|
|
"learning_rate": 2.7938186366118046e-05,
|
|
"loss": 0.5234,
|
|
"num_tokens": 291515165.0,
|
|
"step": 1015
|
|
},
|
|
{
|
|
"epoch": 0.522273425499232,
|
|
"grad_norm": 0.32030275503920985,
|
|
"learning_rate": 2.7747681210760652e-05,
|
|
"loss": 0.501,
|
|
"num_tokens": 293034663.0,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.5248335893497184,
|
|
"grad_norm": 0.33937578466378343,
|
|
"learning_rate": 2.7557158295426504e-05,
|
|
"loss": 0.5057,
|
|
"num_tokens": 294414550.0,
|
|
"step": 1025
|
|
},
|
|
{
|
|
"epoch": 0.5273937532002048,
|
|
"grad_norm": 0.343423192960333,
|
|
"learning_rate": 2.7366631281557815e-05,
|
|
"loss": 0.5119,
|
|
"num_tokens": 295849815.0,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.5299539170506913,
|
|
"grad_norm": 0.33086278901966193,
|
|
"learning_rate": 2.7176113830890676e-05,
|
|
"loss": 0.5119,
|
|
"num_tokens": 297329114.0,
|
|
"step": 1035
|
|
},
|
|
{
|
|
"epoch": 0.5325140809011777,
|
|
"grad_norm": 0.3321917897604643,
|
|
"learning_rate": 2.6985619604475464e-05,
|
|
"loss": 0.5055,
|
|
"num_tokens": 298677895.0,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.5350742447516641,
|
|
"grad_norm": 0.32156383235854147,
|
|
"learning_rate": 2.6795162261697243e-05,
|
|
"loss": 0.5102,
|
|
"num_tokens": 300155732.0,
|
|
"step": 1045
|
|
},
|
|
{
|
|
"epoch": 0.5376344086021505,
|
|
"grad_norm": 0.3376666853062569,
|
|
"learning_rate": 2.6604755459296358e-05,
|
|
"loss": 0.4918,
|
|
"num_tokens": 301439996.0,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.540194572452637,
|
|
"grad_norm": 24.52743811042255,
|
|
"learning_rate": 2.641441285038913e-05,
|
|
"loss": 0.4978,
|
|
"num_tokens": 302889563.0,
|
|
"step": 1055
|
|
},
|
|
{
|
|
"epoch": 0.5427547363031234,
|
|
"grad_norm": 0.31583429087173115,
|
|
"learning_rate": 2.622414808348892e-05,
|
|
"loss": 0.5147,
|
|
"num_tokens": 304293467.0,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.5453149001536098,
|
|
"grad_norm": 0.3025080495010937,
|
|
"learning_rate": 2.60339748015274e-05,
|
|
"loss": 0.5206,
|
|
"num_tokens": 305733918.0,
|
|
"step": 1065
|
|
},
|
|
{
|
|
"epoch": 0.5478750640040962,
|
|
"grad_norm": 0.3401001846477805,
|
|
"learning_rate": 2.5843906640876324e-05,
|
|
"loss": 0.521,
|
|
"num_tokens": 307101894.0,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.5504352278545827,
|
|
"grad_norm": 0.36697600951920656,
|
|
"learning_rate": 2.565395723036975e-05,
|
|
"loss": 0.5147,
|
|
"num_tokens": 308503184.0,
|
|
"step": 1075
|
|
},
|
|
{
|
|
"epoch": 0.5529953917050692,
|
|
"grad_norm": 0.3377142425790515,
|
|
"learning_rate": 2.5464140190326713e-05,
|
|
"loss": 0.5016,
|
|
"num_tokens": 309952008.0,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.5555555555555556,
|
|
"grad_norm": 0.32847589022838863,
|
|
"learning_rate": 2.5274469131574668e-05,
|
|
"loss": 0.5008,
|
|
"num_tokens": 311381277.0,
|
|
"step": 1085
|
|
},
|
|
{
|
|
"epoch": 0.558115719406042,
|
|
"grad_norm": 0.3369602267811668,
|
|
"learning_rate": 2.508495765447345e-05,
|
|
"loss": 0.5024,
|
|
"num_tokens": 312757412.0,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.5606758832565284,
|
|
"grad_norm": 0.33572534746555616,
|
|
"learning_rate": 2.489561934794012e-05,
|
|
"loss": 0.4971,
|
|
"num_tokens": 314240646.0,
|
|
"step": 1095
|
|
},
|
|
{
|
|
"epoch": 0.5632360471070148,
|
|
"grad_norm": 0.32431813833704604,
|
|
"learning_rate": 2.4706467788474545e-05,
|
|
"loss": 0.5017,
|
|
"num_tokens": 315690477.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.5657962109575013,
|
|
"grad_norm": 0.339051619694251,
|
|
"learning_rate": 2.4517516539185874e-05,
|
|
"loss": 0.5115,
|
|
"num_tokens": 317116993.0,
|
|
"step": 1105
|
|
},
|
|
{
|
|
"epoch": 0.5683563748079877,
|
|
"grad_norm": 0.3178829911696031,
|
|
"learning_rate": 2.4328779148820052e-05,
|
|
"loss": 0.5044,
|
|
"num_tokens": 318525249.0,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.5709165386584741,
|
|
"grad_norm": 0.32988852761069204,
|
|
"learning_rate": 2.4140269150788234e-05,
|
|
"loss": 0.5025,
|
|
"num_tokens": 319919730.0,
|
|
"step": 1115
|
|
},
|
|
{
|
|
"epoch": 0.5734767025089605,
|
|
"grad_norm": 0.31864247317808075,
|
|
"learning_rate": 2.3952000062196457e-05,
|
|
"loss": 0.5057,
|
|
"num_tokens": 321410101.0,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.576036866359447,
|
|
"grad_norm": 0.322237539555902,
|
|
"learning_rate": 2.3763985382876308e-05,
|
|
"loss": 0.494,
|
|
"num_tokens": 322800384.0,
|
|
"step": 1125
|
|
},
|
|
{
|
|
"epoch": 0.5785970302099335,
|
|
"grad_norm": 0.327704612338539,
|
|
"learning_rate": 2.3576238594416982e-05,
|
|
"loss": 0.5088,
|
|
"num_tokens": 324223484.0,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.5811571940604199,
|
|
"grad_norm": 0.3273678473775592,
|
|
"learning_rate": 2.3388773159198584e-05,
|
|
"loss": 0.4849,
|
|
"num_tokens": 325542109.0,
|
|
"step": 1135
|
|
},
|
|
{
|
|
"epoch": 0.5837173579109063,
|
|
"grad_norm": 0.35657134235847027,
|
|
"learning_rate": 2.3201602519426753e-05,
|
|
"loss": 0.5186,
|
|
"num_tokens": 326903105.0,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.5862775217613927,
|
|
"grad_norm": 0.3268149668552262,
|
|
"learning_rate": 2.3014740096168852e-05,
|
|
"loss": 0.505,
|
|
"num_tokens": 328412100.0,
|
|
"step": 1145
|
|
},
|
|
{
|
|
"epoch": 0.5888376856118792,
|
|
"grad_norm": 0.31391147589489,
|
|
"learning_rate": 2.2828199288391554e-05,
|
|
"loss": 0.505,
|
|
"num_tokens": 329868872.0,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.5913978494623656,
|
|
"grad_norm": 0.30279044804465655,
|
|
"learning_rate": 2.264199347200015e-05,
|
|
"loss": 0.4859,
|
|
"num_tokens": 331334112.0,
|
|
"step": 1155
|
|
},
|
|
{
|
|
"epoch": 0.593958013312852,
|
|
"grad_norm": 0.30637014426590536,
|
|
"learning_rate": 2.2456135998879337e-05,
|
|
"loss": 0.4909,
|
|
"num_tokens": 332767044.0,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.5965181771633384,
|
|
"grad_norm": 0.3143728562152214,
|
|
"learning_rate": 2.227064019593586e-05,
|
|
"loss": 0.5025,
|
|
"num_tokens": 334217974.0,
|
|
"step": 1165
|
|
},
|
|
{
|
|
"epoch": 0.5990783410138248,
|
|
"grad_norm": 0.3148463439977503,
|
|
"learning_rate": 2.2085519364142963e-05,
|
|
"loss": 0.4945,
|
|
"num_tokens": 335631960.0,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.6016385048643114,
|
|
"grad_norm": 0.30199220017487055,
|
|
"learning_rate": 2.1900786777586536e-05,
|
|
"loss": 0.4719,
|
|
"num_tokens": 337027727.0,
|
|
"step": 1175
|
|
},
|
|
{
|
|
"epoch": 0.6041986687147978,
|
|
"grad_norm": 0.31274180205589397,
|
|
"learning_rate": 2.171645568251337e-05,
|
|
"loss": 0.5054,
|
|
"num_tokens": 338382273.0,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.6067588325652842,
|
|
"grad_norm": 0.331592184963782,
|
|
"learning_rate": 2.1532539296381315e-05,
|
|
"loss": 0.5105,
|
|
"num_tokens": 339817793.0,
|
|
"step": 1185
|
|
},
|
|
{
|
|
"epoch": 0.6093189964157706,
|
|
"grad_norm": 0.32732315160077313,
|
|
"learning_rate": 2.1349050806911536e-05,
|
|
"loss": 0.5163,
|
|
"num_tokens": 341292117.0,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.611879160266257,
|
|
"grad_norm": 0.3274853237739203,
|
|
"learning_rate": 2.1166003371142856e-05,
|
|
"loss": 0.4884,
|
|
"num_tokens": 342665229.0,
|
|
"step": 1195
|
|
},
|
|
{
|
|
"epoch": 0.6144393241167435,
|
|
"grad_norm": 0.33470676147491346,
|
|
"learning_rate": 2.0983410114488365e-05,
|
|
"loss": 0.5032,
|
|
"num_tokens": 344239058.0,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.6169994879672299,
|
|
"grad_norm": 0.33258168348539047,
|
|
"learning_rate": 2.0801284129794253e-05,
|
|
"loss": 0.5159,
|
|
"num_tokens": 345743548.0,
|
|
"step": 1205
|
|
},
|
|
{
|
|
"epoch": 0.6195596518177163,
|
|
"grad_norm": 0.2869944507757293,
|
|
"learning_rate": 2.0619638476400944e-05,
|
|
"loss": 0.5001,
|
|
"num_tokens": 347162272.0,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.6221198156682027,
|
|
"grad_norm": 0.30571480808679774,
|
|
"learning_rate": 2.0438486179206773e-05,
|
|
"loss": 0.509,
|
|
"num_tokens": 348546634.0,
|
|
"step": 1215
|
|
},
|
|
{
|
|
"epoch": 0.6246799795186891,
|
|
"grad_norm": 0.30799618343717494,
|
|
"learning_rate": 2.0257840227733926e-05,
|
|
"loss": 0.5053,
|
|
"num_tokens": 349930176.0,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.6272401433691757,
|
|
"grad_norm": 0.300862581280986,
|
|
"learning_rate": 2.0077713575197115e-05,
|
|
"loss": 0.4963,
|
|
"num_tokens": 351357678.0,
|
|
"step": 1225
|
|
},
|
|
{
|
|
"epoch": 0.6298003072196621,
|
|
"grad_norm": 0.3307532985588016,
|
|
"learning_rate": 1.98981191375747e-05,
|
|
"loss": 0.4803,
|
|
"num_tokens": 352776042.0,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.6323604710701485,
|
|
"grad_norm": 0.29549726088263856,
|
|
"learning_rate": 1.9719069792682605e-05,
|
|
"loss": 0.5026,
|
|
"num_tokens": 354221560.0,
|
|
"step": 1235
|
|
},
|
|
{
|
|
"epoch": 0.6349206349206349,
|
|
"grad_norm": 0.30737613147546144,
|
|
"learning_rate": 1.954057837925089e-05,
|
|
"loss": 0.4884,
|
|
"num_tokens": 355639183.0,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.6374807987711214,
|
|
"grad_norm": 0.3434881969758962,
|
|
"learning_rate": 1.9362657696003116e-05,
|
|
"loss": 0.5032,
|
|
"num_tokens": 357029839.0,
|
|
"step": 1245
|
|
},
|
|
{
|
|
"epoch": 0.6400409626216078,
|
|
"grad_norm": 0.30449489663457785,
|
|
"learning_rate": 1.91853205007387e-05,
|
|
"loss": 0.5221,
|
|
"num_tokens": 358490723.0,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.6426011264720942,
|
|
"grad_norm": 0.29564416296357837,
|
|
"learning_rate": 1.9008579509418017e-05,
|
|
"loss": 0.4934,
|
|
"num_tokens": 359939876.0,
|
|
"step": 1255
|
|
},
|
|
{
|
|
"epoch": 0.6451612903225806,
|
|
"grad_norm": 0.31602372659073663,
|
|
"learning_rate": 1.8832447395250657e-05,
|
|
"loss": 0.4946,
|
|
"num_tokens": 361329513.0,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.647721454173067,
|
|
"grad_norm": 0.32121231141149115,
|
|
"learning_rate": 1.8656936787786722e-05,
|
|
"loss": 0.4935,
|
|
"num_tokens": 362696180.0,
|
|
"step": 1265
|
|
},
|
|
{
|
|
"epoch": 0.6502816180235536,
|
|
"grad_norm": 0.33989353727165256,
|
|
"learning_rate": 1.8482060272011137e-05,
|
|
"loss": 0.499,
|
|
"num_tokens": 364110130.0,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.65284178187404,
|
|
"grad_norm": 0.2970519780305765,
|
|
"learning_rate": 1.830783038744132e-05,
|
|
"loss": 0.4885,
|
|
"num_tokens": 365514096.0,
|
|
"step": 1275
|
|
},
|
|
{
|
|
"epoch": 0.6554019457245264,
|
|
"grad_norm": 0.30104331629462844,
|
|
"learning_rate": 1.8134259627228007e-05,
|
|
"loss": 0.4814,
|
|
"num_tokens": 366863209.0,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.6579621095750128,
|
|
"grad_norm": 0.3008301278392379,
|
|
"learning_rate": 1.796136043725944e-05,
|
|
"loss": 0.5019,
|
|
"num_tokens": 368362600.0,
|
|
"step": 1285
|
|
},
|
|
{
|
|
"epoch": 0.6605222734254992,
|
|
"grad_norm": 0.3155236902871057,
|
|
"learning_rate": 1.7789145215268906e-05,
|
|
"loss": 0.5121,
|
|
"num_tokens": 369747920.0,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.6630824372759857,
|
|
"grad_norm": 0.30136184442479447,
|
|
"learning_rate": 1.7617626309945805e-05,
|
|
"loss": 0.4883,
|
|
"num_tokens": 371157438.0,
|
|
"step": 1295
|
|
},
|
|
{
|
|
"epoch": 0.6656426011264721,
|
|
"grad_norm": 0.3018130489276304,
|
|
"learning_rate": 1.7446816020050165e-05,
|
|
"loss": 0.4811,
|
|
"num_tokens": 372569301.0,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.6682027649769585,
|
|
"grad_norm": 0.32179054820119746,
|
|
"learning_rate": 1.727672659353072e-05,
|
|
"loss": 0.5046,
|
|
"num_tokens": 374038949.0,
|
|
"step": 1305
|
|
},
|
|
{
|
|
"epoch": 0.6707629288274449,
|
|
"grad_norm": 0.3208926155051817,
|
|
"learning_rate": 1.7107370226646765e-05,
|
|
"loss": 0.4916,
|
|
"num_tokens": 375512046.0,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.6733230926779313,
|
|
"grad_norm": 0.3038488866319765,
|
|
"learning_rate": 1.6938759063093508e-05,
|
|
"loss": 0.4715,
|
|
"num_tokens": 376925423.0,
|
|
"step": 1315
|
|
},
|
|
{
|
|
"epoch": 0.6758832565284179,
|
|
"grad_norm": 0.32157519864227907,
|
|
"learning_rate": 1.6770905193131426e-05,
|
|
"loss": 0.4881,
|
|
"num_tokens": 378330489.0,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.6784434203789043,
|
|
"grad_norm": 0.3003661648491186,
|
|
"learning_rate": 1.6603820652719215e-05,
|
|
"loss": 0.4731,
|
|
"num_tokens": 379761945.0,
|
|
"step": 1325
|
|
},
|
|
{
|
|
"epoch": 0.6810035842293907,
|
|
"grad_norm": 0.3294880462597273,
|
|
"learning_rate": 1.643751742265084e-05,
|
|
"loss": 0.4947,
|
|
"num_tokens": 381147994.0,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.6835637480798771,
|
|
"grad_norm": 0.32768994451253647,
|
|
"learning_rate": 1.627200742769644e-05,
|
|
"loss": 0.5066,
|
|
"num_tokens": 382656408.0,
|
|
"step": 1335
|
|
},
|
|
{
|
|
"epoch": 0.6861239119303636,
|
|
"grad_norm": 0.35664636068246003,
|
|
"learning_rate": 1.6107302535747194e-05,
|
|
"loss": 0.502,
|
|
"num_tokens": 384011945.0,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.68868407578085,
|
|
"grad_norm": 0.32167567481927783,
|
|
"learning_rate": 1.5943414556964423e-05,
|
|
"loss": 0.4773,
|
|
"num_tokens": 385410611.0,
|
|
"step": 1345
|
|
},
|
|
{
|
|
"epoch": 0.6912442396313364,
|
|
"grad_norm": 0.3008589470969583,
|
|
"learning_rate": 1.5780355242932694e-05,
|
|
"loss": 0.5025,
|
|
"num_tokens": 386806367.0,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.6938044034818228,
|
|
"grad_norm": 0.3061659756831739,
|
|
"learning_rate": 1.5618136285817188e-05,
|
|
"loss": 0.4902,
|
|
"num_tokens": 388162723.0,
|
|
"step": 1355
|
|
},
|
|
{
|
|
"epoch": 0.6963645673323092,
|
|
"grad_norm": 0.2995901651838368,
|
|
"learning_rate": 1.5456769317525292e-05,
|
|
"loss": 0.4891,
|
|
"num_tokens": 389586680.0,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.6989247311827957,
|
|
"grad_norm": 0.2992449174863845,
|
|
"learning_rate": 1.5296265908872575e-05,
|
|
"loss": 0.4988,
|
|
"num_tokens": 391084870.0,
|
|
"step": 1365
|
|
},
|
|
{
|
|
"epoch": 0.7014848950332822,
|
|
"grad_norm": 0.29255637740631535,
|
|
"learning_rate": 1.5136637568753054e-05,
|
|
"loss": 0.4998,
|
|
"num_tokens": 392562290.0,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.7040450588837686,
|
|
"grad_norm": 0.31902059267199107,
|
|
"learning_rate": 1.4977895743313982e-05,
|
|
"loss": 0.4926,
|
|
"num_tokens": 393877019.0,
|
|
"step": 1375
|
|
},
|
|
{
|
|
"epoch": 0.706605222734255,
|
|
"grad_norm": 0.3271510883584078,
|
|
"learning_rate": 1.4820051815135106e-05,
|
|
"loss": 0.5076,
|
|
"num_tokens": 395264025.0,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.7091653865847414,
|
|
"grad_norm": 0.3207752343511156,
|
|
"learning_rate": 1.4663117102412421e-05,
|
|
"loss": 0.5064,
|
|
"num_tokens": 396724050.0,
|
|
"step": 1385
|
|
},
|
|
{
|
|
"epoch": 0.7117255504352279,
|
|
"grad_norm": 0.3239779535731452,
|
|
"learning_rate": 1.450710285814671e-05,
|
|
"loss": 0.5028,
|
|
"num_tokens": 398144482.0,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.7142857142857143,
|
|
"grad_norm": 0.30927431616207984,
|
|
"learning_rate": 1.4352020269336544e-05,
|
|
"loss": 0.4854,
|
|
"num_tokens": 399518681.0,
|
|
"step": 1395
|
|
},
|
|
{
|
|
"epoch": 0.7168458781362007,
|
|
"grad_norm": 0.295576415194351,
|
|
"learning_rate": 1.4197880456176133e-05,
|
|
"loss": 0.5032,
|
|
"num_tokens": 400923938.0,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.7194060419866871,
|
|
"grad_norm": 0.3304348733503482,
|
|
"learning_rate": 1.404469447125804e-05,
|
|
"loss": 0.4849,
|
|
"num_tokens": 402276977.0,
|
|
"step": 1405
|
|
},
|
|
{
|
|
"epoch": 0.7219662058371735,
|
|
"grad_norm": 0.31414043159806515,
|
|
"learning_rate": 1.3892473298780529e-05,
|
|
"loss": 0.4875,
|
|
"num_tokens": 403697414.0,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.72452636968766,
|
|
"grad_norm": 0.293707592797367,
|
|
"learning_rate": 1.3741227853760036e-05,
|
|
"loss": 0.4869,
|
|
"num_tokens": 405184791.0,
|
|
"step": 1415
|
|
},
|
|
{
|
|
"epoch": 0.7270865335381465,
|
|
"grad_norm": 0.2917931706359394,
|
|
"learning_rate": 1.3590968981248475e-05,
|
|
"loss": 0.4972,
|
|
"num_tokens": 406613744.0,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.7296466973886329,
|
|
"grad_norm": 0.2925047196736336,
|
|
"learning_rate": 1.3441707455555586e-05,
|
|
"loss": 0.4887,
|
|
"num_tokens": 408108512.0,
|
|
"step": 1425
|
|
},
|
|
{
|
|
"epoch": 0.7322068612391193,
|
|
"grad_norm": 0.31031812911699463,
|
|
"learning_rate": 1.3293453979476403e-05,
|
|
"loss": 0.5044,
|
|
"num_tokens": 409481975.0,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.7347670250896058,
|
|
"grad_norm": 0.27792874846246257,
|
|
"learning_rate": 1.3146219183523733e-05,
|
|
"loss": 0.4971,
|
|
"num_tokens": 410936746.0,
|
|
"step": 1435
|
|
},
|
|
{
|
|
"epoch": 0.7373271889400922,
|
|
"grad_norm": 0.30336750716441363,
|
|
"learning_rate": 1.300001362516598e-05,
|
|
"loss": 0.4954,
|
|
"num_tokens": 412386009.0,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.7398873527905786,
|
|
"grad_norm": 0.3141965578167415,
|
|
"learning_rate": 1.2854847788070062e-05,
|
|
"loss": 0.5113,
|
|
"num_tokens": 413894931.0,
|
|
"step": 1445
|
|
},
|
|
{
|
|
"epoch": 0.742447516641065,
|
|
"grad_norm": 0.29141158673021467,
|
|
"learning_rate": 1.2710732081349728e-05,
|
|
"loss": 0.4857,
|
|
"num_tokens": 415276073.0,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.7450076804915514,
|
|
"grad_norm": 0.30621570583092955,
|
|
"learning_rate": 1.2567676838819126e-05,
|
|
"loss": 0.5131,
|
|
"num_tokens": 416854522.0,
|
|
"step": 1455
|
|
},
|
|
{
|
|
"epoch": 0.7475678443420379,
|
|
"grad_norm": 0.2996375860658252,
|
|
"learning_rate": 1.242569231825184e-05,
|
|
"loss": 0.4815,
|
|
"num_tokens": 418265973.0,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.7501280081925243,
|
|
"grad_norm": 0.29742624077313945,
|
|
"learning_rate": 1.2284788700645366e-05,
|
|
"loss": 0.4729,
|
|
"num_tokens": 419579047.0,
|
|
"step": 1465
|
|
},
|
|
{
|
|
"epoch": 0.7526881720430108,
|
|
"grad_norm": 0.2876262413928932,
|
|
"learning_rate": 1.2144976089491066e-05,
|
|
"loss": 0.4931,
|
|
"num_tokens": 420993188.0,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.7552483358934972,
|
|
"grad_norm": 0.3003949442350352,
|
|
"learning_rate": 1.2006264510049709e-05,
|
|
"loss": 0.4814,
|
|
"num_tokens": 422319491.0,
|
|
"step": 1475
|
|
},
|
|
{
|
|
"epoch": 0.7578084997439836,
|
|
"grad_norm": 0.2998727379349409,
|
|
"learning_rate": 1.1868663908632584e-05,
|
|
"loss": 0.4877,
|
|
"num_tokens": 423685709.0,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.7603686635944701,
|
|
"grad_norm": 0.28529665031013146,
|
|
"learning_rate": 1.1732184151888361e-05,
|
|
"loss": 0.4792,
|
|
"num_tokens": 425094824.0,
|
|
"step": 1485
|
|
},
|
|
{
|
|
"epoch": 0.7629288274449565,
|
|
"grad_norm": 0.2871379807334584,
|
|
"learning_rate": 1.159683502609552e-05,
|
|
"loss": 0.4874,
|
|
"num_tokens": 426477121.0,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.7654889912954429,
|
|
"grad_norm": 0.30522496747710626,
|
|
"learning_rate": 1.1462626236460684e-05,
|
|
"loss": 0.4813,
|
|
"num_tokens": 427829811.0,
|
|
"step": 1495
|
|
},
|
|
{
|
|
"epoch": 0.7680491551459293,
|
|
"grad_norm": 0.2741051636776696,
|
|
"learning_rate": 1.1329567406422728e-05,
|
|
"loss": 0.4853,
|
|
"num_tokens": 429329969.0,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.7706093189964157,
|
|
"grad_norm": 0.2928440443692563,
|
|
"learning_rate": 1.119766807696263e-05,
|
|
"loss": 0.4828,
|
|
"num_tokens": 430812616.0,
|
|
"step": 1505
|
|
},
|
|
{
|
|
"epoch": 0.7731694828469022,
|
|
"grad_norm": 0.3099228470971491,
|
|
"learning_rate": 1.1066937705919447e-05,
|
|
"loss": 0.4866,
|
|
"num_tokens": 432223616.0,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.7757296466973886,
|
|
"grad_norm": 0.33444985175560565,
|
|
"learning_rate": 1.0937385667312079e-05,
|
|
"loss": 0.4916,
|
|
"num_tokens": 433695672.0,
|
|
"step": 1515
|
|
},
|
|
{
|
|
"epoch": 0.778289810547875,
|
|
"grad_norm": 0.3042082054817302,
|
|
"learning_rate": 1.080902125066712e-05,
|
|
"loss": 0.4739,
|
|
"num_tokens": 435077995.0,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.7808499743983615,
|
|
"grad_norm": 0.2877442262157545,
|
|
"learning_rate": 1.0681853660352754e-05,
|
|
"loss": 0.4896,
|
|
"num_tokens": 436598018.0,
|
|
"step": 1525
|
|
},
|
|
{
|
|
"epoch": 0.783410138248848,
|
|
"grad_norm": 0.29033199345344185,
|
|
"learning_rate": 1.0555892014918756e-05,
|
|
"loss": 0.4723,
|
|
"num_tokens": 437989065.0,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.7859703020993344,
|
|
"grad_norm": 0.26896626455528305,
|
|
"learning_rate": 1.043114534644264e-05,
|
|
"loss": 0.4978,
|
|
"num_tokens": 439592308.0,
|
|
"step": 1535
|
|
},
|
|
{
|
|
"epoch": 0.7885304659498208,
|
|
"grad_norm": 0.28027984282770046,
|
|
"learning_rate": 1.0307622599882024e-05,
|
|
"loss": 0.5006,
|
|
"num_tokens": 441123044.0,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.7910906298003072,
|
|
"grad_norm": 0.32557121721379073,
|
|
"learning_rate": 1.0185332632433225e-05,
|
|
"loss": 0.5,
|
|
"num_tokens": 442556480.0,
|
|
"step": 1545
|
|
},
|
|
{
|
|
"epoch": 0.7936507936507936,
|
|
"grad_norm": 0.29930273815229413,
|
|
"learning_rate": 1.0064284212896155e-05,
|
|
"loss": 0.4917,
|
|
"num_tokens": 443975813.0,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.7962109575012801,
|
|
"grad_norm": 0.2942854997758679,
|
|
"learning_rate": 9.94448602104556e-06,
|
|
"loss": 0.491,
|
|
"num_tokens": 445398183.0,
|
|
"step": 1555
|
|
},
|
|
{
|
|
"epoch": 0.7987711213517665,
|
|
"grad_norm": 0.3417049818433575,
|
|
"learning_rate": 9.825946647008625e-06,
|
|
"loss": 0.4838,
|
|
"num_tokens": 446852018.0,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.8013312852022529,
|
|
"grad_norm": 0.3110713639757364,
|
|
"learning_rate": 9.708674590649033e-06,
|
|
"loss": 0.4954,
|
|
"num_tokens": 448203015.0,
|
|
"step": 1565
|
|
},
|
|
{
|
|
"epoch": 0.8038914490527393,
|
|
"grad_norm": 0.2863245218320564,
|
|
"learning_rate": 9.592678260957477e-06,
|
|
"loss": 0.5004,
|
|
"num_tokens": 449677830.0,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.8064516129032258,
|
|
"grad_norm": 0.30623250453538414,
|
|
"learning_rate": 9.477965975448672e-06,
|
|
"loss": 0.5048,
|
|
"num_tokens": 451198726.0,
|
|
"step": 1575
|
|
},
|
|
{
|
|
"epoch": 0.8090117767537123,
|
|
"grad_norm": 0.31441211447411255,
|
|
"learning_rate": 9.364545959565008e-06,
|
|
"loss": 0.4853,
|
|
"num_tokens": 452550699.0,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.8115719406041987,
|
|
"grad_norm": 0.2899725376508166,
|
|
"learning_rate": 9.252426346086662e-06,
|
|
"loss": 0.4862,
|
|
"num_tokens": 454037549.0,
|
|
"step": 1585
|
|
},
|
|
{
|
|
"epoch": 0.8141321044546851,
|
|
"grad_norm": 0.2876448343744089,
|
|
"learning_rate": 9.141615174548498e-06,
|
|
"loss": 0.4831,
|
|
"num_tokens": 455448327.0,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.8166922683051715,
|
|
"grad_norm": 0.3150105189344012,
|
|
"learning_rate": 9.032120390663599e-06,
|
|
"loss": 0.473,
|
|
"num_tokens": 456841238.0,
|
|
"step": 1595
|
|
},
|
|
{
|
|
"epoch": 0.819252432155658,
|
|
"grad_norm": 0.3148609594442674,
|
|
"learning_rate": 8.923949845753446e-06,
|
|
"loss": 0.4994,
|
|
"num_tokens": 458325861.0,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.8218125960061444,
|
|
"grad_norm": 0.3163188507122811,
|
|
"learning_rate": 8.817111296185022e-06,
|
|
"loss": 0.4989,
|
|
"num_tokens": 459735693.0,
|
|
"step": 1605
|
|
},
|
|
{
|
|
"epoch": 0.8243727598566308,
|
|
"grad_norm": 0.29646225208261745,
|
|
"learning_rate": 8.711612402814601e-06,
|
|
"loss": 0.4993,
|
|
"num_tokens": 461096140.0,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.8269329237071172,
|
|
"grad_norm": 0.2882775663501845,
|
|
"learning_rate": 8.607460730438433e-06,
|
|
"loss": 0.4683,
|
|
"num_tokens": 462591927.0,
|
|
"step": 1615
|
|
},
|
|
{
|
|
"epoch": 0.8294930875576036,
|
|
"grad_norm": 0.2896126848753792,
|
|
"learning_rate": 8.504663747250308e-06,
|
|
"loss": 0.4758,
|
|
"num_tokens": 463988579.0,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.8320532514080902,
|
|
"grad_norm": 0.2709621924913574,
|
|
"learning_rate": 8.40322882430605e-06,
|
|
"loss": 0.4972,
|
|
"num_tokens": 465476544.0,
|
|
"step": 1625
|
|
},
|
|
{
|
|
"epoch": 0.8346134152585766,
|
|
"grad_norm": 0.28371639303121554,
|
|
"learning_rate": 8.30316323499499e-06,
|
|
"loss": 0.4923,
|
|
"num_tokens": 466970023.0,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.837173579109063,
|
|
"grad_norm": 0.30091947072806213,
|
|
"learning_rate": 8.204474154518401e-06,
|
|
"loss": 0.4657,
|
|
"num_tokens": 468358911.0,
|
|
"step": 1635
|
|
},
|
|
{
|
|
"epoch": 0.8397337429595494,
|
|
"grad_norm": 0.28915057217034223,
|
|
"learning_rate": 8.107168659375027e-06,
|
|
"loss": 0.4849,
|
|
"num_tokens": 469791498.0,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.8422939068100358,
|
|
"grad_norm": 0.28431593278850475,
|
|
"learning_rate": 8.011253726853635e-06,
|
|
"loss": 0.4785,
|
|
"num_tokens": 471149149.0,
|
|
"step": 1645
|
|
},
|
|
{
|
|
"epoch": 0.8448540706605223,
|
|
"grad_norm": 0.2856958772207184,
|
|
"learning_rate": 7.91673623453274e-06,
|
|
"loss": 0.4804,
|
|
"num_tokens": 472574525.0,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.8474142345110087,
|
|
"grad_norm": 0.2855350115269561,
|
|
"learning_rate": 7.823622959787428e-06,
|
|
"loss": 0.4897,
|
|
"num_tokens": 474105694.0,
|
|
"step": 1655
|
|
},
|
|
{
|
|
"epoch": 0.8499743983614951,
|
|
"grad_norm": 0.2868985556625894,
|
|
"learning_rate": 7.731920579303386e-06,
|
|
"loss": 0.4852,
|
|
"num_tokens": 475518680.0,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.8525345622119815,
|
|
"grad_norm": 0.2916500681957567,
|
|
"learning_rate": 7.641635668598158e-06,
|
|
"loss": 0.4985,
|
|
"num_tokens": 476914257.0,
|
|
"step": 1665
|
|
},
|
|
{
|
|
"epoch": 0.855094726062468,
|
|
"grad_norm": 0.2745676904872679,
|
|
"learning_rate": 7.552774701549626e-06,
|
|
"loss": 0.4968,
|
|
"num_tokens": 478362887.0,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.8576548899129545,
|
|
"grad_norm": 0.2745878531398645,
|
|
"learning_rate": 7.4653440499318535e-06,
|
|
"loss": 0.4949,
|
|
"num_tokens": 479928118.0,
|
|
"step": 1675
|
|
},
|
|
{
|
|
"epoch": 0.8602150537634409,
|
|
"grad_norm": 0.2821489873706457,
|
|
"learning_rate": 7.379349982958122e-06,
|
|
"loss": 0.4918,
|
|
"num_tokens": 481348892.0,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.8627752176139273,
|
|
"grad_norm": 0.271295044029271,
|
|
"learning_rate": 7.294798666831461e-06,
|
|
"loss": 0.4771,
|
|
"num_tokens": 482716247.0,
|
|
"step": 1685
|
|
},
|
|
{
|
|
"epoch": 0.8653353814644137,
|
|
"grad_norm": 0.28104573941056804,
|
|
"learning_rate": 7.211696164302477e-06,
|
|
"loss": 0.5012,
|
|
"num_tokens": 484233296.0,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.8678955453149002,
|
|
"grad_norm": 0.2957280428536685,
|
|
"learning_rate": 7.130048434234607e-06,
|
|
"loss": 0.5016,
|
|
"num_tokens": 485654939.0,
|
|
"step": 1695
|
|
},
|
|
{
|
|
"epoch": 0.8704557091653866,
|
|
"grad_norm": 0.29842573022629226,
|
|
"learning_rate": 7.04986133117687e-06,
|
|
"loss": 0.472,
|
|
"num_tokens": 487118187.0,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.873015873015873,
|
|
"grad_norm": 0.27752781431190215,
|
|
"learning_rate": 6.971140604944043e-06,
|
|
"loss": 0.4759,
|
|
"num_tokens": 488555283.0,
|
|
"step": 1705
|
|
},
|
|
{
|
|
"epoch": 0.8755760368663594,
|
|
"grad_norm": 0.28275812713137627,
|
|
"learning_rate": 6.893891900204386e-06,
|
|
"loss": 0.4771,
|
|
"num_tokens": 490023831.0,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.8781362007168458,
|
|
"grad_norm": 0.2724652278686122,
|
|
"learning_rate": 6.818120756074882e-06,
|
|
"loss": 0.4779,
|
|
"num_tokens": 491499071.0,
|
|
"step": 1715
|
|
},
|
|
{
|
|
"epoch": 0.8806963645673324,
|
|
"grad_norm": 0.30075082424153865,
|
|
"learning_rate": 6.7438326057240515e-06,
|
|
"loss": 0.4866,
|
|
"num_tokens": 492893029.0,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.8832565284178188,
|
|
"grad_norm": 0.3014573577017195,
|
|
"learning_rate": 6.671032775982385e-06,
|
|
"loss": 0.503,
|
|
"num_tokens": 494428864.0,
|
|
"step": 1725
|
|
},
|
|
{
|
|
"epoch": 0.8858166922683052,
|
|
"grad_norm": 0.269216412511789,
|
|
"learning_rate": 6.599726486960354e-06,
|
|
"loss": 0.4806,
|
|
"num_tokens": 495957666.0,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.8883768561187916,
|
|
"grad_norm": 0.300942238716259,
|
|
"learning_rate": 6.5299188516741356e-06,
|
|
"loss": 0.4782,
|
|
"num_tokens": 497373983.0,
|
|
"step": 1735
|
|
},
|
|
{
|
|
"epoch": 0.890937019969278,
|
|
"grad_norm": 0.2595571322368428,
|
|
"learning_rate": 6.461614875678952e-06,
|
|
"loss": 0.49,
|
|
"num_tokens": 498889766.0,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.8934971838197645,
|
|
"grad_norm": 0.29056745385147914,
|
|
"learning_rate": 6.394819456710184e-06,
|
|
"loss": 0.4819,
|
|
"num_tokens": 500381376.0,
|
|
"step": 1745
|
|
},
|
|
{
|
|
"epoch": 0.8960573476702509,
|
|
"grad_norm": 0.3018418573283892,
|
|
"learning_rate": 6.3295373843321345e-06,
|
|
"loss": 0.4825,
|
|
"num_tokens": 501890304.0,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.8986175115207373,
|
|
"grad_norm": 0.26998785489591526,
|
|
"learning_rate": 6.265773339594632e-06,
|
|
"loss": 0.4957,
|
|
"num_tokens": 503426170.0,
|
|
"step": 1755
|
|
},
|
|
{
|
|
"epoch": 0.9011776753712237,
|
|
"grad_norm": 0.277118284646257,
|
|
"learning_rate": 6.2035318946973595e-06,
|
|
"loss": 0.478,
|
|
"num_tokens": 504810366.0,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.9037378392217101,
|
|
"grad_norm": 0.29706164348947733,
|
|
"learning_rate": 6.142817512661996e-06,
|
|
"loss": 0.4879,
|
|
"num_tokens": 506278240.0,
|
|
"step": 1765
|
|
},
|
|
{
|
|
"epoch": 0.9062980030721967,
|
|
"grad_norm": 0.29155906377324525,
|
|
"learning_rate": 6.083634547012214e-06,
|
|
"loss": 0.4803,
|
|
"num_tokens": 507734020.0,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.9088581669226831,
|
|
"grad_norm": 0.3031603930086798,
|
|
"learning_rate": 6.025987241461502e-06,
|
|
"loss": 0.48,
|
|
"num_tokens": 509060196.0,
|
|
"step": 1775
|
|
},
|
|
{
|
|
"epoch": 0.9114183307731695,
|
|
"grad_norm": 0.27849230517998186,
|
|
"learning_rate": 5.969879729608859e-06,
|
|
"loss": 0.4863,
|
|
"num_tokens": 510543906.0,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.9139784946236559,
|
|
"grad_norm": 0.28179744087308695,
|
|
"learning_rate": 5.915316034642423e-06,
|
|
"loss": 0.4918,
|
|
"num_tokens": 512004574.0,
|
|
"step": 1785
|
|
},
|
|
{
|
|
"epoch": 0.9165386584741424,
|
|
"grad_norm": 0.27891068130431096,
|
|
"learning_rate": 5.8623000690509515e-06,
|
|
"loss": 0.4897,
|
|
"num_tokens": 513390253.0,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.9190988223246288,
|
|
"grad_norm": 0.2878008444055862,
|
|
"learning_rate": 5.810835634343303e-06,
|
|
"loss": 0.4626,
|
|
"num_tokens": 514805282.0,
|
|
"step": 1795
|
|
},
|
|
{
|
|
"epoch": 0.9216589861751152,
|
|
"grad_norm": 0.2884405084072758,
|
|
"learning_rate": 5.760926420775841e-06,
|
|
"loss": 0.501,
|
|
"num_tokens": 516216104.0,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.9242191500256016,
|
|
"grad_norm": 0.31938785710437007,
|
|
"learning_rate": 5.7125760070878255e-06,
|
|
"loss": 0.4953,
|
|
"num_tokens": 517626797.0,
|
|
"step": 1805
|
|
},
|
|
{
|
|
"epoch": 0.926779313876088,
|
|
"grad_norm": 0.29407038604883545,
|
|
"learning_rate": 5.665787860244802e-06,
|
|
"loss": 0.483,
|
|
"num_tokens": 519045526.0,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.9293394777265745,
|
|
"grad_norm": 0.2991358474772203,
|
|
"learning_rate": 5.620565335189995e-06,
|
|
"loss": 0.4847,
|
|
"num_tokens": 520409227.0,
|
|
"step": 1815
|
|
},
|
|
{
|
|
"epoch": 0.931899641577061,
|
|
"grad_norm": 0.26190349794211626,
|
|
"learning_rate": 5.5769116746037495e-06,
|
|
"loss": 0.4724,
|
|
"num_tokens": 521824255.0,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.9344598054275474,
|
|
"grad_norm": 0.2698236840204939,
|
|
"learning_rate": 5.5348300086710105e-06,
|
|
"loss": 0.4659,
|
|
"num_tokens": 523275977.0,
|
|
"step": 1825
|
|
},
|
|
{
|
|
"epoch": 0.9370199692780338,
|
|
"grad_norm": 0.297865292125968,
|
|
"learning_rate": 5.494323354856877e-06,
|
|
"loss": 0.4979,
|
|
"num_tokens": 524701946.0,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.9395801331285202,
|
|
"grad_norm": 0.27405358140783065,
|
|
"learning_rate": 5.455394617690228e-06,
|
|
"loss": 0.496,
|
|
"num_tokens": 526258803.0,
|
|
"step": 1835
|
|
},
|
|
{
|
|
"epoch": 0.9421402969790067,
|
|
"grad_norm": 0.28899988844600377,
|
|
"learning_rate": 5.418046588555467e-06,
|
|
"loss": 0.487,
|
|
"num_tokens": 527666864.0,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.9447004608294931,
|
|
"grad_norm": 0.28180470184097417,
|
|
"learning_rate": 5.382281945492346e-06,
|
|
"loss": 0.4907,
|
|
"num_tokens": 529179204.0,
|
|
"step": 1845
|
|
},
|
|
{
|
|
"epoch": 0.9472606246799795,
|
|
"grad_norm": 0.27995615405921437,
|
|
"learning_rate": 5.348103253003949e-06,
|
|
"loss": 0.4792,
|
|
"num_tokens": 530544972.0,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.9498207885304659,
|
|
"grad_norm": 0.2786164141099446,
|
|
"learning_rate": 5.315512961872814e-06,
|
|
"loss": 0.4901,
|
|
"num_tokens": 531933919.0,
|
|
"step": 1855
|
|
},
|
|
{
|
|
"epoch": 0.9523809523809523,
|
|
"grad_norm": 0.2915059031097496,
|
|
"learning_rate": 5.2845134089851725e-06,
|
|
"loss": 0.5038,
|
|
"num_tokens": 533360398.0,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.9549411162314388,
|
|
"grad_norm": 0.2851387952457403,
|
|
"learning_rate": 5.255106817163407e-06,
|
|
"loss": 0.4837,
|
|
"num_tokens": 534792678.0,
|
|
"step": 1865
|
|
},
|
|
{
|
|
"epoch": 0.9575012800819253,
|
|
"grad_norm": 0.282843765069955,
|
|
"learning_rate": 5.227295295006656e-06,
|
|
"loss": 0.4655,
|
|
"num_tokens": 536208655.0,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.9600614439324117,
|
|
"grad_norm": 0.264482505603993,
|
|
"learning_rate": 5.201080836739616e-06,
|
|
"loss": 0.4768,
|
|
"num_tokens": 537749179.0,
|
|
"step": 1875
|
|
},
|
|
{
|
|
"epoch": 0.9626216077828981,
|
|
"grad_norm": 0.2811560109382176,
|
|
"learning_rate": 5.17646532206954e-06,
|
|
"loss": 0.4789,
|
|
"num_tokens": 539137436.0,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.9651817716333846,
|
|
"grad_norm": 0.30559827845866433,
|
|
"learning_rate": 5.153450516051468e-06,
|
|
"loss": 0.4798,
|
|
"num_tokens": 540499463.0,
|
|
"step": 1885
|
|
},
|
|
{
|
|
"epoch": 0.967741935483871,
|
|
"grad_norm": 0.2658142933819692,
|
|
"learning_rate": 5.1320380689616515e-06,
|
|
"loss": 0.4931,
|
|
"num_tokens": 541890275.0,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.9703020993343574,
|
|
"grad_norm": 0.2605044724608484,
|
|
"learning_rate": 5.112229516179219e-06,
|
|
"loss": 0.4741,
|
|
"num_tokens": 543324871.0,
|
|
"step": 1895
|
|
},
|
|
{
|
|
"epoch": 0.9728622631848438,
|
|
"grad_norm": 0.29507408098207166,
|
|
"learning_rate": 5.094026278076097e-06,
|
|
"loss": 0.5124,
|
|
"num_tokens": 544761712.0,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.9754224270353302,
|
|
"grad_norm": 0.30280140466290034,
|
|
"learning_rate": 5.077429659915142e-06,
|
|
"loss": 0.4797,
|
|
"num_tokens": 546087948.0,
|
|
"step": 1905
|
|
},
|
|
{
|
|
"epoch": 0.9779825908858167,
|
|
"grad_norm": 0.26933897726485434,
|
|
"learning_rate": 5.06244085175656e-06,
|
|
"loss": 0.4907,
|
|
"num_tokens": 547530783.0,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.9805427547363031,
|
|
"grad_norm": 0.2907848547554815,
|
|
"learning_rate": 5.049060928372572e-06,
|
|
"loss": 0.4893,
|
|
"num_tokens": 548976069.0,
|
|
"step": 1915
|
|
},
|
|
{
|
|
"epoch": 0.9831029185867896,
|
|
"grad_norm": 0.28031239120027995,
|
|
"learning_rate": 5.037290849170344e-06,
|
|
"loss": 0.4765,
|
|
"num_tokens": 550377811.0,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.985663082437276,
|
|
"grad_norm": 0.278756232502413,
|
|
"learning_rate": 5.027131458123189e-06,
|
|
"loss": 0.4793,
|
|
"num_tokens": 551824071.0,
|
|
"step": 1925
|
|
},
|
|
{
|
|
"epoch": 0.9882232462877624,
|
|
"grad_norm": 0.2852733725303888,
|
|
"learning_rate": 5.018583483710062e-06,
|
|
"loss": 0.4796,
|
|
"num_tokens": 553287217.0,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.9907834101382489,
|
|
"grad_norm": 0.302966944953007,
|
|
"learning_rate": 5.011647538863315e-06,
|
|
"loss": 0.47,
|
|
"num_tokens": 554664197.0,
|
|
"step": 1935
|
|
},
|
|
{
|
|
"epoch": 0.9933435739887353,
|
|
"grad_norm": 0.31117848291567346,
|
|
"learning_rate": 5.0063241209247474e-06,
|
|
"loss": 0.4989,
|
|
"num_tokens": 556055771.0,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.9959037378392217,
|
|
"grad_norm": 0.2691219016508246,
|
|
"learning_rate": 5.002613611609945e-06,
|
|
"loss": 0.4807,
|
|
"num_tokens": 557506305.0,
|
|
"step": 1945
|
|
},
|
|
{
|
|
"epoch": 0.9984639016897081,
|
|
"grad_norm": 0.28140804352817256,
|
|
"learning_rate": 5.000516276980914e-06,
|
|
"loss": 0.4782,
|
|
"num_tokens": 558905839.0,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"num_tokens": 559820337.0,
|
|
"step": 1953,
|
|
"total_flos": 1146505946202112.0,
|
|
"train_loss": 0.522347574165645,
|
|
"train_runtime": 14437.3811,
|
|
"train_samples_per_second": 6.492,
|
|
"train_steps_per_second": 0.135
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 1953,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 100,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 1146505946202112.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|