595 lines
16 KiB
JSON
595 lines
16 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 567,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 0.9081946484744549,
|
|
"epoch": 0.053015241882041084,
|
|
"grad_norm": 51.5,
|
|
"learning_rate": 3e-06,
|
|
"loss": 2.320839500427246,
|
|
"mean_token_accuracy": 0.5708254482597113,
|
|
"num_tokens": 30761.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"entropy": 1.032745758444071,
|
|
"epoch": 0.10603048376408217,
|
|
"grad_norm": 21.125,
|
|
"learning_rate": 6.333333333333333e-06,
|
|
"loss": 1.9631355285644532,
|
|
"mean_token_accuracy": 0.5809337079524994,
|
|
"num_tokens": 61277.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 1.2370933540165425,
|
|
"epoch": 0.15904572564612326,
|
|
"grad_norm": 11.0625,
|
|
"learning_rate": 9.666666666666667e-06,
|
|
"loss": 1.4653306961059571,
|
|
"mean_token_accuracy": 0.6311864696443081,
|
|
"num_tokens": 91730.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"entropy": 1.1646109029650689,
|
|
"epoch": 0.21206096752816433,
|
|
"grad_norm": 8.5625,
|
|
"learning_rate": 1.3000000000000001e-05,
|
|
"loss": 1.200753402709961,
|
|
"mean_token_accuracy": 0.6730783767998219,
|
|
"num_tokens": 122896.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 1.0138436160981654,
|
|
"epoch": 0.26507620941020543,
|
|
"grad_norm": 7.4375,
|
|
"learning_rate": 1.6333333333333335e-05,
|
|
"loss": 1.0109354972839355,
|
|
"mean_token_accuracy": 0.7080431424081326,
|
|
"num_tokens": 153323.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"entropy": 0.8669569574296474,
|
|
"epoch": 0.31809145129224653,
|
|
"grad_norm": 6.75,
|
|
"learning_rate": 1.9666666666666666e-05,
|
|
"loss": 0.8859419822692871,
|
|
"mean_token_accuracy": 0.7356305047869682,
|
|
"num_tokens": 183928.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 0.856675973534584,
|
|
"epoch": 0.3711066931742876,
|
|
"grad_norm": 7.6875,
|
|
"learning_rate": 1.9666666666666666e-05,
|
|
"loss": 0.8721119880676269,
|
|
"mean_token_accuracy": 0.7318881675601006,
|
|
"num_tokens": 215054.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"entropy": 0.7872001446783543,
|
|
"epoch": 0.42412193505632867,
|
|
"grad_norm": 7.46875,
|
|
"learning_rate": 1.92962962962963e-05,
|
|
"loss": 0.7920477390289307,
|
|
"mean_token_accuracy": 0.7553018122911453,
|
|
"num_tokens": 245614.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 0.7739131577312947,
|
|
"epoch": 0.47713717693836977,
|
|
"grad_norm": 7.09375,
|
|
"learning_rate": 1.8925925925925928e-05,
|
|
"loss": 0.7949441432952881,
|
|
"mean_token_accuracy": 0.7514781631529331,
|
|
"num_tokens": 276651.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"entropy": 0.7763891488313674,
|
|
"epoch": 0.5301524188204109,
|
|
"grad_norm": 7.8125,
|
|
"learning_rate": 1.8555555555555557e-05,
|
|
"loss": 0.7867375373840332,
|
|
"mean_token_accuracy": 0.7523738898336887,
|
|
"num_tokens": 308561.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 0.7757978036999702,
|
|
"epoch": 0.583167660702452,
|
|
"grad_norm": 7.21875,
|
|
"learning_rate": 1.8185185185185186e-05,
|
|
"loss": 0.7776416301727295,
|
|
"mean_token_accuracy": 0.7548527352511882,
|
|
"num_tokens": 339537.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"entropy": 0.7528716944158077,
|
|
"epoch": 0.6361829025844931,
|
|
"grad_norm": 6.28125,
|
|
"learning_rate": 1.7814814814814815e-05,
|
|
"loss": 0.7566296100616455,
|
|
"mean_token_accuracy": 0.7637521453201771,
|
|
"num_tokens": 370938.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 0.7483146995306015,
|
|
"epoch": 0.6891981444665342,
|
|
"grad_norm": 7.0,
|
|
"learning_rate": 1.7444444444444448e-05,
|
|
"loss": 0.7508650302886963,
|
|
"mean_token_accuracy": 0.758705198019743,
|
|
"num_tokens": 402708.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"entropy": 0.7206243231892586,
|
|
"epoch": 0.7422133863485753,
|
|
"grad_norm": 6.65625,
|
|
"learning_rate": 1.7074074074074077e-05,
|
|
"loss": 0.7531259536743165,
|
|
"mean_token_accuracy": 0.7641572073101998,
|
|
"num_tokens": 433645.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 0.7596748031675815,
|
|
"epoch": 0.7952286282306164,
|
|
"grad_norm": 6.03125,
|
|
"learning_rate": 1.6703703703703703e-05,
|
|
"loss": 0.7591583728790283,
|
|
"mean_token_accuracy": 0.7570627965033054,
|
|
"num_tokens": 465791.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"entropy": 0.7133973561227321,
|
|
"epoch": 0.8482438701126573,
|
|
"grad_norm": 7.125,
|
|
"learning_rate": 1.6333333333333335e-05,
|
|
"loss": 0.7458220005035401,
|
|
"mean_token_accuracy": 0.7626615799963474,
|
|
"num_tokens": 497296.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 0.7134327009320259,
|
|
"epoch": 0.9012591119946984,
|
|
"grad_norm": 7.0,
|
|
"learning_rate": 1.5962962962962964e-05,
|
|
"loss": 0.7290266990661621,
|
|
"mean_token_accuracy": 0.7681502625346184,
|
|
"num_tokens": 528181.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"entropy": 0.7324420973658562,
|
|
"epoch": 0.9542743538767395,
|
|
"grad_norm": 7.5,
|
|
"learning_rate": 1.5592592592592593e-05,
|
|
"loss": 0.731742525100708,
|
|
"mean_token_accuracy": 0.7631207950413227,
|
|
"num_tokens": 558915.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 0.7100333733992144,
|
|
"epoch": 1.0053015241882042,
|
|
"grad_norm": 8.3125,
|
|
"learning_rate": 1.5222222222222223e-05,
|
|
"loss": 0.7303330898284912,
|
|
"mean_token_accuracy": 0.7633489200047084,
|
|
"num_tokens": 588567.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"entropy": 0.6628884050995112,
|
|
"epoch": 1.0583167660702453,
|
|
"grad_norm": 7.0625,
|
|
"learning_rate": 1.4851851851851853e-05,
|
|
"loss": 0.6481011867523193,
|
|
"mean_token_accuracy": 0.7933171860873699,
|
|
"num_tokens": 619081.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 0.6257969986647367,
|
|
"epoch": 1.1113320079522864,
|
|
"grad_norm": 7.25,
|
|
"learning_rate": 1.4481481481481483e-05,
|
|
"loss": 0.642666244506836,
|
|
"mean_token_accuracy": 0.7921255677938461,
|
|
"num_tokens": 650402.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"entropy": 0.6399037420749665,
|
|
"epoch": 1.1643472498343272,
|
|
"grad_norm": 6.03125,
|
|
"learning_rate": 1.4111111111111113e-05,
|
|
"loss": 0.6525338172912598,
|
|
"mean_token_accuracy": 0.7887163788080216,
|
|
"num_tokens": 681837.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 0.6367121540009976,
|
|
"epoch": 1.2173624917163686,
|
|
"grad_norm": 7.59375,
|
|
"learning_rate": 1.3740740740740741e-05,
|
|
"loss": 0.6356833457946778,
|
|
"mean_token_accuracy": 0.7893597863614559,
|
|
"num_tokens": 712571.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"entropy": 0.6334792256355286,
|
|
"epoch": 1.2703777335984094,
|
|
"grad_norm": 6.6875,
|
|
"learning_rate": 1.3370370370370372e-05,
|
|
"loss": 0.6391499996185303,
|
|
"mean_token_accuracy": 0.7947010740637779,
|
|
"num_tokens": 743396.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 0.6417076423764229,
|
|
"epoch": 1.3233929754804508,
|
|
"grad_norm": 7.4375,
|
|
"learning_rate": 1.3000000000000001e-05,
|
|
"loss": 0.6563210010528564,
|
|
"mean_token_accuracy": 0.787147381901741,
|
|
"num_tokens": 774883.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"entropy": 0.6213644035160542,
|
|
"epoch": 1.3764082173624916,
|
|
"grad_norm": 7.1875,
|
|
"learning_rate": 1.2629629629629632e-05,
|
|
"loss": 0.6295370578765869,
|
|
"mean_token_accuracy": 0.7955153584480286,
|
|
"num_tokens": 805520.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 0.6552363462746144,
|
|
"epoch": 1.4294234592445327,
|
|
"grad_norm": 6.84375,
|
|
"learning_rate": 1.225925925925926e-05,
|
|
"loss": 0.6632986068725586,
|
|
"mean_token_accuracy": 0.7826757229864597,
|
|
"num_tokens": 836758.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"entropy": 0.6550090927630663,
|
|
"epoch": 1.4824387011265738,
|
|
"grad_norm": 7.25,
|
|
"learning_rate": 1.188888888888889e-05,
|
|
"loss": 0.6479949474334716,
|
|
"mean_token_accuracy": 0.78950335085392,
|
|
"num_tokens": 868056.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 0.6275094959884882,
|
|
"epoch": 1.535453943008615,
|
|
"grad_norm": 6.4375,
|
|
"learning_rate": 1.151851851851852e-05,
|
|
"loss": 0.6384372234344482,
|
|
"mean_token_accuracy": 0.7930210866034031,
|
|
"num_tokens": 899420.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"entropy": 0.6052290312945843,
|
|
"epoch": 1.588469184890656,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 1.114814814814815e-05,
|
|
"loss": 0.609743309020996,
|
|
"mean_token_accuracy": 0.7990770071744919,
|
|
"num_tokens": 929571.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 0.6174360021948815,
|
|
"epoch": 1.6414844267726971,
|
|
"grad_norm": 7.0625,
|
|
"learning_rate": 1.0777777777777778e-05,
|
|
"loss": 0.6313478946685791,
|
|
"mean_token_accuracy": 0.7964761383831501,
|
|
"num_tokens": 960928.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"entropy": 0.6127162992954254,
|
|
"epoch": 1.6944996686547382,
|
|
"grad_norm": 6.5625,
|
|
"learning_rate": 1.0407407407407408e-05,
|
|
"loss": 0.6150648593902588,
|
|
"mean_token_accuracy": 0.8008938111364842,
|
|
"num_tokens": 991961.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 0.5947048265486956,
|
|
"epoch": 1.7475149105367793,
|
|
"grad_norm": 7.65625,
|
|
"learning_rate": 1.003703703703704e-05,
|
|
"loss": 0.6053301334381104,
|
|
"mean_token_accuracy": 0.801536674797535,
|
|
"num_tokens": 1022013.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"entropy": 0.6275908622890711,
|
|
"epoch": 1.8005301524188204,
|
|
"grad_norm": 6.9375,
|
|
"learning_rate": 9.666666666666667e-06,
|
|
"loss": 0.6307861804962158,
|
|
"mean_token_accuracy": 0.7918523952364922,
|
|
"num_tokens": 1053061.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 0.6406017865985631,
|
|
"epoch": 1.8535453943008615,
|
|
"grad_norm": 7.0,
|
|
"learning_rate": 9.296296296296296e-06,
|
|
"loss": 0.6534273147583007,
|
|
"mean_token_accuracy": 0.789715650677681,
|
|
"num_tokens": 1085392.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"entropy": 0.6285702683031559,
|
|
"epoch": 1.9065606361829026,
|
|
"grad_norm": 7.8125,
|
|
"learning_rate": 8.925925925925927e-06,
|
|
"loss": 0.6331883907318115,
|
|
"mean_token_accuracy": 0.7924636773765087,
|
|
"num_tokens": 1117204.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 0.615465309098363,
|
|
"epoch": 1.9595758780649437,
|
|
"grad_norm": 7.25,
|
|
"learning_rate": 8.555555555555556e-06,
|
|
"loss": 0.6278977870941163,
|
|
"mean_token_accuracy": 0.7965366452932358,
|
|
"num_tokens": 1147711.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"entropy": 0.6297557744886968,
|
|
"epoch": 2.0106030483764084,
|
|
"grad_norm": 6.46875,
|
|
"learning_rate": 8.185185185185187e-06,
|
|
"loss": 0.6424172878265381,
|
|
"mean_token_accuracy": 0.7955868112576472,
|
|
"num_tokens": 1177908.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 0.599795724824071,
|
|
"epoch": 2.0636182902584492,
|
|
"grad_norm": 6.4375,
|
|
"learning_rate": 7.814814814814816e-06,
|
|
"loss": 0.5880236625671387,
|
|
"mean_token_accuracy": 0.805521210283041,
|
|
"num_tokens": 1209330.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"entropy": 0.5943233758211136,
|
|
"epoch": 2.1166335321404905,
|
|
"grad_norm": 6.9375,
|
|
"learning_rate": 7.444444444444445e-06,
|
|
"loss": 0.5809715270996094,
|
|
"mean_token_accuracy": 0.8090382158756256,
|
|
"num_tokens": 1241043.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 0.583736015111208,
|
|
"epoch": 2.1696487740225314,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 7.074074074074074e-06,
|
|
"loss": 0.595209789276123,
|
|
"mean_token_accuracy": 0.8054515205323696,
|
|
"num_tokens": 1272875.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"entropy": 0.5891257628798485,
|
|
"epoch": 2.2226640159045727,
|
|
"grad_norm": 7.875,
|
|
"learning_rate": 6.703703703703704e-06,
|
|
"loss": 0.6067119598388672,
|
|
"mean_token_accuracy": 0.8018631525337696,
|
|
"num_tokens": 1303473.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 0.5886123344302178,
|
|
"epoch": 2.2756792577866136,
|
|
"grad_norm": 7.65625,
|
|
"learning_rate": 6.333333333333333e-06,
|
|
"loss": 0.577965784072876,
|
|
"mean_token_accuracy": 0.8081556148827076,
|
|
"num_tokens": 1335327.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"entropy": 0.5796562805771828,
|
|
"epoch": 2.3286944996686545,
|
|
"grad_norm": 8.125,
|
|
"learning_rate": 5.962962962962963e-06,
|
|
"loss": 0.5881845951080322,
|
|
"mean_token_accuracy": 0.8047773309051991,
|
|
"num_tokens": 1366019.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 0.590113490819931,
|
|
"epoch": 2.381709741550696,
|
|
"grad_norm": 6.96875,
|
|
"learning_rate": 5.5925925925925926e-06,
|
|
"loss": 0.593454647064209,
|
|
"mean_token_accuracy": 0.809637114405632,
|
|
"num_tokens": 1397101.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"entropy": 0.570321362093091,
|
|
"epoch": 2.434724983432737,
|
|
"grad_norm": 7.34375,
|
|
"learning_rate": 5.2222222222222226e-06,
|
|
"loss": 0.5790628910064697,
|
|
"mean_token_accuracy": 0.809483564645052,
|
|
"num_tokens": 1428115.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 0.6001236628741026,
|
|
"epoch": 2.487740225314778,
|
|
"grad_norm": 7.21875,
|
|
"learning_rate": 4.851851851851852e-06,
|
|
"loss": 0.6089425086975098,
|
|
"mean_token_accuracy": 0.8031827084720135,
|
|
"num_tokens": 1460337.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"entropy": 0.5818770505487919,
|
|
"epoch": 2.540755467196819,
|
|
"grad_norm": 6.5625,
|
|
"learning_rate": 4.481481481481482e-06,
|
|
"loss": 0.5913994789123536,
|
|
"mean_token_accuracy": 0.8058472029864788,
|
|
"num_tokens": 1491465.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 0.5761014901101589,
|
|
"epoch": 2.59377070907886,
|
|
"grad_norm": 7.25,
|
|
"learning_rate": 4.111111111111111e-06,
|
|
"loss": 0.573754072189331,
|
|
"mean_token_accuracy": 0.8119833193719387,
|
|
"num_tokens": 1522283.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"entropy": 0.5881835252046586,
|
|
"epoch": 2.6467859509609015,
|
|
"grad_norm": 7.8125,
|
|
"learning_rate": 3.740740740740741e-06,
|
|
"loss": 0.5855993270874024,
|
|
"mean_token_accuracy": 0.8061533674597741,
|
|
"num_tokens": 1553066.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 0.5923898722976446,
|
|
"epoch": 2.6998011928429424,
|
|
"grad_norm": 7.25,
|
|
"learning_rate": 3.3703703703703705e-06,
|
|
"loss": 0.5783270835876465,
|
|
"mean_token_accuracy": 0.8092506147921086,
|
|
"num_tokens": 1583785.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"entropy": 0.5890947930514813,
|
|
"epoch": 2.7528164347249833,
|
|
"grad_norm": 7.09375,
|
|
"learning_rate": 3e-06,
|
|
"loss": 0.5847487449645996,
|
|
"mean_token_accuracy": 0.8107674546539784,
|
|
"num_tokens": 1615166.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"entropy": 0.5714535418897867,
|
|
"epoch": 2.8058316766070246,
|
|
"grad_norm": 7.0,
|
|
"learning_rate": 2.6296296296296297e-06,
|
|
"loss": 0.5737119197845459,
|
|
"mean_token_accuracy": 0.8139248043298721,
|
|
"num_tokens": 1646166.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"entropy": 0.5585082449018955,
|
|
"epoch": 2.8588469184890655,
|
|
"grad_norm": 8.75,
|
|
"learning_rate": 2.2592592592592592e-06,
|
|
"loss": 0.5831988334655762,
|
|
"mean_token_accuracy": 0.8078319534659386,
|
|
"num_tokens": 1675970.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"entropy": 0.5816058319061994,
|
|
"epoch": 2.911862160371107,
|
|
"grad_norm": 6.6875,
|
|
"learning_rate": 1.888888888888889e-06,
|
|
"loss": 0.585014820098877,
|
|
"mean_token_accuracy": 0.8070621207356453,
|
|
"num_tokens": 1707365.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"entropy": 0.5718538902699948,
|
|
"epoch": 2.9648774022531477,
|
|
"grad_norm": 7.4375,
|
|
"learning_rate": 1.5185185185185186e-06,
|
|
"loss": 0.5958520889282226,
|
|
"mean_token_accuracy": 0.8062198340892792,
|
|
"num_tokens": 1737306.0,
|
|
"step": 560
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 600,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 4,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 1.696568888819712e+16,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|