2046 lines
54 KiB
JSON
2046 lines
54 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.2827254735651682,
|
||
|
|
"eval_steps": 1000,
|
||
|
|
"global_step": 1000,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"entropy": 10.74207363128662,
|
||
|
|
"epoch": 0.0014136273678258412,
|
||
|
|
"grad_norm": 6.1875,
|
||
|
|
"learning_rate": 2e-06,
|
||
|
|
"loss": 10.5511,
|
||
|
|
"mean_token_accuracy": 0.0,
|
||
|
|
"num_tokens": 4340.0,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.742022514343262,
|
||
|
|
"epoch": 0.0028272547356516823,
|
||
|
|
"grad_norm": 6.71875,
|
||
|
|
"learning_rate": 4.5e-06,
|
||
|
|
"loss": 10.5265,
|
||
|
|
"mean_token_accuracy": 0.0,
|
||
|
|
"num_tokens": 8692.0,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.742014980316162,
|
||
|
|
"epoch": 0.004240882103477523,
|
||
|
|
"grad_norm": 6.78125,
|
||
|
|
"learning_rate": 7e-06,
|
||
|
|
"loss": 10.4477,
|
||
|
|
"mean_token_accuracy": 0.0,
|
||
|
|
"num_tokens": 12453.0,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.742022132873535,
|
||
|
|
"epoch": 0.005654509471303365,
|
||
|
|
"grad_norm": 6.71875,
|
||
|
|
"learning_rate": 9.5e-06,
|
||
|
|
"loss": 10.4201,
|
||
|
|
"mean_token_accuracy": 0.0,
|
||
|
|
"num_tokens": 16439.0,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.741988182067871,
|
||
|
|
"epoch": 0.007068136839129205,
|
||
|
|
"grad_norm": 6.8125,
|
||
|
|
"learning_rate": 1.2e-05,
|
||
|
|
"loss": 10.295,
|
||
|
|
"mean_token_accuracy": 0.0014771121321246028,
|
||
|
|
"num_tokens": 20515.0,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.74176025390625,
|
||
|
|
"epoch": 0.008481764206955046,
|
||
|
|
"grad_norm": 5.5,
|
||
|
|
"learning_rate": 1.4500000000000002e-05,
|
||
|
|
"loss": 10.1401,
|
||
|
|
"mean_token_accuracy": 0.029323107562959194,
|
||
|
|
"num_tokens": 24336.0,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.741047763824463,
|
||
|
|
"epoch": 0.009895391574780888,
|
||
|
|
"grad_norm": 4.5625,
|
||
|
|
"learning_rate": 1.7000000000000003e-05,
|
||
|
|
"loss": 10.0398,
|
||
|
|
"mean_token_accuracy": 0.05071598924696445,
|
||
|
|
"num_tokens": 28465.0,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.739372825622558,
|
||
|
|
"epoch": 0.01130901894260673,
|
||
|
|
"grad_norm": 3.90625,
|
||
|
|
"learning_rate": 1.95e-05,
|
||
|
|
"loss": 9.828,
|
||
|
|
"mean_token_accuracy": 0.06147046238183975,
|
||
|
|
"num_tokens": 32217.0,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.73618106842041,
|
||
|
|
"epoch": 0.01272264631043257,
|
||
|
|
"grad_norm": 3.015625,
|
||
|
|
"learning_rate": 2.2e-05,
|
||
|
|
"loss": 9.7393,
|
||
|
|
"mean_token_accuracy": 0.05857079476118088,
|
||
|
|
"num_tokens": 36369.0,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.732595252990723,
|
||
|
|
"epoch": 0.01413627367825841,
|
||
|
|
"grad_norm": 2.859375,
|
||
|
|
"learning_rate": 2.4500000000000003e-05,
|
||
|
|
"loss": 9.6251,
|
||
|
|
"mean_token_accuracy": 0.0617414329200983,
|
||
|
|
"num_tokens": 40227.0,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.730756187438965,
|
||
|
|
"epoch": 0.015549901046084252,
|
||
|
|
"grad_norm": 2.5,
|
||
|
|
"learning_rate": 2.7e-05,
|
||
|
|
"loss": 9.6041,
|
||
|
|
"mean_token_accuracy": 0.058244453743100166,
|
||
|
|
"num_tokens": 44266.0,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.728981113433838,
|
||
|
|
"epoch": 0.016963528413910092,
|
||
|
|
"grad_norm": 2.640625,
|
||
|
|
"learning_rate": 2.95e-05,
|
||
|
|
"loss": 9.5679,
|
||
|
|
"mean_token_accuracy": 0.06274866946041584,
|
||
|
|
"num_tokens": 48187.0,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.726116180419922,
|
||
|
|
"epoch": 0.018377155781735936,
|
||
|
|
"grad_norm": 2.453125,
|
||
|
|
"learning_rate": 3.2e-05,
|
||
|
|
"loss": 9.4557,
|
||
|
|
"mean_token_accuracy": 0.05981031022965908,
|
||
|
|
"num_tokens": 52252.0,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.721884059906007,
|
||
|
|
"epoch": 0.019790783149561775,
|
||
|
|
"grad_norm": 2.53125,
|
||
|
|
"learning_rate": 3.4500000000000005e-05,
|
||
|
|
"loss": 9.4336,
|
||
|
|
"mean_token_accuracy": 0.05559282228350639,
|
||
|
|
"num_tokens": 56421.0,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.7160475730896,
|
||
|
|
"epoch": 0.021204410517387615,
|
||
|
|
"grad_norm": 2.421875,
|
||
|
|
"learning_rate": 3.7e-05,
|
||
|
|
"loss": 9.3486,
|
||
|
|
"mean_token_accuracy": 0.06220139227807522,
|
||
|
|
"num_tokens": 60341.0,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.70057315826416,
|
||
|
|
"epoch": 0.02261803788521346,
|
||
|
|
"grad_norm": 2.421875,
|
||
|
|
"learning_rate": 3.95e-05,
|
||
|
|
"loss": 9.216,
|
||
|
|
"mean_token_accuracy": 0.06268311068415641,
|
||
|
|
"num_tokens": 64465.0,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.682071495056153,
|
||
|
|
"epoch": 0.0240316652530393,
|
||
|
|
"grad_norm": 2.21875,
|
||
|
|
"learning_rate": 4.2000000000000004e-05,
|
||
|
|
"loss": 9.1917,
|
||
|
|
"mean_token_accuracy": 0.06404139809310436,
|
||
|
|
"num_tokens": 68514.0,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.654492664337159,
|
||
|
|
"epoch": 0.02544529262086514,
|
||
|
|
"grad_norm": 2.265625,
|
||
|
|
"learning_rate": 4.45e-05,
|
||
|
|
"loss": 9.1028,
|
||
|
|
"mean_token_accuracy": 0.06569066680967808,
|
||
|
|
"num_tokens": 72661.0,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.621818923950196,
|
||
|
|
"epoch": 0.02685891998869098,
|
||
|
|
"grad_norm": 2.203125,
|
||
|
|
"learning_rate": 4.7000000000000004e-05,
|
||
|
|
"loss": 8.9739,
|
||
|
|
"mean_token_accuracy": 0.07454013973474502,
|
||
|
|
"num_tokens": 76698.0,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.587986373901368,
|
||
|
|
"epoch": 0.02827254735651682,
|
||
|
|
"grad_norm": 2.109375,
|
||
|
|
"learning_rate": 4.9500000000000004e-05,
|
||
|
|
"loss": 8.8935,
|
||
|
|
"mean_token_accuracy": 0.07267784662544727,
|
||
|
|
"num_tokens": 80732.0,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.545151424407958,
|
||
|
|
"epoch": 0.029686174724342665,
|
||
|
|
"grad_norm": 2.15625,
|
||
|
|
"learning_rate": 5.2e-05,
|
||
|
|
"loss": 8.8022,
|
||
|
|
"mean_token_accuracy": 0.06954946778714657,
|
||
|
|
"num_tokens": 84797.0,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.516861534118652,
|
||
|
|
"epoch": 0.031099802092168505,
|
||
|
|
"grad_norm": 2.234375,
|
||
|
|
"learning_rate": 5.45e-05,
|
||
|
|
"loss": 8.722,
|
||
|
|
"mean_token_accuracy": 0.07358665019273758,
|
||
|
|
"num_tokens": 88758.0,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.459677600860596,
|
||
|
|
"epoch": 0.032513429459994345,
|
||
|
|
"grad_norm": 1.953125,
|
||
|
|
"learning_rate": 5.7e-05,
|
||
|
|
"loss": 8.6039,
|
||
|
|
"mean_token_accuracy": 0.07231484800577163,
|
||
|
|
"num_tokens": 92722.0,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.407901668548584,
|
||
|
|
"epoch": 0.033927056827820185,
|
||
|
|
"grad_norm": 2.125,
|
||
|
|
"learning_rate": 5.9499999999999996e-05,
|
||
|
|
"loss": 8.4476,
|
||
|
|
"mean_token_accuracy": 0.07675158120691776,
|
||
|
|
"num_tokens": 96602.0,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.325055503845215,
|
||
|
|
"epoch": 0.035340684195646024,
|
||
|
|
"grad_norm": 1.9453125,
|
||
|
|
"learning_rate": 6.2e-05,
|
||
|
|
"loss": 8.3606,
|
||
|
|
"mean_token_accuracy": 0.07958476431667805,
|
||
|
|
"num_tokens": 100598.0,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.222711753845214,
|
||
|
|
"epoch": 0.03675431156347187,
|
||
|
|
"grad_norm": 1.875,
|
||
|
|
"learning_rate": 6.450000000000001e-05,
|
||
|
|
"loss": 8.2593,
|
||
|
|
"mean_token_accuracy": 0.07609154582023621,
|
||
|
|
"num_tokens": 104460.0,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.145068645477295,
|
||
|
|
"epoch": 0.03816793893129771,
|
||
|
|
"grad_norm": 2.109375,
|
||
|
|
"learning_rate": 6.7e-05,
|
||
|
|
"loss": 8.1373,
|
||
|
|
"mean_token_accuracy": 0.07444728389382363,
|
||
|
|
"num_tokens": 108360.0,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 10.019375801086426,
|
||
|
|
"epoch": 0.03958156629912355,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 6.950000000000001e-05,
|
||
|
|
"loss": 7.9991,
|
||
|
|
"mean_token_accuracy": 0.07835616618394851,
|
||
|
|
"num_tokens": 112176.0,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 9.877924728393555,
|
||
|
|
"epoch": 0.04099519366694939,
|
||
|
|
"grad_norm": 1.671875,
|
||
|
|
"learning_rate": 7.2e-05,
|
||
|
|
"loss": 8.0465,
|
||
|
|
"mean_token_accuracy": 0.07128159664571285,
|
||
|
|
"num_tokens": 116192.0,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 9.766163635253907,
|
||
|
|
"epoch": 0.04240882103477523,
|
||
|
|
"grad_norm": 1.5390625,
|
||
|
|
"learning_rate": 7.45e-05,
|
||
|
|
"loss": 7.9125,
|
||
|
|
"mean_token_accuracy": 0.07681304663419723,
|
||
|
|
"num_tokens": 120221.0,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 9.596385955810547,
|
||
|
|
"epoch": 0.04382244840260108,
|
||
|
|
"grad_norm": 1.3515625,
|
||
|
|
"learning_rate": 7.7e-05,
|
||
|
|
"loss": 7.8471,
|
||
|
|
"mean_token_accuracy": 0.07160197533667087,
|
||
|
|
"num_tokens": 124364.0,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 9.397625064849853,
|
||
|
|
"epoch": 0.04523607577042692,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 7.950000000000001e-05,
|
||
|
|
"loss": 7.6486,
|
||
|
|
"mean_token_accuracy": 0.08006830923259259,
|
||
|
|
"num_tokens": 128266.0,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 9.197708892822266,
|
||
|
|
"epoch": 0.04664970313825276,
|
||
|
|
"grad_norm": 1.234375,
|
||
|
|
"learning_rate": 8.2e-05,
|
||
|
|
"loss": 7.5695,
|
||
|
|
"mean_token_accuracy": 0.08508073762059212,
|
||
|
|
"num_tokens": 131985.0,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 9.003838157653808,
|
||
|
|
"epoch": 0.0480633305060786,
|
||
|
|
"grad_norm": 1.1640625,
|
||
|
|
"learning_rate": 8.450000000000001e-05,
|
||
|
|
"loss": 7.4586,
|
||
|
|
"mean_token_accuracy": 0.08466509208083153,
|
||
|
|
"num_tokens": 135892.0,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.81080961227417,
|
||
|
|
"epoch": 0.04947695787390444,
|
||
|
|
"grad_norm": 1.28125,
|
||
|
|
"learning_rate": 8.7e-05,
|
||
|
|
"loss": 7.5228,
|
||
|
|
"mean_token_accuracy": 0.075396229326725,
|
||
|
|
"num_tokens": 139937.0,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.659114837646484,
|
||
|
|
"epoch": 0.05089058524173028,
|
||
|
|
"grad_norm": 1.1484375,
|
||
|
|
"learning_rate": 8.95e-05,
|
||
|
|
"loss": 7.3426,
|
||
|
|
"mean_token_accuracy": 0.08097245395183564,
|
||
|
|
"num_tokens": 143659.0,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.553722858428955,
|
||
|
|
"epoch": 0.052304212609556124,
|
||
|
|
"grad_norm": 1.3515625,
|
||
|
|
"learning_rate": 9.2e-05,
|
||
|
|
"loss": 7.4259,
|
||
|
|
"mean_token_accuracy": 0.08198288567364216,
|
||
|
|
"num_tokens": 147394.0,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.326274490356445,
|
||
|
|
"epoch": 0.05371783997738196,
|
||
|
|
"grad_norm": 1.1015625,
|
||
|
|
"learning_rate": 9.45e-05,
|
||
|
|
"loss": 7.2628,
|
||
|
|
"mean_token_accuracy": 0.08642936870455742,
|
||
|
|
"num_tokens": 151082.0,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.238064765930176,
|
||
|
|
"epoch": 0.0551314673452078,
|
||
|
|
"grad_norm": 1.359375,
|
||
|
|
"learning_rate": 9.7e-05,
|
||
|
|
"loss": 7.25,
|
||
|
|
"mean_token_accuracy": 0.08761365041136741,
|
||
|
|
"num_tokens": 155099.0,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.188647174835205,
|
||
|
|
"epoch": 0.05654509471303364,
|
||
|
|
"grad_norm": 0.9921875,
|
||
|
|
"learning_rate": 9.95e-05,
|
||
|
|
"loss": 7.4052,
|
||
|
|
"mean_token_accuracy": 0.08035821095108986,
|
||
|
|
"num_tokens": 159297.0,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.190512943267823,
|
||
|
|
"epoch": 0.05795872208085948,
|
||
|
|
"grad_norm": 1.21875,
|
||
|
|
"learning_rate": 0.000102,
|
||
|
|
"loss": 7.268,
|
||
|
|
"mean_token_accuracy": 0.08253459706902504,
|
||
|
|
"num_tokens": 163337.0,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.12247543334961,
|
||
|
|
"epoch": 0.05937234944868533,
|
||
|
|
"grad_norm": 1.1875,
|
||
|
|
"learning_rate": 0.00010449999999999999,
|
||
|
|
"loss": 7.3087,
|
||
|
|
"mean_token_accuracy": 0.08554964698851109,
|
||
|
|
"num_tokens": 167440.0,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.021088409423829,
|
||
|
|
"epoch": 0.06078597681651117,
|
||
|
|
"grad_norm": 1.2265625,
|
||
|
|
"learning_rate": 0.000107,
|
||
|
|
"loss": 7.1661,
|
||
|
|
"mean_token_accuracy": 0.08840125799179077,
|
||
|
|
"num_tokens": 171238.0,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 8.036321210861207,
|
||
|
|
"epoch": 0.06219960418433701,
|
||
|
|
"grad_norm": 1.15625,
|
||
|
|
"learning_rate": 0.0001095,
|
||
|
|
"loss": 7.2139,
|
||
|
|
"mean_token_accuracy": 0.0908890649676323,
|
||
|
|
"num_tokens": 175476.0,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.951605987548828,
|
||
|
|
"epoch": 0.06361323155216285,
|
||
|
|
"grad_norm": 1.6796875,
|
||
|
|
"learning_rate": 0.000112,
|
||
|
|
"loss": 7.1261,
|
||
|
|
"mean_token_accuracy": 0.08661228194832801,
|
||
|
|
"num_tokens": 179398.0,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.849934148788452,
|
||
|
|
"epoch": 0.06502685891998869,
|
||
|
|
"grad_norm": 1.1640625,
|
||
|
|
"learning_rate": 0.0001145,
|
||
|
|
"loss": 7.1123,
|
||
|
|
"mean_token_accuracy": 0.09166858419775963,
|
||
|
|
"num_tokens": 183355.0,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.945307731628418,
|
||
|
|
"epoch": 0.06644048628781453,
|
||
|
|
"grad_norm": 1.3828125,
|
||
|
|
"learning_rate": 0.00011700000000000001,
|
||
|
|
"loss": 7.1135,
|
||
|
|
"mean_token_accuracy": 0.08504104316234588,
|
||
|
|
"num_tokens": 187278.0,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.837193965911865,
|
||
|
|
"epoch": 0.06785411365564037,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 0.00011949999999999999,
|
||
|
|
"loss": 7.1333,
|
||
|
|
"mean_token_accuracy": 0.09098687320947647,
|
||
|
|
"num_tokens": 191136.0,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.820778036117554,
|
||
|
|
"epoch": 0.06926774102346621,
|
||
|
|
"grad_norm": 1.671875,
|
||
|
|
"learning_rate": 0.000122,
|
||
|
|
"loss": 7.1943,
|
||
|
|
"mean_token_accuracy": 0.0956549420952797,
|
||
|
|
"num_tokens": 195096.0,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.859644556045533,
|
||
|
|
"epoch": 0.07068136839129205,
|
||
|
|
"grad_norm": 1.3984375,
|
||
|
|
"learning_rate": 0.0001245,
|
||
|
|
"loss": 7.1855,
|
||
|
|
"mean_token_accuracy": 0.08993390724062919,
|
||
|
|
"num_tokens": 199483.0,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.814686632156372,
|
||
|
|
"epoch": 0.0720949957591179,
|
||
|
|
"grad_norm": 1.53125,
|
||
|
|
"learning_rate": 0.000127,
|
||
|
|
"loss": 7.1139,
|
||
|
|
"mean_token_accuracy": 0.09194719940423965,
|
||
|
|
"num_tokens": 203515.0,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.748371124267578,
|
||
|
|
"epoch": 0.07350862312694374,
|
||
|
|
"grad_norm": 1.28125,
|
||
|
|
"learning_rate": 0.0001295,
|
||
|
|
"loss": 7.116,
|
||
|
|
"mean_token_accuracy": 0.09589750841259956,
|
||
|
|
"num_tokens": 207451.0,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.774532127380371,
|
||
|
|
"epoch": 0.07492225049476958,
|
||
|
|
"grad_norm": 1.4453125,
|
||
|
|
"learning_rate": 0.000132,
|
||
|
|
"loss": 7.0308,
|
||
|
|
"mean_token_accuracy": 0.1008292630314827,
|
||
|
|
"num_tokens": 211507.0,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.691170597076416,
|
||
|
|
"epoch": 0.07633587786259542,
|
||
|
|
"grad_norm": 1.640625,
|
||
|
|
"learning_rate": 0.00013450000000000002,
|
||
|
|
"loss": 7.09,
|
||
|
|
"mean_token_accuracy": 0.09460079073905944,
|
||
|
|
"num_tokens": 215236.0,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.701872444152832,
|
||
|
|
"epoch": 0.07774950523042126,
|
||
|
|
"grad_norm": 1.3828125,
|
||
|
|
"learning_rate": 0.00013700000000000002,
|
||
|
|
"loss": 7.0577,
|
||
|
|
"mean_token_accuracy": 0.10146123990416527,
|
||
|
|
"num_tokens": 218934.0,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.6655951023101805,
|
||
|
|
"epoch": 0.0791631325982471,
|
||
|
|
"grad_norm": 1.3359375,
|
||
|
|
"learning_rate": 0.0001395,
|
||
|
|
"loss": 6.9994,
|
||
|
|
"mean_token_accuracy": 0.10914008393883705,
|
||
|
|
"num_tokens": 223185.0,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.753876638412476,
|
||
|
|
"epoch": 0.08057675996607294,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.00014199999999999998,
|
||
|
|
"loss": 7.1232,
|
||
|
|
"mean_token_accuracy": 0.09365936666727066,
|
||
|
|
"num_tokens": 227044.0,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.66315860748291,
|
||
|
|
"epoch": 0.08199038733389878,
|
||
|
|
"grad_norm": 1.171875,
|
||
|
|
"learning_rate": 0.0001445,
|
||
|
|
"loss": 7.0308,
|
||
|
|
"mean_token_accuracy": 0.10147744789719582,
|
||
|
|
"num_tokens": 231359.0,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.710978984832764,
|
||
|
|
"epoch": 0.08340401470172462,
|
||
|
|
"grad_norm": 1.4765625,
|
||
|
|
"learning_rate": 0.000147,
|
||
|
|
"loss": 7.0018,
|
||
|
|
"mean_token_accuracy": 0.11066402792930603,
|
||
|
|
"num_tokens": 235224.0,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.5690467834472654,
|
||
|
|
"epoch": 0.08481764206955046,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 0.0001495,
|
||
|
|
"loss": 6.8653,
|
||
|
|
"mean_token_accuracy": 0.11461045518517494,
|
||
|
|
"num_tokens": 239119.0,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.518357563018799,
|
||
|
|
"epoch": 0.0862312694373763,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.000152,
|
||
|
|
"loss": 6.9653,
|
||
|
|
"mean_token_accuracy": 0.10772728994488716,
|
||
|
|
"num_tokens": 243275.0,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.694993162155152,
|
||
|
|
"epoch": 0.08764489680520215,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.00015450000000000001,
|
||
|
|
"loss": 6.9709,
|
||
|
|
"mean_token_accuracy": 0.11304818466305733,
|
||
|
|
"num_tokens": 247415.0,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.528787612915039,
|
||
|
|
"epoch": 0.089058524173028,
|
||
|
|
"grad_norm": 1.4765625,
|
||
|
|
"learning_rate": 0.000157,
|
||
|
|
"loss": 6.9654,
|
||
|
|
"mean_token_accuracy": 0.10677048042416573,
|
||
|
|
"num_tokens": 251575.0,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.5503821849823,
|
||
|
|
"epoch": 0.09047215154085383,
|
||
|
|
"grad_norm": 1.65625,
|
||
|
|
"learning_rate": 0.0001595,
|
||
|
|
"loss": 6.9441,
|
||
|
|
"mean_token_accuracy": 0.11381662338972091,
|
||
|
|
"num_tokens": 255686.0,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.599794197082519,
|
||
|
|
"epoch": 0.09188577890867967,
|
||
|
|
"grad_norm": 1.4296875,
|
||
|
|
"learning_rate": 0.000162,
|
||
|
|
"loss": 6.9401,
|
||
|
|
"mean_token_accuracy": 0.11095880568027497,
|
||
|
|
"num_tokens": 259850.0,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.6137940883636475,
|
||
|
|
"epoch": 0.09329940627650551,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.00016450000000000001,
|
||
|
|
"loss": 6.9622,
|
||
|
|
"mean_token_accuracy": 0.10780008137226105,
|
||
|
|
"num_tokens": 264043.0,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.65163402557373,
|
||
|
|
"epoch": 0.09471303364433135,
|
||
|
|
"grad_norm": 1.40625,
|
||
|
|
"learning_rate": 0.00016700000000000002,
|
||
|
|
"loss": 7.0215,
|
||
|
|
"mean_token_accuracy": 0.1125954583287239,
|
||
|
|
"num_tokens": 268296.0,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.420255088806153,
|
||
|
|
"epoch": 0.0961266610121572,
|
||
|
|
"grad_norm": 1.4765625,
|
||
|
|
"learning_rate": 0.00016950000000000003,
|
||
|
|
"loss": 6.836,
|
||
|
|
"mean_token_accuracy": 0.11534447073936463,
|
||
|
|
"num_tokens": 272062.0,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.355507898330688,
|
||
|
|
"epoch": 0.09754028837998303,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.00017199999999999998,
|
||
|
|
"loss": 6.8008,
|
||
|
|
"mean_token_accuracy": 0.12810793220996858,
|
||
|
|
"num_tokens": 275936.0,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.469290637969971,
|
||
|
|
"epoch": 0.09895391574780887,
|
||
|
|
"grad_norm": 1.75,
|
||
|
|
"learning_rate": 0.00017449999999999999,
|
||
|
|
"loss": 6.8213,
|
||
|
|
"mean_token_accuracy": 0.1251584582030773,
|
||
|
|
"num_tokens": 279526.0,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.5056859970092775,
|
||
|
|
"epoch": 0.10036754311563471,
|
||
|
|
"grad_norm": 1.3984375,
|
||
|
|
"learning_rate": 0.000177,
|
||
|
|
"loss": 6.8874,
|
||
|
|
"mean_token_accuracy": 0.11884394437074661,
|
||
|
|
"num_tokens": 283577.0,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.507189226150513,
|
||
|
|
"epoch": 0.10178117048346055,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.0001795,
|
||
|
|
"loss": 6.7619,
|
||
|
|
"mean_token_accuracy": 0.112952870875597,
|
||
|
|
"num_tokens": 287377.0,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.379469537734986,
|
||
|
|
"epoch": 0.10319479785128641,
|
||
|
|
"grad_norm": 1.3515625,
|
||
|
|
"learning_rate": 0.000182,
|
||
|
|
"loss": 6.9101,
|
||
|
|
"mean_token_accuracy": 0.1114804707467556,
|
||
|
|
"num_tokens": 291582.0,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.508454275131226,
|
||
|
|
"epoch": 0.10460842521911225,
|
||
|
|
"grad_norm": 1.4453125,
|
||
|
|
"learning_rate": 0.0001845,
|
||
|
|
"loss": 6.7891,
|
||
|
|
"mean_token_accuracy": 0.12308030501008034,
|
||
|
|
"num_tokens": 295608.0,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.4715001583099365,
|
||
|
|
"epoch": 0.10602205258693809,
|
||
|
|
"grad_norm": 1.671875,
|
||
|
|
"learning_rate": 0.000187,
|
||
|
|
"loss": 6.8013,
|
||
|
|
"mean_token_accuracy": 0.12096530124545098,
|
||
|
|
"num_tokens": 299660.0,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.332260084152222,
|
||
|
|
"epoch": 0.10743567995476393,
|
||
|
|
"grad_norm": 1.6328125,
|
||
|
|
"learning_rate": 0.0001895,
|
||
|
|
"loss": 6.7183,
|
||
|
|
"mean_token_accuracy": 0.13139156624674797,
|
||
|
|
"num_tokens": 303412.0,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.269100999832153,
|
||
|
|
"epoch": 0.10884930732258977,
|
||
|
|
"grad_norm": 1.6796875,
|
||
|
|
"learning_rate": 0.000192,
|
||
|
|
"loss": 6.7782,
|
||
|
|
"mean_token_accuracy": 0.12529514580965043,
|
||
|
|
"num_tokens": 307352.0,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.428966236114502,
|
||
|
|
"epoch": 0.1102629346904156,
|
||
|
|
"grad_norm": 1.75,
|
||
|
|
"learning_rate": 0.0001945,
|
||
|
|
"loss": 6.8533,
|
||
|
|
"mean_token_accuracy": 0.1281038463115692,
|
||
|
|
"num_tokens": 311418.0,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.324939203262329,
|
||
|
|
"epoch": 0.11167656205824145,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.00019700000000000002,
|
||
|
|
"loss": 6.6783,
|
||
|
|
"mean_token_accuracy": 0.13713634759187698,
|
||
|
|
"num_tokens": 315150.0,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.591144180297851,
|
||
|
|
"epoch": 0.11309018942606729,
|
||
|
|
"grad_norm": 1.6875,
|
||
|
|
"learning_rate": 0.00019950000000000002,
|
||
|
|
"loss": 6.8727,
|
||
|
|
"mean_token_accuracy": 0.11698746904730797,
|
||
|
|
"num_tokens": 318898.0,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.242686319351196,
|
||
|
|
"epoch": 0.11450381679389313,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 0.000202,
|
||
|
|
"loss": 6.6935,
|
||
|
|
"mean_token_accuracy": 0.13000213503837585,
|
||
|
|
"num_tokens": 322495.0,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.333932304382325,
|
||
|
|
"epoch": 0.11591744416171897,
|
||
|
|
"grad_norm": 1.375,
|
||
|
|
"learning_rate": 0.00020449999999999998,
|
||
|
|
"loss": 6.8398,
|
||
|
|
"mean_token_accuracy": 0.12843888401985168,
|
||
|
|
"num_tokens": 326473.0,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.408268976211548,
|
||
|
|
"epoch": 0.1173310715295448,
|
||
|
|
"grad_norm": 1.5625,
|
||
|
|
"learning_rate": 0.000207,
|
||
|
|
"loss": 6.7863,
|
||
|
|
"mean_token_accuracy": 0.12803610786795616,
|
||
|
|
"num_tokens": 330393.0,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.3277123928070065,
|
||
|
|
"epoch": 0.11874469889737066,
|
||
|
|
"grad_norm": 1.3671875,
|
||
|
|
"learning_rate": 0.0002095,
|
||
|
|
"loss": 6.7054,
|
||
|
|
"mean_token_accuracy": 0.13229965120553971,
|
||
|
|
"num_tokens": 334449.0,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.445288515090942,
|
||
|
|
"epoch": 0.1201583262651965,
|
||
|
|
"grad_norm": 1.3671875,
|
||
|
|
"learning_rate": 0.000212,
|
||
|
|
"loss": 6.8181,
|
||
|
|
"mean_token_accuracy": 0.11966062635183335,
|
||
|
|
"num_tokens": 338740.0,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.247825288772583,
|
||
|
|
"epoch": 0.12157195363302234,
|
||
|
|
"grad_norm": 1.546875,
|
||
|
|
"learning_rate": 0.0002145,
|
||
|
|
"loss": 6.6722,
|
||
|
|
"mean_token_accuracy": 0.13701122775673866,
|
||
|
|
"num_tokens": 342429.0,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.355695915222168,
|
||
|
|
"epoch": 0.12298558100084818,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.00021700000000000002,
|
||
|
|
"loss": 6.6994,
|
||
|
|
"mean_token_accuracy": 0.13420747891068457,
|
||
|
|
"num_tokens": 346218.0,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.303587532043457,
|
||
|
|
"epoch": 0.12439920836867402,
|
||
|
|
"grad_norm": 1.4140625,
|
||
|
|
"learning_rate": 0.0002195,
|
||
|
|
"loss": 6.696,
|
||
|
|
"mean_token_accuracy": 0.1381567671895027,
|
||
|
|
"num_tokens": 350310.0,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.251111030578613,
|
||
|
|
"epoch": 0.12581283573649985,
|
||
|
|
"grad_norm": 1.59375,
|
||
|
|
"learning_rate": 0.000222,
|
||
|
|
"loss": 6.6712,
|
||
|
|
"mean_token_accuracy": 0.13844901993870734,
|
||
|
|
"num_tokens": 353952.0,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.367745399475098,
|
||
|
|
"epoch": 0.1272264631043257,
|
||
|
|
"grad_norm": 1.59375,
|
||
|
|
"learning_rate": 0.0002245,
|
||
|
|
"loss": 6.851,
|
||
|
|
"mean_token_accuracy": 0.1284424804151058,
|
||
|
|
"num_tokens": 357667.0,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.356730890274048,
|
||
|
|
"epoch": 0.12864009047215155,
|
||
|
|
"grad_norm": 1.5546875,
|
||
|
|
"learning_rate": 0.00022700000000000002,
|
||
|
|
"loss": 6.6561,
|
||
|
|
"mean_token_accuracy": 0.138033926486969,
|
||
|
|
"num_tokens": 361309.0,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.101002645492554,
|
||
|
|
"epoch": 0.13005371783997738,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 0.00022950000000000002,
|
||
|
|
"loss": 6.6954,
|
||
|
|
"mean_token_accuracy": 0.1395917169749737,
|
||
|
|
"num_tokens": 365099.0,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.374086189270019,
|
||
|
|
"epoch": 0.13146734520780323,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.00023200000000000003,
|
||
|
|
"loss": 6.7496,
|
||
|
|
"mean_token_accuracy": 0.13763813823461532,
|
||
|
|
"num_tokens": 369141.0,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.359116935729981,
|
||
|
|
"epoch": 0.13288097257562906,
|
||
|
|
"grad_norm": 1.6015625,
|
||
|
|
"learning_rate": 0.00023449999999999998,
|
||
|
|
"loss": 6.6675,
|
||
|
|
"mean_token_accuracy": 0.1343040108680725,
|
||
|
|
"num_tokens": 373128.0,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.0868512153625485,
|
||
|
|
"epoch": 0.1342945999434549,
|
||
|
|
"grad_norm": 1.53125,
|
||
|
|
"learning_rate": 0.000237,
|
||
|
|
"loss": 6.6654,
|
||
|
|
"mean_token_accuracy": 0.14124030545353888,
|
||
|
|
"num_tokens": 377103.0,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.250053071975708,
|
||
|
|
"epoch": 0.13570822731128074,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 0.0002395,
|
||
|
|
"loss": 6.7219,
|
||
|
|
"mean_token_accuracy": 0.13121307715773584,
|
||
|
|
"num_tokens": 381573.0,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.313079786300659,
|
||
|
|
"epoch": 0.1371218546791066,
|
||
|
|
"grad_norm": 1.4453125,
|
||
|
|
"learning_rate": 0.000242,
|
||
|
|
"loss": 6.7674,
|
||
|
|
"mean_token_accuracy": 0.13335683792829514,
|
||
|
|
"num_tokens": 385759.0,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.177998399734497,
|
||
|
|
"epoch": 0.13853548204693242,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 0.0002445,
|
||
|
|
"loss": 6.6308,
|
||
|
|
"mean_token_accuracy": 0.14019295275211335,
|
||
|
|
"num_tokens": 389700.0,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.207444953918457,
|
||
|
|
"epoch": 0.13994910941475827,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.000247,
|
||
|
|
"loss": 6.5526,
|
||
|
|
"mean_token_accuracy": 0.1400491252541542,
|
||
|
|
"num_tokens": 393279.0,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.098467206954956,
|
||
|
|
"epoch": 0.1413627367825841,
|
||
|
|
"grad_norm": 1.625,
|
||
|
|
"learning_rate": 0.0002495,
|
||
|
|
"loss": 6.639,
|
||
|
|
"mean_token_accuracy": 0.13535916954278945,
|
||
|
|
"num_tokens": 397379.0,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.206504201889038,
|
||
|
|
"epoch": 0.14277636415040995,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 0.000252,
|
||
|
|
"loss": 6.6984,
|
||
|
|
"mean_token_accuracy": 0.13445916026830673,
|
||
|
|
"num_tokens": 401349.0,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.222359657287598,
|
||
|
|
"epoch": 0.1441899915182358,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.0002545,
|
||
|
|
"loss": 6.6017,
|
||
|
|
"mean_token_accuracy": 0.1456646129488945,
|
||
|
|
"num_tokens": 405361.0,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.192652940750122,
|
||
|
|
"epoch": 0.14560361888606163,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.000257,
|
||
|
|
"loss": 6.6375,
|
||
|
|
"mean_token_accuracy": 0.14064312353730202,
|
||
|
|
"num_tokens": 409614.0,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.041819143295288,
|
||
|
|
"epoch": 0.14701724625388748,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 0.0002595,
|
||
|
|
"loss": 6.5656,
|
||
|
|
"mean_token_accuracy": 0.14632121846079826,
|
||
|
|
"num_tokens": 413529.0,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.031795978546143,
|
||
|
|
"epoch": 0.1484308736217133,
|
||
|
|
"grad_norm": 1.6171875,
|
||
|
|
"learning_rate": 0.000262,
|
||
|
|
"loss": 6.5605,
|
||
|
|
"mean_token_accuracy": 0.14058664664626122,
|
||
|
|
"num_tokens": 417365.0,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.1277703762054445,
|
||
|
|
"epoch": 0.14984450098953916,
|
||
|
|
"grad_norm": 1.5390625,
|
||
|
|
"learning_rate": 0.00026450000000000003,
|
||
|
|
"loss": 6.5435,
|
||
|
|
"mean_token_accuracy": 0.1460522949695587,
|
||
|
|
"num_tokens": 421232.0,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.189267683029175,
|
||
|
|
"epoch": 0.151258128357365,
|
||
|
|
"grad_norm": 1.6640625,
|
||
|
|
"learning_rate": 0.00026700000000000004,
|
||
|
|
"loss": 6.6245,
|
||
|
|
"mean_token_accuracy": 0.1368570201098919,
|
||
|
|
"num_tokens": 425503.0,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.121035575866699,
|
||
|
|
"epoch": 0.15267175572519084,
|
||
|
|
"grad_norm": 1.4296875,
|
||
|
|
"learning_rate": 0.00026950000000000005,
|
||
|
|
"loss": 6.6376,
|
||
|
|
"mean_token_accuracy": 0.15090529546141623,
|
||
|
|
"num_tokens": 429770.0,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.14065523147583,
|
||
|
|
"epoch": 0.15408538309301667,
|
||
|
|
"grad_norm": 1.8046875,
|
||
|
|
"learning_rate": 0.00027200000000000005,
|
||
|
|
"loss": 6.5419,
|
||
|
|
"mean_token_accuracy": 0.14979021847248078,
|
||
|
|
"num_tokens": 433657.0,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.119353485107422,
|
||
|
|
"epoch": 0.15549901046084252,
|
||
|
|
"grad_norm": 1.6171875,
|
||
|
|
"learning_rate": 0.0002745,
|
||
|
|
"loss": 6.6487,
|
||
|
|
"mean_token_accuracy": 0.1421989232301712,
|
||
|
|
"num_tokens": 437732.0,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.953150367736816,
|
||
|
|
"epoch": 0.15691263782866835,
|
||
|
|
"grad_norm": 1.515625,
|
||
|
|
"learning_rate": 0.000277,
|
||
|
|
"loss": 6.3952,
|
||
|
|
"mean_token_accuracy": 0.15552812665700913,
|
||
|
|
"num_tokens": 441866.0,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.980320453643799,
|
||
|
|
"epoch": 0.1583262651964942,
|
||
|
|
"grad_norm": 1.6015625,
|
||
|
|
"learning_rate": 0.0002795,
|
||
|
|
"loss": 6.4783,
|
||
|
|
"mean_token_accuracy": 0.14983998835086823,
|
||
|
|
"num_tokens": 445437.0,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.0185740947723385,
|
||
|
|
"epoch": 0.15973989256432006,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 0.00028199999999999997,
|
||
|
|
"loss": 6.4586,
|
||
|
|
"mean_token_accuracy": 0.15214325338602067,
|
||
|
|
"num_tokens": 449602.0,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.880740737915039,
|
||
|
|
"epoch": 0.16115351993214588,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.0002845,
|
||
|
|
"loss": 6.3774,
|
||
|
|
"mean_token_accuracy": 0.14884328693151475,
|
||
|
|
"num_tokens": 453140.0,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.237173557281494,
|
||
|
|
"epoch": 0.16256714729997174,
|
||
|
|
"grad_norm": 1.5625,
|
||
|
|
"learning_rate": 0.000287,
|
||
|
|
"loss": 6.6234,
|
||
|
|
"mean_token_accuracy": 0.13322453647851945,
|
||
|
|
"num_tokens": 457210.0,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.069291353225708,
|
||
|
|
"epoch": 0.16398077466779756,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.0002895,
|
||
|
|
"loss": 6.5821,
|
||
|
|
"mean_token_accuracy": 0.14147427529096604,
|
||
|
|
"num_tokens": 461534.0,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.033083820343018,
|
||
|
|
"epoch": 0.16539440203562342,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.000292,
|
||
|
|
"loss": 6.4198,
|
||
|
|
"mean_token_accuracy": 0.14874927774071695,
|
||
|
|
"num_tokens": 465055.0,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.058513498306274,
|
||
|
|
"epoch": 0.16680802940344924,
|
||
|
|
"grad_norm": 1.7734375,
|
||
|
|
"learning_rate": 0.0002945,
|
||
|
|
"loss": 6.444,
|
||
|
|
"mean_token_accuracy": 0.14927454963326453,
|
||
|
|
"num_tokens": 469133.0,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.091696453094483,
|
||
|
|
"epoch": 0.1682216567712751,
|
||
|
|
"grad_norm": 1.546875,
|
||
|
|
"learning_rate": 0.000297,
|
||
|
|
"loss": 6.5729,
|
||
|
|
"mean_token_accuracy": 0.14638903141021728,
|
||
|
|
"num_tokens": 473078.0,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.0970518589019775,
|
||
|
|
"epoch": 0.16963528413910092,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.0002995,
|
||
|
|
"loss": 6.5146,
|
||
|
|
"mean_token_accuracy": 0.14790180325508118,
|
||
|
|
"num_tokens": 476988.0,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.953248834609985,
|
||
|
|
"epoch": 0.17104891150692678,
|
||
|
|
"grad_norm": 1.6484375,
|
||
|
|
"learning_rate": 0.000302,
|
||
|
|
"loss": 6.3748,
|
||
|
|
"mean_token_accuracy": 0.15677656084299088,
|
||
|
|
"num_tokens": 481170.0,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.832110261917114,
|
||
|
|
"epoch": 0.1724625388747526,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.0003045,
|
||
|
|
"loss": 6.3622,
|
||
|
|
"mean_token_accuracy": 0.1458218924701214,
|
||
|
|
"num_tokens": 484853.0,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.015735912322998,
|
||
|
|
"epoch": 0.17387616624257846,
|
||
|
|
"grad_norm": 1.5390625,
|
||
|
|
"learning_rate": 0.000307,
|
||
|
|
"loss": 6.4019,
|
||
|
|
"mean_token_accuracy": 0.15347179323434829,
|
||
|
|
"num_tokens": 488550.0,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.82008581161499,
|
||
|
|
"epoch": 0.1752897936104043,
|
||
|
|
"grad_norm": 1.515625,
|
||
|
|
"learning_rate": 0.0003095,
|
||
|
|
"loss": 6.4353,
|
||
|
|
"mean_token_accuracy": 0.15502589270472528,
|
||
|
|
"num_tokens": 492621.0,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.970222520828247,
|
||
|
|
"epoch": 0.17670342097823014,
|
||
|
|
"grad_norm": 1.671875,
|
||
|
|
"learning_rate": 0.000312,
|
||
|
|
"loss": 6.4046,
|
||
|
|
"mean_token_accuracy": 0.14566168785095215,
|
||
|
|
"num_tokens": 496627.0,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.795285272598266,
|
||
|
|
"epoch": 0.178117048346056,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.0003145,
|
||
|
|
"loss": 6.4612,
|
||
|
|
"mean_token_accuracy": 0.14936821013689042,
|
||
|
|
"num_tokens": 500794.0,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.059206104278564,
|
||
|
|
"epoch": 0.17953067571388182,
|
||
|
|
"grad_norm": 1.578125,
|
||
|
|
"learning_rate": 0.000317,
|
||
|
|
"loss": 6.4831,
|
||
|
|
"mean_token_accuracy": 0.14905075654387473,
|
||
|
|
"num_tokens": 504672.0,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.794796943664551,
|
||
|
|
"epoch": 0.18094430308170767,
|
||
|
|
"grad_norm": 1.3984375,
|
||
|
|
"learning_rate": 0.0003195,
|
||
|
|
"loss": 6.4242,
|
||
|
|
"mean_token_accuracy": 0.16338785141706466,
|
||
|
|
"num_tokens": 508834.0,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.082453823089599,
|
||
|
|
"epoch": 0.1823579304495335,
|
||
|
|
"grad_norm": 1.546875,
|
||
|
|
"learning_rate": 0.000322,
|
||
|
|
"loss": 6.388,
|
||
|
|
"mean_token_accuracy": 0.15189965814352036,
|
||
|
|
"num_tokens": 512722.0,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.872324419021607,
|
||
|
|
"epoch": 0.18377155781735935,
|
||
|
|
"grad_norm": 1.4609375,
|
||
|
|
"learning_rate": 0.00032450000000000003,
|
||
|
|
"loss": 6.4785,
|
||
|
|
"mean_token_accuracy": 0.14856359511613845,
|
||
|
|
"num_tokens": 517270.0,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.005723524093628,
|
||
|
|
"epoch": 0.18518518518518517,
|
||
|
|
"grad_norm": 1.7734375,
|
||
|
|
"learning_rate": 0.00032700000000000003,
|
||
|
|
"loss": 6.4087,
|
||
|
|
"mean_token_accuracy": 0.15056324899196624,
|
||
|
|
"num_tokens": 521438.0,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.857723665237427,
|
||
|
|
"epoch": 0.18659881255301103,
|
||
|
|
"grad_norm": 1.421875,
|
||
|
|
"learning_rate": 0.00032950000000000004,
|
||
|
|
"loss": 6.3936,
|
||
|
|
"mean_token_accuracy": 0.14879612773656845,
|
||
|
|
"num_tokens": 525890.0,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 7.036524677276612,
|
||
|
|
"epoch": 0.18801243992083685,
|
||
|
|
"grad_norm": 1.46875,
|
||
|
|
"learning_rate": 0.00033200000000000005,
|
||
|
|
"loss": 6.4232,
|
||
|
|
"mean_token_accuracy": 0.149623654037714,
|
||
|
|
"num_tokens": 529775.0,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.84811635017395,
|
||
|
|
"epoch": 0.1894260672886627,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.00033450000000000005,
|
||
|
|
"loss": 6.4021,
|
||
|
|
"mean_token_accuracy": 0.1505955122411251,
|
||
|
|
"num_tokens": 533889.0,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.79489049911499,
|
||
|
|
"epoch": 0.19083969465648856,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 0.000337,
|
||
|
|
"loss": 6.3423,
|
||
|
|
"mean_token_accuracy": 0.16277977973222732,
|
||
|
|
"num_tokens": 537701.0,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.869364881515503,
|
||
|
|
"epoch": 0.1922533220243144,
|
||
|
|
"grad_norm": 1.359375,
|
||
|
|
"learning_rate": 0.0003395,
|
||
|
|
"loss": 6.3895,
|
||
|
|
"mean_token_accuracy": 0.15102225691080093,
|
||
|
|
"num_tokens": 542160.0,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.848371076583862,
|
||
|
|
"epoch": 0.19366694939214024,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.000342,
|
||
|
|
"loss": 6.3673,
|
||
|
|
"mean_token_accuracy": 0.1594443902373314,
|
||
|
|
"num_tokens": 546317.0,
|
||
|
|
"step": 685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.84291672706604,
|
||
|
|
"epoch": 0.19508057675996607,
|
||
|
|
"grad_norm": 1.59375,
|
||
|
|
"learning_rate": 0.00034449999999999997,
|
||
|
|
"loss": 6.2123,
|
||
|
|
"mean_token_accuracy": 0.1704620197415352,
|
||
|
|
"num_tokens": 550304.0,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.779688024520874,
|
||
|
|
"epoch": 0.19649420412779192,
|
||
|
|
"grad_norm": 1.65625,
|
||
|
|
"learning_rate": 0.000347,
|
||
|
|
"loss": 6.3247,
|
||
|
|
"mean_token_accuracy": 0.1501743510365486,
|
||
|
|
"num_tokens": 554276.0,
|
||
|
|
"step": 695
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.807392883300781,
|
||
|
|
"epoch": 0.19790783149561775,
|
||
|
|
"grad_norm": 1.5859375,
|
||
|
|
"learning_rate": 0.0003495,
|
||
|
|
"loss": 6.3376,
|
||
|
|
"mean_token_accuracy": 0.15815870016813277,
|
||
|
|
"num_tokens": 558044.0,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.69501748085022,
|
||
|
|
"epoch": 0.1993214588634436,
|
||
|
|
"grad_norm": 1.6171875,
|
||
|
|
"learning_rate": 0.000352,
|
||
|
|
"loss": 6.3189,
|
||
|
|
"mean_token_accuracy": 0.15602569431066513,
|
||
|
|
"num_tokens": 561667.0,
|
||
|
|
"step": 705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.884977722167969,
|
||
|
|
"epoch": 0.20073508623126943,
|
||
|
|
"grad_norm": 1.4296875,
|
||
|
|
"learning_rate": 0.0003545,
|
||
|
|
"loss": 6.3569,
|
||
|
|
"mean_token_accuracy": 0.14620595276355744,
|
||
|
|
"num_tokens": 565738.0,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.865601634979248,
|
||
|
|
"epoch": 0.20214871359909528,
|
||
|
|
"grad_norm": 1.578125,
|
||
|
|
"learning_rate": 0.000357,
|
||
|
|
"loss": 6.3868,
|
||
|
|
"mean_token_accuracy": 0.15818096846342086,
|
||
|
|
"num_tokens": 569629.0,
|
||
|
|
"step": 715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.779451847076416,
|
||
|
|
"epoch": 0.2035623409669211,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.0003595,
|
||
|
|
"loss": 6.2616,
|
||
|
|
"mean_token_accuracy": 0.1626145839691162,
|
||
|
|
"num_tokens": 573272.0,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.587831258773804,
|
||
|
|
"epoch": 0.20497596833474696,
|
||
|
|
"grad_norm": 1.4140625,
|
||
|
|
"learning_rate": 0.000362,
|
||
|
|
"loss": 6.3088,
|
||
|
|
"mean_token_accuracy": 0.15946338027715684,
|
||
|
|
"num_tokens": 577224.0,
|
||
|
|
"step": 725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.784856128692627,
|
||
|
|
"epoch": 0.20638959570257281,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.0003645,
|
||
|
|
"loss": 6.2521,
|
||
|
|
"mean_token_accuracy": 0.16293586939573287,
|
||
|
|
"num_tokens": 580919.0,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.7658926486969,
|
||
|
|
"epoch": 0.20780322307039864,
|
||
|
|
"grad_norm": 1.609375,
|
||
|
|
"learning_rate": 0.000367,
|
||
|
|
"loss": 6.3651,
|
||
|
|
"mean_token_accuracy": 0.15333567410707474,
|
||
|
|
"num_tokens": 584985.0,
|
||
|
|
"step": 735
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.826333856582641,
|
||
|
|
"epoch": 0.2092168504382245,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.0003695,
|
||
|
|
"loss": 6.3085,
|
||
|
|
"mean_token_accuracy": 0.16295087337493896,
|
||
|
|
"num_tokens": 588979.0,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.648220825195312,
|
||
|
|
"epoch": 0.21063047780605032,
|
||
|
|
"grad_norm": 1.421875,
|
||
|
|
"learning_rate": 0.000372,
|
||
|
|
"loss": 6.2787,
|
||
|
|
"mean_token_accuracy": 0.16390681266784668,
|
||
|
|
"num_tokens": 592930.0,
|
||
|
|
"step": 745
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.663014554977417,
|
||
|
|
"epoch": 0.21204410517387617,
|
||
|
|
"grad_norm": 1.4609375,
|
||
|
|
"learning_rate": 0.0003745,
|
||
|
|
"loss": 6.3273,
|
||
|
|
"mean_token_accuracy": 0.16125326007604598,
|
||
|
|
"num_tokens": 596832.0,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.937385177612304,
|
||
|
|
"epoch": 0.213457732541702,
|
||
|
|
"grad_norm": 1.7421875,
|
||
|
|
"learning_rate": 0.000377,
|
||
|
|
"loss": 6.3617,
|
||
|
|
"mean_token_accuracy": 0.15750757604837418,
|
||
|
|
"num_tokens": 600934.0,
|
||
|
|
"step": 755
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.72532844543457,
|
||
|
|
"epoch": 0.21487135990952785,
|
||
|
|
"grad_norm": 1.6796875,
|
||
|
|
"learning_rate": 0.0003795,
|
||
|
|
"loss": 6.2373,
|
||
|
|
"mean_token_accuracy": 0.15954903662204742,
|
||
|
|
"num_tokens": 604868.0,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.742078399658203,
|
||
|
|
"epoch": 0.21628498727735368,
|
||
|
|
"grad_norm": 1.59375,
|
||
|
|
"learning_rate": 0.000382,
|
||
|
|
"loss": 6.3596,
|
||
|
|
"mean_token_accuracy": 0.150635027885437,
|
||
|
|
"num_tokens": 608747.0,
|
||
|
|
"step": 765
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.704383659362793,
|
||
|
|
"epoch": 0.21769861464517953,
|
||
|
|
"grad_norm": 1.546875,
|
||
|
|
"learning_rate": 0.0003845,
|
||
|
|
"loss": 6.3572,
|
||
|
|
"mean_token_accuracy": 0.15894525349140168,
|
||
|
|
"num_tokens": 612627.0,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.658515930175781,
|
||
|
|
"epoch": 0.21911224201300536,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.00038700000000000003,
|
||
|
|
"loss": 6.2251,
|
||
|
|
"mean_token_accuracy": 0.17111287266016006,
|
||
|
|
"num_tokens": 616455.0,
|
||
|
|
"step": 775
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.6746655941009525,
|
||
|
|
"epoch": 0.2205258693808312,
|
||
|
|
"grad_norm": 1.375,
|
||
|
|
"learning_rate": 0.00038950000000000003,
|
||
|
|
"loss": 6.326,
|
||
|
|
"mean_token_accuracy": 0.15957299321889878,
|
||
|
|
"num_tokens": 620617.0,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.608477210998535,
|
||
|
|
"epoch": 0.22193949674865707,
|
||
|
|
"grad_norm": 1.6953125,
|
||
|
|
"learning_rate": 0.00039200000000000004,
|
||
|
|
"loss": 6.2692,
|
||
|
|
"mean_token_accuracy": 0.1650322839617729,
|
||
|
|
"num_tokens": 624863.0,
|
||
|
|
"step": 785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.672313022613525,
|
||
|
|
"epoch": 0.2233531241164829,
|
||
|
|
"grad_norm": 1.4140625,
|
||
|
|
"learning_rate": 0.00039450000000000005,
|
||
|
|
"loss": 6.2819,
|
||
|
|
"mean_token_accuracy": 0.165033021569252,
|
||
|
|
"num_tokens": 629073.0,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.696346044540405,
|
||
|
|
"epoch": 0.22476675148430875,
|
||
|
|
"grad_norm": 1.53125,
|
||
|
|
"learning_rate": 0.00039700000000000005,
|
||
|
|
"loss": 6.3095,
|
||
|
|
"mean_token_accuracy": 0.16314611732959747,
|
||
|
|
"num_tokens": 633550.0,
|
||
|
|
"step": 795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.60796046257019,
|
||
|
|
"epoch": 0.22618037885213457,
|
||
|
|
"grad_norm": 1.5859375,
|
||
|
|
"learning_rate": 0.0003995,
|
||
|
|
"loss": 6.2888,
|
||
|
|
"mean_token_accuracy": 0.1602451756596565,
|
||
|
|
"num_tokens": 637459.0,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.599572229385376,
|
||
|
|
"epoch": 0.22759400621996043,
|
||
|
|
"grad_norm": 1.3125,
|
||
|
|
"learning_rate": 0.000402,
|
||
|
|
"loss": 6.2433,
|
||
|
|
"mean_token_accuracy": 0.15662450045347215,
|
||
|
|
"num_tokens": 641587.0,
|
||
|
|
"step": 805
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.646263837814331,
|
||
|
|
"epoch": 0.22900763358778625,
|
||
|
|
"grad_norm": 1.5859375,
|
||
|
|
"learning_rate": 0.0004045,
|
||
|
|
"loss": 6.421,
|
||
|
|
"mean_token_accuracy": 0.14964068084955215,
|
||
|
|
"num_tokens": 645726.0,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.735714673995972,
|
||
|
|
"epoch": 0.2304212609556121,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.00040699999999999997,
|
||
|
|
"loss": 6.2004,
|
||
|
|
"mean_token_accuracy": 0.16830723136663436,
|
||
|
|
"num_tokens": 649688.0,
|
||
|
|
"step": 815
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.588094997406006,
|
||
|
|
"epoch": 0.23183488832343793,
|
||
|
|
"grad_norm": 1.7109375,
|
||
|
|
"learning_rate": 0.0004095,
|
||
|
|
"loss": 6.2291,
|
||
|
|
"mean_token_accuracy": 0.15805790424346924,
|
||
|
|
"num_tokens": 653745.0,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.485406589508057,
|
||
|
|
"epoch": 0.23324851569126379,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 0.000412,
|
||
|
|
"loss": 6.1713,
|
||
|
|
"mean_token_accuracy": 0.1704267144203186,
|
||
|
|
"num_tokens": 657650.0,
|
||
|
|
"step": 825
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.624208736419678,
|
||
|
|
"epoch": 0.2346621430590896,
|
||
|
|
"grad_norm": 1.640625,
|
||
|
|
"learning_rate": 0.0004145,
|
||
|
|
"loss": 6.25,
|
||
|
|
"mean_token_accuracy": 0.1644523784518242,
|
||
|
|
"num_tokens": 661614.0,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.544505929946899,
|
||
|
|
"epoch": 0.23607577042691547,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.000417,
|
||
|
|
"loss": 6.1953,
|
||
|
|
"mean_token_accuracy": 0.1738327383995056,
|
||
|
|
"num_tokens": 665484.0,
|
||
|
|
"step": 835
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.586504888534546,
|
||
|
|
"epoch": 0.23748939779474132,
|
||
|
|
"grad_norm": 1.65625,
|
||
|
|
"learning_rate": 0.0004195,
|
||
|
|
"loss": 6.1872,
|
||
|
|
"mean_token_accuracy": 0.16504454612731934,
|
||
|
|
"num_tokens": 669718.0,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.625461673736572,
|
||
|
|
"epoch": 0.23890302516256715,
|
||
|
|
"grad_norm": 1.375,
|
||
|
|
"learning_rate": 0.000422,
|
||
|
|
"loss": 6.2064,
|
||
|
|
"mean_token_accuracy": 0.16486725360155105,
|
||
|
|
"num_tokens": 673913.0,
|
||
|
|
"step": 845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.616315221786499,
|
||
|
|
"epoch": 0.240316652530393,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.0004245,
|
||
|
|
"loss": 6.2657,
|
||
|
|
"mean_token_accuracy": 0.15634770691394806,
|
||
|
|
"num_tokens": 677911.0,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.552777242660523,
|
||
|
|
"epoch": 0.24173027989821882,
|
||
|
|
"grad_norm": 1.4921875,
|
||
|
|
"learning_rate": 0.000427,
|
||
|
|
"loss": 6.0792,
|
||
|
|
"mean_token_accuracy": 0.16897292137145997,
|
||
|
|
"num_tokens": 681629.0,
|
||
|
|
"step": 855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.523988723754883,
|
||
|
|
"epoch": 0.24314390726604468,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 0.0004295,
|
||
|
|
"loss": 6.1307,
|
||
|
|
"mean_token_accuracy": 0.16514982432126998,
|
||
|
|
"num_tokens": 685675.0,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.562773180007935,
|
||
|
|
"epoch": 0.2445575346338705,
|
||
|
|
"grad_norm": 1.3828125,
|
||
|
|
"learning_rate": 0.000432,
|
||
|
|
"loss": 6.0325,
|
||
|
|
"mean_token_accuracy": 0.17125135958194732,
|
||
|
|
"num_tokens": 689630.0,
|
||
|
|
"step": 865
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.422208642959594,
|
||
|
|
"epoch": 0.24597116200169636,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.0004345,
|
||
|
|
"loss": 6.1486,
|
||
|
|
"mean_token_accuracy": 0.17256792038679122,
|
||
|
|
"num_tokens": 693580.0,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.469058227539063,
|
||
|
|
"epoch": 0.24738478936952218,
|
||
|
|
"grad_norm": 1.578125,
|
||
|
|
"learning_rate": 0.000437,
|
||
|
|
"loss": 6.2098,
|
||
|
|
"mean_token_accuracy": 0.1618221327662468,
|
||
|
|
"num_tokens": 697531.0,
|
||
|
|
"step": 875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.530948734283447,
|
||
|
|
"epoch": 0.24879841673734804,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 0.0004395,
|
||
|
|
"loss": 6.1695,
|
||
|
|
"mean_token_accuracy": 0.1632336474955082,
|
||
|
|
"num_tokens": 701421.0,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.483053684234619,
|
||
|
|
"epoch": 0.2502120441051739,
|
||
|
|
"grad_norm": 1.3515625,
|
||
|
|
"learning_rate": 0.000442,
|
||
|
|
"loss": 6.1415,
|
||
|
|
"mean_token_accuracy": 0.16531084030866622,
|
||
|
|
"num_tokens": 705557.0,
|
||
|
|
"step": 885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.433886241912842,
|
||
|
|
"epoch": 0.2516256714729997,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.0004445,
|
||
|
|
"loss": 6.0636,
|
||
|
|
"mean_token_accuracy": 0.1743975192308426,
|
||
|
|
"num_tokens": 709173.0,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.3847051620483395,
|
||
|
|
"epoch": 0.25303929884082554,
|
||
|
|
"grad_norm": 1.515625,
|
||
|
|
"learning_rate": 0.000447,
|
||
|
|
"loss": 6.0107,
|
||
|
|
"mean_token_accuracy": 0.1733053870499134,
|
||
|
|
"num_tokens": 713144.0,
|
||
|
|
"step": 895
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.407188081741333,
|
||
|
|
"epoch": 0.2544529262086514,
|
||
|
|
"grad_norm": 1.375,
|
||
|
|
"learning_rate": 0.00044950000000000003,
|
||
|
|
"loss": 6.3262,
|
||
|
|
"mean_token_accuracy": 0.15888329446315766,
|
||
|
|
"num_tokens": 717187.0,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.68216462135315,
|
||
|
|
"epoch": 0.25586655357647725,
|
||
|
|
"grad_norm": 1.3203125,
|
||
|
|
"learning_rate": 0.00045200000000000004,
|
||
|
|
"loss": 6.1792,
|
||
|
|
"mean_token_accuracy": 0.16036764532327652,
|
||
|
|
"num_tokens": 721311.0,
|
||
|
|
"step": 905
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.426914548873901,
|
||
|
|
"epoch": 0.2572801809443031,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 0.00045450000000000004,
|
||
|
|
"loss": 6.1407,
|
||
|
|
"mean_token_accuracy": 0.17487923800945282,
|
||
|
|
"num_tokens": 725193.0,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.396089220046997,
|
||
|
|
"epoch": 0.2586938083121289,
|
||
|
|
"grad_norm": 1.421875,
|
||
|
|
"learning_rate": 0.00045700000000000005,
|
||
|
|
"loss": 6.1083,
|
||
|
|
"mean_token_accuracy": 0.16730499863624573,
|
||
|
|
"num_tokens": 729208.0,
|
||
|
|
"step": 915
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.535867500305176,
|
||
|
|
"epoch": 0.26010743567995476,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 0.00045950000000000006,
|
||
|
|
"loss": 6.1726,
|
||
|
|
"mean_token_accuracy": 0.17124995291233064,
|
||
|
|
"num_tokens": 733364.0,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.3439921855926515,
|
||
|
|
"epoch": 0.2615210630477806,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 0.000462,
|
||
|
|
"loss": 6.1221,
|
||
|
|
"mean_token_accuracy": 0.16099516451358795,
|
||
|
|
"num_tokens": 737549.0,
|
||
|
|
"step": 925
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.488129186630249,
|
||
|
|
"epoch": 0.26293469041560646,
|
||
|
|
"grad_norm": 1.2421875,
|
||
|
|
"learning_rate": 0.0004645,
|
||
|
|
"loss": 6.1294,
|
||
|
|
"mean_token_accuracy": 0.16554402112960814,
|
||
|
|
"num_tokens": 742009.0,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.518905687332153,
|
||
|
|
"epoch": 0.26434831778343226,
|
||
|
|
"grad_norm": 1.4609375,
|
||
|
|
"learning_rate": 0.000467,
|
||
|
|
"loss": 6.2303,
|
||
|
|
"mean_token_accuracy": 0.16439489349722863,
|
||
|
|
"num_tokens": 746258.0,
|
||
|
|
"step": 935
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.532473945617676,
|
||
|
|
"epoch": 0.2657619451512581,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 0.0004695,
|
||
|
|
"loss": 6.1433,
|
||
|
|
"mean_token_accuracy": 0.1789155274629593,
|
||
|
|
"num_tokens": 750027.0,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.243679428100586,
|
||
|
|
"epoch": 0.26717557251908397,
|
||
|
|
"grad_norm": 1.4296875,
|
||
|
|
"learning_rate": 0.000472,
|
||
|
|
"loss": 6.0869,
|
||
|
|
"mean_token_accuracy": 0.16812117397785187,
|
||
|
|
"num_tokens": 754030.0,
|
||
|
|
"step": 945
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.291708707809448,
|
||
|
|
"epoch": 0.2685891998869098,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.0004745,
|
||
|
|
"loss": 6.0033,
|
||
|
|
"mean_token_accuracy": 0.17290493547916413,
|
||
|
|
"num_tokens": 757663.0,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.357456398010254,
|
||
|
|
"epoch": 0.2700028272547357,
|
||
|
|
"grad_norm": 1.5078125,
|
||
|
|
"learning_rate": 0.000477,
|
||
|
|
"loss": 6.1225,
|
||
|
|
"mean_token_accuracy": 0.16563030779361726,
|
||
|
|
"num_tokens": 761806.0,
|
||
|
|
"step": 955
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.466005516052246,
|
||
|
|
"epoch": 0.2714164546225615,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 0.0004795,
|
||
|
|
"loss": 6.1282,
|
||
|
|
"mean_token_accuracy": 0.16799217388033866,
|
||
|
|
"num_tokens": 765900.0,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.314237213134765,
|
||
|
|
"epoch": 0.27283008199038733,
|
||
|
|
"grad_norm": 1.3984375,
|
||
|
|
"learning_rate": 0.000482,
|
||
|
|
"loss": 6.0802,
|
||
|
|
"mean_token_accuracy": 0.16931777596473693,
|
||
|
|
"num_tokens": 769842.0,
|
||
|
|
"step": 965
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.361982011795044,
|
||
|
|
"epoch": 0.2742437093582132,
|
||
|
|
"grad_norm": 1.515625,
|
||
|
|
"learning_rate": 0.0004845,
|
||
|
|
"loss": 6.2017,
|
||
|
|
"mean_token_accuracy": 0.16852450519800186,
|
||
|
|
"num_tokens": 773717.0,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.245587730407715,
|
||
|
|
"epoch": 0.27565733672603904,
|
||
|
|
"grad_norm": 1.3984375,
|
||
|
|
"learning_rate": 0.000487,
|
||
|
|
"loss": 5.96,
|
||
|
|
"mean_token_accuracy": 0.17952348291873932,
|
||
|
|
"num_tokens": 777720.0,
|
||
|
|
"step": 975
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.214972496032715,
|
||
|
|
"epoch": 0.27707096409386484,
|
||
|
|
"grad_norm": 1.4921875,
|
||
|
|
"learning_rate": 0.0004895,
|
||
|
|
"loss": 5.9622,
|
||
|
|
"mean_token_accuracy": 0.17511400282382966,
|
||
|
|
"num_tokens": 781482.0,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.306806802749634,
|
||
|
|
"epoch": 0.2784845914616907,
|
||
|
|
"grad_norm": 1.7109375,
|
||
|
|
"learning_rate": 0.000492,
|
||
|
|
"loss": 6.0785,
|
||
|
|
"mean_token_accuracy": 0.16686865836381912,
|
||
|
|
"num_tokens": 785390.0,
|
||
|
|
"step": 985
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.173373794555664,
|
||
|
|
"epoch": 0.27989821882951654,
|
||
|
|
"grad_norm": 1.421875,
|
||
|
|
"learning_rate": 0.0004945,
|
||
|
|
"loss": 6.017,
|
||
|
|
"mean_token_accuracy": 0.18208521008491516,
|
||
|
|
"num_tokens": 789464.0,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.59720025062561,
|
||
|
|
"epoch": 0.2813118461973424,
|
||
|
|
"grad_norm": 1.4921875,
|
||
|
|
"learning_rate": 0.000497,
|
||
|
|
"loss": 6.1232,
|
||
|
|
"mean_token_accuracy": 0.16400160863995553,
|
||
|
|
"num_tokens": 793229.0,
|
||
|
|
"step": 995
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 6.182021236419677,
|
||
|
|
"epoch": 0.2827254735651682,
|
||
|
|
"grad_norm": 1.4140625,
|
||
|
|
"learning_rate": 0.0004995,
|
||
|
|
"loss": 6.0322,
|
||
|
|
"mean_token_accuracy": 0.17415720224380493,
|
||
|
|
"num_tokens": 797490.0,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2827254735651682,
|
||
|
|
"eval_entropy": 6.051144407963093,
|
||
|
|
"eval_loss": 6.294586181640625,
|
||
|
|
"eval_mean_token_accuracy": 0.16992966667567475,
|
||
|
|
"eval_num_tokens": 797490.0,
|
||
|
|
"eval_runtime": 4.8981,
|
||
|
|
"eval_samples_per_second": 2303.769,
|
||
|
|
"eval_steps_per_second": 288.073,
|
||
|
|
"step": 1000
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 35360,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 10,
|
||
|
|
"save_steps": 1000,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 144869306204160.0,
|
||
|
|
"train_batch_size": 16,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|