435 lines
12 KiB
JSON
435 lines
12 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.5625,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2000,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"entropy": 1.128870008289814,
|
||
|
|
"epoch": 0.0390625,
|
||
|
|
"grad_norm": 1.1484375,
|
||
|
|
"learning_rate": 3.828125000000001e-06,
|
||
|
|
"loss": 1.2677252960205079,
|
||
|
|
"mean_token_accuracy": 0.7461660766601562,
|
||
|
|
"num_tokens": 284943.0,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.5300011084228754,
|
||
|
|
"epoch": 0.078125,
|
||
|
|
"grad_norm": 0.9296875,
|
||
|
|
"learning_rate": 7.734375e-06,
|
||
|
|
"loss": 0.5345380783081055,
|
||
|
|
"mean_token_accuracy": 0.8863006618618965,
|
||
|
|
"num_tokens": 568567.0,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.22015576468780637,
|
||
|
|
"epoch": 0.1171875,
|
||
|
|
"grad_norm": 0.73828125,
|
||
|
|
"learning_rate": 1.1640625000000002e-05,
|
||
|
|
"loss": 0.21523065567016603,
|
||
|
|
"mean_token_accuracy": 0.9527248337864875,
|
||
|
|
"num_tokens": 847818.0,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.1493791827932,
|
||
|
|
"epoch": 0.15625,
|
||
|
|
"grad_norm": 1.859375,
|
||
|
|
"learning_rate": 1.5546875e-05,
|
||
|
|
"loss": 0.14242181777954102,
|
||
|
|
"mean_token_accuracy": 0.9662237629294396,
|
||
|
|
"num_tokens": 1132068.0,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.10193594665266574,
|
||
|
|
"epoch": 0.1953125,
|
||
|
|
"grad_norm": 0.6953125,
|
||
|
|
"learning_rate": 1.9453125e-05,
|
||
|
|
"loss": 0.09328505516052246,
|
||
|
|
"mean_token_accuracy": 0.9757984274625778,
|
||
|
|
"num_tokens": 1414380.0,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.08452916431240737,
|
||
|
|
"epoch": 0.234375,
|
||
|
|
"grad_norm": 0.3671875,
|
||
|
|
"learning_rate": 1.9626736111111114e-05,
|
||
|
|
"loss": 0.07474044799804687,
|
||
|
|
"mean_token_accuracy": 0.9797678524255753,
|
||
|
|
"num_tokens": 1693722.0,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.07067977173253893,
|
||
|
|
"epoch": 0.2734375,
|
||
|
|
"grad_norm": 0.5078125,
|
||
|
|
"learning_rate": 1.9192708333333335e-05,
|
||
|
|
"loss": 0.06389600276947022,
|
||
|
|
"mean_token_accuracy": 0.9820445743203163,
|
||
|
|
"num_tokens": 1969318.0,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.056383224558085206,
|
||
|
|
"epoch": 0.3125,
|
||
|
|
"grad_norm": 0.70703125,
|
||
|
|
"learning_rate": 1.8758680555555557e-05,
|
||
|
|
"loss": 0.047198328971862796,
|
||
|
|
"mean_token_accuracy": 0.9858593827486039,
|
||
|
|
"num_tokens": 2253231.0,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.04955546020530164,
|
||
|
|
"epoch": 0.3515625,
|
||
|
|
"grad_norm": 0.2041015625,
|
||
|
|
"learning_rate": 1.8324652777777778e-05,
|
||
|
|
"loss": 0.041943116188049315,
|
||
|
|
"mean_token_accuracy": 0.9871881914138794,
|
||
|
|
"num_tokens": 2536963.0,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.042752508148550986,
|
||
|
|
"epoch": 0.390625,
|
||
|
|
"grad_norm": 0.486328125,
|
||
|
|
"learning_rate": 1.7890625000000003e-05,
|
||
|
|
"loss": 0.03515239715576172,
|
||
|
|
"mean_token_accuracy": 0.9886215713620186,
|
||
|
|
"num_tokens": 2818045.0,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.04651518008671701,
|
||
|
|
"epoch": 0.4296875,
|
||
|
|
"grad_norm": 0.2578125,
|
||
|
|
"learning_rate": 1.7456597222222224e-05,
|
||
|
|
"loss": 0.03923729181289673,
|
||
|
|
"mean_token_accuracy": 0.9879419809579849,
|
||
|
|
"num_tokens": 3095214.0,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03884609056636691,
|
||
|
|
"epoch": 0.46875,
|
||
|
|
"grad_norm": 0.251953125,
|
||
|
|
"learning_rate": 1.7022569444444446e-05,
|
||
|
|
"loss": 0.033338229656219485,
|
||
|
|
"mean_token_accuracy": 0.989266748726368,
|
||
|
|
"num_tokens": 3379390.0,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.04037581194192171,
|
||
|
|
"epoch": 0.5078125,
|
||
|
|
"grad_norm": 0.361328125,
|
||
|
|
"learning_rate": 1.6588541666666667e-05,
|
||
|
|
"loss": 0.03338507175445557,
|
||
|
|
"mean_token_accuracy": 0.9893018490076065,
|
||
|
|
"num_tokens": 3658209.0,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.0347819222509861,
|
||
|
|
"epoch": 0.546875,
|
||
|
|
"grad_norm": 0.25,
|
||
|
|
"learning_rate": 1.615451388888889e-05,
|
||
|
|
"loss": 0.02811858892440796,
|
||
|
|
"mean_token_accuracy": 0.9900272658467293,
|
||
|
|
"num_tokens": 3943542.0,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03437940481118858,
|
||
|
|
"epoch": 0.5859375,
|
||
|
|
"grad_norm": 0.275390625,
|
||
|
|
"learning_rate": 1.5720486111111114e-05,
|
||
|
|
"loss": 0.02796731948852539,
|
||
|
|
"mean_token_accuracy": 0.9901902380585671,
|
||
|
|
"num_tokens": 4219880.0,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03252768125385046,
|
||
|
|
"epoch": 0.625,
|
||
|
|
"grad_norm": 0.291015625,
|
||
|
|
"learning_rate": 1.5286458333333335e-05,
|
||
|
|
"loss": 0.02655085802078247,
|
||
|
|
"mean_token_accuracy": 0.9905701473355293,
|
||
|
|
"num_tokens": 4501872.0,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03492376470938325,
|
||
|
|
"epoch": 0.6640625,
|
||
|
|
"grad_norm": 0.30859375,
|
||
|
|
"learning_rate": 1.4852430555555556e-05,
|
||
|
|
"loss": 0.028916885852813722,
|
||
|
|
"mean_token_accuracy": 0.9901969498395919,
|
||
|
|
"num_tokens": 4777456.0,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03390513272024691,
|
||
|
|
"epoch": 0.703125,
|
||
|
|
"grad_norm": 0.23046875,
|
||
|
|
"learning_rate": 1.4418402777777778e-05,
|
||
|
|
"loss": 0.027390704154968262,
|
||
|
|
"mean_token_accuracy": 0.9903441447019578,
|
||
|
|
"num_tokens": 5058199.0,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03103839300107211,
|
||
|
|
"epoch": 0.7421875,
|
||
|
|
"grad_norm": 0.248046875,
|
||
|
|
"learning_rate": 1.3984375000000001e-05,
|
||
|
|
"loss": 0.025448167324066163,
|
||
|
|
"mean_token_accuracy": 0.990986846089363,
|
||
|
|
"num_tokens": 5346139.0,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.030520746470429003,
|
||
|
|
"epoch": 0.78125,
|
||
|
|
"grad_norm": 0.322265625,
|
||
|
|
"learning_rate": 1.3550347222222224e-05,
|
||
|
|
"loss": 0.024664547443389893,
|
||
|
|
"mean_token_accuracy": 0.9911650615930557,
|
||
|
|
"num_tokens": 5630409.0,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03099320237990469,
|
||
|
|
"epoch": 0.8203125,
|
||
|
|
"grad_norm": 0.232421875,
|
||
|
|
"learning_rate": 1.3116319444444446e-05,
|
||
|
|
"loss": 0.025322895050048828,
|
||
|
|
"mean_token_accuracy": 0.9909613582491875,
|
||
|
|
"num_tokens": 5910790.0,
|
||
|
|
"step": 1050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.03205171652603894,
|
||
|
|
"epoch": 0.859375,
|
||
|
|
"grad_norm": 0.271484375,
|
||
|
|
"learning_rate": 1.2682291666666669e-05,
|
||
|
|
"loss": 0.025876080989837645,
|
||
|
|
"mean_token_accuracy": 0.9909294083714485,
|
||
|
|
"num_tokens": 6192859.0,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.0317235212540254,
|
||
|
|
"epoch": 0.8984375,
|
||
|
|
"grad_norm": 0.25390625,
|
||
|
|
"learning_rate": 1.2248263888888889e-05,
|
||
|
|
"loss": 0.025666613578796387,
|
||
|
|
"mean_token_accuracy": 0.9910975641012192,
|
||
|
|
"num_tokens": 6470736.0,
|
||
|
|
"step": 1150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.02936750628054142,
|
||
|
|
"epoch": 0.9375,
|
||
|
|
"grad_norm": 0.1748046875,
|
||
|
|
"learning_rate": 1.1814236111111112e-05,
|
||
|
|
"loss": 0.023805558681488037,
|
||
|
|
"mean_token_accuracy": 0.9913524460792541,
|
||
|
|
"num_tokens": 6754815.0,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.027699284902773798,
|
||
|
|
"epoch": 0.9765625,
|
||
|
|
"grad_norm": 0.181640625,
|
||
|
|
"learning_rate": 1.1380208333333333e-05,
|
||
|
|
"loss": 0.02219050168991089,
|
||
|
|
"mean_token_accuracy": 0.9916341191530228,
|
||
|
|
"num_tokens": 7038952.0,
|
||
|
|
"step": 1250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.028233024883084,
|
||
|
|
"epoch": 1.015625,
|
||
|
|
"grad_norm": 0.189453125,
|
||
|
|
"learning_rate": 1.0946180555555556e-05,
|
||
|
|
"loss": 0.021835849285125733,
|
||
|
|
"mean_token_accuracy": 0.9917337450385094,
|
||
|
|
"num_tokens": 7321409.0,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.028355726511217653,
|
||
|
|
"epoch": 1.0546875,
|
||
|
|
"grad_norm": 0.21484375,
|
||
|
|
"learning_rate": 1.0512152777777778e-05,
|
||
|
|
"loss": 0.02162353277206421,
|
||
|
|
"mean_token_accuracy": 0.9918213853240013,
|
||
|
|
"num_tokens": 7600151.0,
|
||
|
|
"step": 1350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.029886504751630127,
|
||
|
|
"epoch": 1.09375,
|
||
|
|
"grad_norm": 0.2099609375,
|
||
|
|
"learning_rate": 1.0078125000000001e-05,
|
||
|
|
"loss": 0.023020203113555907,
|
||
|
|
"mean_token_accuracy": 0.9916045978665352,
|
||
|
|
"num_tokens": 7881901.0,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.029181483979336918,
|
||
|
|
"epoch": 1.1328125,
|
||
|
|
"grad_norm": 0.263671875,
|
||
|
|
"learning_rate": 9.644097222222222e-06,
|
||
|
|
"loss": 0.023541107177734374,
|
||
|
|
"mean_token_accuracy": 0.9915443989634514,
|
||
|
|
"num_tokens": 8165215.0,
|
||
|
|
"step": 1450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.027739713676273824,
|
||
|
|
"epoch": 1.171875,
|
||
|
|
"grad_norm": 0.20703125,
|
||
|
|
"learning_rate": 9.210069444444446e-06,
|
||
|
|
"loss": 0.021458487510681152,
|
||
|
|
"mean_token_accuracy": 0.991859747171402,
|
||
|
|
"num_tokens": 8446729.0,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.030413460680283607,
|
||
|
|
"epoch": 1.2109375,
|
||
|
|
"grad_norm": 0.32421875,
|
||
|
|
"learning_rate": 8.776041666666667e-06,
|
||
|
|
"loss": 0.023718643188476562,
|
||
|
|
"mean_token_accuracy": 0.9913851937651634,
|
||
|
|
"num_tokens": 8726396.0,
|
||
|
|
"step": 1550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.026663755937479435,
|
||
|
|
"epoch": 1.25,
|
||
|
|
"grad_norm": 0.189453125,
|
||
|
|
"learning_rate": 8.342013888888889e-06,
|
||
|
|
"loss": 0.019570586681365968,
|
||
|
|
"mean_token_accuracy": 0.9922190362215042,
|
||
|
|
"num_tokens": 9007932.0,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.029211496827192606,
|
||
|
|
"epoch": 1.2890625,
|
||
|
|
"grad_norm": 0.1796875,
|
||
|
|
"learning_rate": 7.907986111111112e-06,
|
||
|
|
"loss": 0.02281463861465454,
|
||
|
|
"mean_token_accuracy": 0.9916192671656608,
|
||
|
|
"num_tokens": 9284413.0,
|
||
|
|
"step": 1650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.028992561721242965,
|
||
|
|
"epoch": 1.328125,
|
||
|
|
"grad_norm": 0.251953125,
|
||
|
|
"learning_rate": 7.473958333333334e-06,
|
||
|
|
"loss": 0.022474973201751708,
|
||
|
|
"mean_token_accuracy": 0.9916322359442711,
|
||
|
|
"num_tokens": 9563771.0,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.02873168694321066,
|
||
|
|
"epoch": 1.3671875,
|
||
|
|
"grad_norm": 0.19921875,
|
||
|
|
"learning_rate": 7.039930555555556e-06,
|
||
|
|
"loss": 0.022191219329833985,
|
||
|
|
"mean_token_accuracy": 0.9916000646352768,
|
||
|
|
"num_tokens": 9847273.0,
|
||
|
|
"step": 1750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.02830368483904749,
|
||
|
|
"epoch": 1.40625,
|
||
|
|
"grad_norm": 0.2333984375,
|
||
|
|
"learning_rate": 6.605902777777779e-06,
|
||
|
|
"loss": 0.021763646602630617,
|
||
|
|
"mean_token_accuracy": 0.9919700247049331,
|
||
|
|
"num_tokens": 10127667.0,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.0277809487702325,
|
||
|
|
"epoch": 1.4453125,
|
||
|
|
"grad_norm": 0.19921875,
|
||
|
|
"learning_rate": 6.171875e-06,
|
||
|
|
"loss": 0.021228244304656984,
|
||
|
|
"mean_token_accuracy": 0.9920018136501312,
|
||
|
|
"num_tokens": 10406050.0,
|
||
|
|
"step": 1850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.026233949079178275,
|
||
|
|
"epoch": 1.484375,
|
||
|
|
"grad_norm": 0.212890625,
|
||
|
|
"learning_rate": 5.737847222222222e-06,
|
||
|
|
"loss": 0.01996251106262207,
|
||
|
|
"mean_token_accuracy": 0.9923433768749237,
|
||
|
|
"num_tokens": 10690620.0,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.027143093338236213,
|
||
|
|
"epoch": 1.5234375,
|
||
|
|
"grad_norm": 0.236328125,
|
||
|
|
"learning_rate": 5.303819444444445e-06,
|
||
|
|
"loss": 0.02035560369491577,
|
||
|
|
"mean_token_accuracy": 0.9922614651918411,
|
||
|
|
"num_tokens": 10974611.0,
|
||
|
|
"step": 1950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 0.02908464324194938,
|
||
|
|
"epoch": 1.5625,
|
||
|
|
"grad_norm": 0.265625,
|
||
|
|
"learning_rate": 4.869791666666667e-06,
|
||
|
|
"loss": 0.02261843204498291,
|
||
|
|
"mean_token_accuracy": 0.9916498929262161,
|
||
|
|
"num_tokens": 11254903.0,
|
||
|
|
"step": 2000
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 50,
|
||
|
|
"max_steps": 2560,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 2,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 2.0986070119686144e+17,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|