9045 lines
281 KiB
JSON
9045 lines
281 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 2.0,
|
|
"eval_steps": 3206,
|
|
"global_step": 6412,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0,
|
|
"eval_loss": 90.1102294921875,
|
|
"eval_ppl": 1.3626221098118503e+39,
|
|
"eval_runtime": 41.4918,
|
|
"eval_samples_per_second": 65.073,
|
|
"eval_steps_per_second": 32.537,
|
|
"memory/device_reserved (GiB)": 2.01,
|
|
"memory/max_active (GiB)": 0.86,
|
|
"memory/max_allocated (GiB)": 0.86,
|
|
"step": 0
|
|
},
|
|
{
|
|
"epoch": 0.003119394837401544,
|
|
"grad_norm": 49.6260871887207,
|
|
"learning_rate": 2.3437500000000002e-06,
|
|
"loss": 82.08431396484374,
|
|
"memory/device_reserved (GiB)": 4.42,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.454147613426413e+35,
|
|
"step": 10,
|
|
"tokens/total": 63744,
|
|
"tokens/train_per_sec_per_gpu": 87.13,
|
|
"tokens/trainable": 41268
|
|
},
|
|
{
|
|
"epoch": 0.006238789674803088,
|
|
"grad_norm": 47.38286209106445,
|
|
"learning_rate": 4.947916666666666e-06,
|
|
"loss": 68.94780883789062,
|
|
"memory/device_reserved (GiB)": 4.42,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.783202943469593e+29,
|
|
"step": 20,
|
|
"tokens/total": 128384,
|
|
"tokens/train_per_sec_per_gpu": 72.28,
|
|
"tokens/trainable": 83185
|
|
},
|
|
{
|
|
"epoch": 0.009358184512204633,
|
|
"grad_norm": 40.237239837646484,
|
|
"learning_rate": 7.552083333333333e-06,
|
|
"loss": 38.969424438476565,
|
|
"memory/device_reserved (GiB)": 4.42,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.398582575271874e+16,
|
|
"step": 30,
|
|
"tokens/total": 195456,
|
|
"tokens/train_per_sec_per_gpu": 38.27,
|
|
"tokens/trainable": 126526
|
|
},
|
|
{
|
|
"epoch": 0.012477579349606176,
|
|
"grad_norm": 5.358484268188477,
|
|
"learning_rate": 1.0156250000000001e-05,
|
|
"loss": 8.704107666015625,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6027.62089,
|
|
"step": 40,
|
|
"tokens/total": 260480,
|
|
"tokens/train_per_sec_per_gpu": 48.44,
|
|
"tokens/trainable": 168410
|
|
},
|
|
{
|
|
"epoch": 0.015596974187007721,
|
|
"grad_norm": 1.5739214420318604,
|
|
"learning_rate": 1.2760416666666666e-05,
|
|
"loss": 3.963784027099609,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 52.6562,
|
|
"step": 50,
|
|
"tokens/total": 328064,
|
|
"tokens/train_per_sec_per_gpu": 65.19,
|
|
"tokens/trainable": 213107
|
|
},
|
|
{
|
|
"epoch": 0.018716369024409266,
|
|
"grad_norm": 1.625490665435791,
|
|
"learning_rate": 1.5364583333333335e-05,
|
|
"loss": 3.2287776947021483,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 25.24878,
|
|
"step": 60,
|
|
"tokens/total": 392704,
|
|
"tokens/train_per_sec_per_gpu": 61.36,
|
|
"tokens/trainable": 255044
|
|
},
|
|
{
|
|
"epoch": 0.021835763861810808,
|
|
"grad_norm": 0.4607709050178528,
|
|
"learning_rate": 1.796875e-05,
|
|
"loss": 2.828261947631836,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 16.91603,
|
|
"step": 70,
|
|
"tokens/total": 458496,
|
|
"tokens/train_per_sec_per_gpu": 60.93,
|
|
"tokens/trainable": 297001
|
|
},
|
|
{
|
|
"epoch": 0.024955158699212353,
|
|
"grad_norm": 0.7430190443992615,
|
|
"learning_rate": 2.0572916666666668e-05,
|
|
"loss": 2.623426818847656,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 13.78287,
|
|
"step": 80,
|
|
"tokens/total": 522624,
|
|
"tokens/train_per_sec_per_gpu": 61.94,
|
|
"tokens/trainable": 340094
|
|
},
|
|
{
|
|
"epoch": 0.028074553536613898,
|
|
"grad_norm": 0.4917757511138916,
|
|
"learning_rate": 2.3177083333333335e-05,
|
|
"loss": 2.547990417480469,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 12.78139,
|
|
"step": 90,
|
|
"tokens/total": 589696,
|
|
"tokens/train_per_sec_per_gpu": 56.2,
|
|
"tokens/trainable": 383310
|
|
},
|
|
{
|
|
"epoch": 0.031193948374015443,
|
|
"grad_norm": 0.3311872184276581,
|
|
"learning_rate": 2.578125e-05,
|
|
"loss": 2.5273923873901367,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 12.52081,
|
|
"step": 100,
|
|
"tokens/total": 656000,
|
|
"tokens/train_per_sec_per_gpu": 57.83,
|
|
"tokens/trainable": 426832
|
|
},
|
|
{
|
|
"epoch": 0.034313343211416984,
|
|
"grad_norm": 0.34330612421035767,
|
|
"learning_rate": 2.838541666666667e-05,
|
|
"loss": 2.350965118408203,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 10.49569,
|
|
"step": 110,
|
|
"tokens/total": 721536,
|
|
"tokens/train_per_sec_per_gpu": 96.23,
|
|
"tokens/trainable": 470517
|
|
},
|
|
{
|
|
"epoch": 0.03743273804881853,
|
|
"grad_norm": 0.3654789924621582,
|
|
"learning_rate": 3.0989583333333336e-05,
|
|
"loss": 2.3728742599487305,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 10.72818,
|
|
"step": 120,
|
|
"tokens/total": 788992,
|
|
"tokens/train_per_sec_per_gpu": 76.1,
|
|
"tokens/trainable": 513147
|
|
},
|
|
{
|
|
"epoch": 0.040552132886220074,
|
|
"grad_norm": 0.33035415410995483,
|
|
"learning_rate": 3.359375e-05,
|
|
"loss": 2.380273628234863,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 10.80786,
|
|
"step": 130,
|
|
"tokens/total": 852224,
|
|
"tokens/train_per_sec_per_gpu": 58.62,
|
|
"tokens/trainable": 554040
|
|
},
|
|
{
|
|
"epoch": 0.043671527723621616,
|
|
"grad_norm": 0.2839641869068146,
|
|
"learning_rate": 3.619791666666667e-05,
|
|
"loss": 2.198690414428711,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 9.0132,
|
|
"step": 140,
|
|
"tokens/total": 918784,
|
|
"tokens/train_per_sec_per_gpu": 71.63,
|
|
"tokens/trainable": 596747
|
|
},
|
|
{
|
|
"epoch": 0.046790922561023164,
|
|
"grad_norm": 0.35258179903030396,
|
|
"learning_rate": 3.8802083333333336e-05,
|
|
"loss": 2.2403385162353517,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 9.39651,
|
|
"step": 150,
|
|
"tokens/total": 982784,
|
|
"tokens/train_per_sec_per_gpu": 21.12,
|
|
"tokens/trainable": 637282
|
|
},
|
|
{
|
|
"epoch": 0.049910317398424706,
|
|
"grad_norm": 0.2685360014438629,
|
|
"learning_rate": 4.140625e-05,
|
|
"loss": 2.174555206298828,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.79827,
|
|
"step": 160,
|
|
"tokens/total": 1048576,
|
|
"tokens/train_per_sec_per_gpu": 65.91,
|
|
"tokens/trainable": 678960
|
|
},
|
|
{
|
|
"epoch": 0.05302971223582625,
|
|
"grad_norm": 0.29472798109054565,
|
|
"learning_rate": 4.401041666666667e-05,
|
|
"loss": 2.1382368087768553,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.48446,
|
|
"step": 170,
|
|
"tokens/total": 1115264,
|
|
"tokens/train_per_sec_per_gpu": 72.5,
|
|
"tokens/trainable": 721314
|
|
},
|
|
{
|
|
"epoch": 0.056149107073227796,
|
|
"grad_norm": 0.2625071406364441,
|
|
"learning_rate": 4.6614583333333336e-05,
|
|
"loss": 2.1903257369995117,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.93812,
|
|
"step": 180,
|
|
"tokens/total": 1178496,
|
|
"tokens/train_per_sec_per_gpu": 86.67,
|
|
"tokens/trainable": 761676
|
|
},
|
|
{
|
|
"epoch": 0.05926850191062934,
|
|
"grad_norm": 0.2765958607196808,
|
|
"learning_rate": 4.921875e-05,
|
|
"loss": 2.144666290283203,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.53919,
|
|
"step": 190,
|
|
"tokens/total": 1243776,
|
|
"tokens/train_per_sec_per_gpu": 59.35,
|
|
"tokens/trainable": 803745
|
|
},
|
|
{
|
|
"epoch": 0.062387896748030885,
|
|
"grad_norm": 0.3011992573738098,
|
|
"learning_rate": 4.9999843748348384e-05,
|
|
"loss": 2.145949363708496,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.55015,
|
|
"step": 200,
|
|
"tokens/total": 1308416,
|
|
"tokens/train_per_sec_per_gpu": 75.54,
|
|
"tokens/trainable": 845619
|
|
},
|
|
{
|
|
"epoch": 0.06550729158543242,
|
|
"grad_norm": 0.2674703299999237,
|
|
"learning_rate": 4.999907843883827e-05,
|
|
"loss": 2.12449951171875,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 8.36871,
|
|
"step": 210,
|
|
"tokens/total": 1371904,
|
|
"tokens/train_per_sec_per_gpu": 65.38,
|
|
"tokens/trainable": 886842
|
|
},
|
|
{
|
|
"epoch": 0.06862668642283397,
|
|
"grad_norm": 0.27003955841064453,
|
|
"learning_rate": 4.99976753916857e-05,
|
|
"loss": 2.0180612564086915,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.52372,
|
|
"step": 220,
|
|
"tokens/total": 1435008,
|
|
"tokens/train_per_sec_per_gpu": 63.63,
|
|
"tokens/trainable": 928518
|
|
},
|
|
{
|
|
"epoch": 0.07174608126023552,
|
|
"grad_norm": 0.2658486068248749,
|
|
"learning_rate": 4.999563464268304e-05,
|
|
"loss": 2.078030014038086,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.98872,
|
|
"step": 230,
|
|
"tokens/total": 1499776,
|
|
"tokens/train_per_sec_per_gpu": 66.59,
|
|
"tokens/trainable": 969894
|
|
},
|
|
{
|
|
"epoch": 0.07486547609763707,
|
|
"grad_norm": 0.24339734017848969,
|
|
"learning_rate": 4.9992956243890696e-05,
|
|
"loss": 2.0598146438598635,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.84452,
|
|
"step": 240,
|
|
"tokens/total": 1567360,
|
|
"tokens/train_per_sec_per_gpu": 81.4,
|
|
"tokens/trainable": 1012853
|
|
},
|
|
{
|
|
"epoch": 0.0779848709350386,
|
|
"grad_norm": 0.2877941131591797,
|
|
"learning_rate": 4.9989640263635816e-05,
|
|
"loss": 1.9591102600097656,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.09301,
|
|
"step": 250,
|
|
"tokens/total": 1632896,
|
|
"tokens/train_per_sec_per_gpu": 86.82,
|
|
"tokens/trainable": 1054907
|
|
},
|
|
{
|
|
"epoch": 0.08110426577244015,
|
|
"grad_norm": 0.29160985350608826,
|
|
"learning_rate": 4.9985686786510546e-05,
|
|
"loss": 2.0569690704345702,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.82223,
|
|
"step": 260,
|
|
"tokens/total": 1699840,
|
|
"tokens/train_per_sec_per_gpu": 55.77,
|
|
"tokens/trainable": 1098294
|
|
},
|
|
{
|
|
"epoch": 0.0842236606098417,
|
|
"grad_norm": 0.23016823828220367,
|
|
"learning_rate": 4.998109591336984e-05,
|
|
"loss": 2.0404693603515627,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.69422,
|
|
"step": 270,
|
|
"tokens/total": 1765120,
|
|
"tokens/train_per_sec_per_gpu": 31.18,
|
|
"tokens/trainable": 1140277
|
|
},
|
|
{
|
|
"epoch": 0.08734305544724323,
|
|
"grad_norm": 0.2534622251987457,
|
|
"learning_rate": 4.9975867761328914e-05,
|
|
"loss": 2.0554771423339844,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.81056,
|
|
"step": 280,
|
|
"tokens/total": 1832576,
|
|
"tokens/train_per_sec_per_gpu": 38.17,
|
|
"tokens/trainable": 1183897
|
|
},
|
|
{
|
|
"epoch": 0.09046245028464478,
|
|
"grad_norm": 0.22597584128379822,
|
|
"learning_rate": 4.997000246376028e-05,
|
|
"loss": 1.9891469955444336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.3093,
|
|
"step": 290,
|
|
"tokens/total": 1899648,
|
|
"tokens/train_per_sec_per_gpu": 73.85,
|
|
"tokens/trainable": 1227795
|
|
},
|
|
{
|
|
"epoch": 0.09358184512204633,
|
|
"grad_norm": 0.253947377204895,
|
|
"learning_rate": 4.996350017029028e-05,
|
|
"loss": 1.95814208984375,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.08615,
|
|
"step": 300,
|
|
"tokens/total": 1963904,
|
|
"tokens/train_per_sec_per_gpu": 75.48,
|
|
"tokens/trainable": 1270604
|
|
},
|
|
{
|
|
"epoch": 0.09670123995944786,
|
|
"grad_norm": 0.23848193883895874,
|
|
"learning_rate": 4.995636104679532e-05,
|
|
"loss": 1.94830322265625,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.01677,
|
|
"step": 310,
|
|
"tokens/total": 2032000,
|
|
"tokens/train_per_sec_per_gpu": 53.96,
|
|
"tokens/trainable": 1316115
|
|
},
|
|
{
|
|
"epoch": 0.09982063479684941,
|
|
"grad_norm": 0.23084941506385803,
|
|
"learning_rate": 4.994858527539763e-05,
|
|
"loss": 1.982783317565918,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.26293,
|
|
"step": 320,
|
|
"tokens/total": 2099968,
|
|
"tokens/train_per_sec_per_gpu": 49.35,
|
|
"tokens/trainable": 1359094
|
|
},
|
|
{
|
|
"epoch": 0.10294002963425096,
|
|
"grad_norm": 0.21452927589416504,
|
|
"learning_rate": 4.9940173054460606e-05,
|
|
"loss": 1.9222988128662108,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.83666,
|
|
"step": 330,
|
|
"tokens/total": 2164736,
|
|
"tokens/train_per_sec_per_gpu": 96.81,
|
|
"tokens/trainable": 1401319
|
|
},
|
|
{
|
|
"epoch": 0.1060594244716525,
|
|
"grad_norm": 0.22331616282463074,
|
|
"learning_rate": 4.993112459858377e-05,
|
|
"loss": 2.0320587158203125,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.62978,
|
|
"step": 340,
|
|
"tokens/total": 2231552,
|
|
"tokens/train_per_sec_per_gpu": 64.15,
|
|
"tokens/trainable": 1442902
|
|
},
|
|
{
|
|
"epoch": 0.10917881930905404,
|
|
"grad_norm": 0.2363523244857788,
|
|
"learning_rate": 4.992144013859724e-05,
|
|
"loss": 2.023745536804199,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.56661,
|
|
"step": 350,
|
|
"tokens/total": 2295552,
|
|
"tokens/train_per_sec_per_gpu": 54.8,
|
|
"tokens/trainable": 1482815
|
|
},
|
|
{
|
|
"epoch": 0.11229821414645559,
|
|
"grad_norm": 0.24678057432174683,
|
|
"learning_rate": 4.9911119921555925e-05,
|
|
"loss": 2.028571891784668,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.60322,
|
|
"step": 360,
|
|
"tokens/total": 2362624,
|
|
"tokens/train_per_sec_per_gpu": 84.0,
|
|
"tokens/trainable": 1526685
|
|
},
|
|
{
|
|
"epoch": 0.11541760898385713,
|
|
"grad_norm": 0.24551840126514435,
|
|
"learning_rate": 4.9900164210733166e-05,
|
|
"loss": 1.9418210983276367,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.97144,
|
|
"step": 370,
|
|
"tokens/total": 2427648,
|
|
"tokens/train_per_sec_per_gpu": 73.57,
|
|
"tokens/trainable": 1569062
|
|
},
|
|
{
|
|
"epoch": 0.11853700382125867,
|
|
"grad_norm": 0.21740281581878662,
|
|
"learning_rate": 4.988857328561404e-05,
|
|
"loss": 1.9779529571533203,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.22793,
|
|
"step": 380,
|
|
"tokens/total": 2492160,
|
|
"tokens/train_per_sec_per_gpu": 58.52,
|
|
"tokens/trainable": 1610460
|
|
},
|
|
{
|
|
"epoch": 0.12165639865866022,
|
|
"grad_norm": 0.20365497469902039,
|
|
"learning_rate": 4.98763474418882e-05,
|
|
"loss": 1.9513744354248046,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.03835,
|
|
"step": 390,
|
|
"tokens/total": 2560000,
|
|
"tokens/train_per_sec_per_gpu": 76.7,
|
|
"tokens/trainable": 1653533
|
|
},
|
|
{
|
|
"epoch": 0.12477579349606177,
|
|
"grad_norm": 0.2227945774793625,
|
|
"learning_rate": 4.98634869914424e-05,
|
|
"loss": 1.893275260925293,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.64108,
|
|
"step": 400,
|
|
"tokens/total": 2626688,
|
|
"tokens/train_per_sec_per_gpu": 71.05,
|
|
"tokens/trainable": 1696971
|
|
},
|
|
{
|
|
"epoch": 0.1278951883334633,
|
|
"grad_norm": 0.22460892796516418,
|
|
"learning_rate": 4.984999226235243e-05,
|
|
"loss": 1.878118896484375,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.54119,
|
|
"step": 410,
|
|
"tokens/total": 2691200,
|
|
"tokens/train_per_sec_per_gpu": 73.25,
|
|
"tokens/trainable": 1737617
|
|
},
|
|
{
|
|
"epoch": 0.13101458317086484,
|
|
"grad_norm": 0.24383381009101868,
|
|
"learning_rate": 4.983586359887488e-05,
|
|
"loss": 1.9458166122436524,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.99935,
|
|
"step": 420,
|
|
"tokens/total": 2756736,
|
|
"tokens/train_per_sec_per_gpu": 66.68,
|
|
"tokens/trainable": 1779608
|
|
},
|
|
{
|
|
"epoch": 0.1341339780082664,
|
|
"grad_norm": 0.25375816226005554,
|
|
"learning_rate": 4.982110136143825e-05,
|
|
"loss": 1.9604574203491212,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.10258,
|
|
"step": 430,
|
|
"tokens/total": 2821120,
|
|
"tokens/train_per_sec_per_gpu": 94.13,
|
|
"tokens/trainable": 1820665
|
|
},
|
|
{
|
|
"epoch": 0.13725337284566794,
|
|
"grad_norm": 0.265867680311203,
|
|
"learning_rate": 4.980570592663379e-05,
|
|
"loss": 1.8942991256713868,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.64789,
|
|
"step": 440,
|
|
"tokens/total": 2887680,
|
|
"tokens/train_per_sec_per_gpu": 46.8,
|
|
"tokens/trainable": 1862721
|
|
},
|
|
{
|
|
"epoch": 0.14037276768306947,
|
|
"grad_norm": 0.2029924988746643,
|
|
"learning_rate": 4.978967768720592e-05,
|
|
"loss": 1.9583168029785156,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.08739,
|
|
"step": 450,
|
|
"tokens/total": 2954752,
|
|
"tokens/train_per_sec_per_gpu": 92.39,
|
|
"tokens/trainable": 1903921
|
|
},
|
|
{
|
|
"epoch": 0.14349216252047103,
|
|
"grad_norm": 0.1997372806072235,
|
|
"learning_rate": 4.9773017052042185e-05,
|
|
"loss": 1.8682659149169922,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.47705,
|
|
"step": 460,
|
|
"tokens/total": 3021056,
|
|
"tokens/train_per_sec_per_gpu": 90.48,
|
|
"tokens/trainable": 1946715
|
|
},
|
|
{
|
|
"epoch": 0.14661155735787257,
|
|
"grad_norm": 0.2532586455345154,
|
|
"learning_rate": 4.97557244461628e-05,
|
|
"loss": 1.9623315811157227,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 7.1159,
|
|
"step": 470,
|
|
"tokens/total": 3084416,
|
|
"tokens/train_per_sec_per_gpu": 50.7,
|
|
"tokens/trainable": 1986684
|
|
},
|
|
{
|
|
"epoch": 0.14973095219527413,
|
|
"grad_norm": 0.21862687170505524,
|
|
"learning_rate": 4.973780031070986e-05,
|
|
"loss": 1.929121971130371,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.88346,
|
|
"step": 480,
|
|
"tokens/total": 3148416,
|
|
"tokens/train_per_sec_per_gpu": 56.65,
|
|
"tokens/trainable": 2028189
|
|
},
|
|
{
|
|
"epoch": 0.15285034703267567,
|
|
"grad_norm": 0.1988321989774704,
|
|
"learning_rate": 4.971924510293607e-05,
|
|
"loss": 1.9046140670776368,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.71681,
|
|
"step": 490,
|
|
"tokens/total": 3214464,
|
|
"tokens/train_per_sec_per_gpu": 91.97,
|
|
"tokens/trainable": 2069839
|
|
},
|
|
{
|
|
"epoch": 0.1559697418700772,
|
|
"grad_norm": 0.2142348736524582,
|
|
"learning_rate": 4.970005929619305e-05,
|
|
"loss": 1.9045169830322266,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.71616,
|
|
"step": 500,
|
|
"tokens/total": 3276544,
|
|
"tokens/train_per_sec_per_gpu": 87.96,
|
|
"tokens/trainable": 2109730
|
|
},
|
|
{
|
|
"epoch": 0.15908913670747876,
|
|
"grad_norm": 0.22039233148097992,
|
|
"learning_rate": 4.968024337991929e-05,
|
|
"loss": 1.8789686203002929,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.54675,
|
|
"step": 510,
|
|
"tokens/total": 3340032,
|
|
"tokens/train_per_sec_per_gpu": 63.22,
|
|
"tokens/trainable": 2150541
|
|
},
|
|
{
|
|
"epoch": 0.1622085315448803,
|
|
"grad_norm": 0.2228291779756546,
|
|
"learning_rate": 4.965979785962764e-05,
|
|
"loss": 1.8697526931762696,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.48669,
|
|
"step": 520,
|
|
"tokens/total": 3406720,
|
|
"tokens/train_per_sec_per_gpu": 71.32,
|
|
"tokens/trainable": 2192549
|
|
},
|
|
{
|
|
"epoch": 0.16532792638228183,
|
|
"grad_norm": 0.21713857352733612,
|
|
"learning_rate": 4.963872325689247e-05,
|
|
"loss": 1.8854345321655273,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.58922,
|
|
"step": 530,
|
|
"tokens/total": 3472128,
|
|
"tokens/train_per_sec_per_gpu": 59.77,
|
|
"tokens/trainable": 2234461
|
|
},
|
|
{
|
|
"epoch": 0.1684473212196834,
|
|
"grad_norm": 0.30795279145240784,
|
|
"learning_rate": 4.9617020109336264e-05,
|
|
"loss": 1.830658531188965,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.23799,
|
|
"step": 540,
|
|
"tokens/total": 3536384,
|
|
"tokens/train_per_sec_per_gpu": 66.47,
|
|
"tokens/trainable": 2274983
|
|
},
|
|
{
|
|
"epoch": 0.17156671605708493,
|
|
"grad_norm": 0.20807485282421112,
|
|
"learning_rate": 4.959468897061602e-05,
|
|
"loss": 1.8827404022216796,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.57149,
|
|
"step": 550,
|
|
"tokens/total": 3603840,
|
|
"tokens/train_per_sec_per_gpu": 55.17,
|
|
"tokens/trainable": 2319397
|
|
},
|
|
{
|
|
"epoch": 0.17468611089448646,
|
|
"grad_norm": 0.22363832592964172,
|
|
"learning_rate": 4.9571730410409046e-05,
|
|
"loss": 1.830014419555664,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.23398,
|
|
"step": 560,
|
|
"tokens/total": 3667712,
|
|
"tokens/train_per_sec_per_gpu": 111.68,
|
|
"tokens/trainable": 2361297
|
|
},
|
|
{
|
|
"epoch": 0.17780550573188802,
|
|
"grad_norm": 0.1778305619955063,
|
|
"learning_rate": 4.954814501439845e-05,
|
|
"loss": 1.9051851272583007,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.72065,
|
|
"step": 570,
|
|
"tokens/total": 3735168,
|
|
"tokens/train_per_sec_per_gpu": 73.9,
|
|
"tokens/trainable": 2404248
|
|
},
|
|
{
|
|
"epoch": 0.18092490056928956,
|
|
"grad_norm": 0.22004452347755432,
|
|
"learning_rate": 4.9523933384258196e-05,
|
|
"loss": 1.8310930252075195,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.2407,
|
|
"step": 580,
|
|
"tokens/total": 3801728,
|
|
"tokens/train_per_sec_per_gpu": 70.79,
|
|
"tokens/trainable": 2446966
|
|
},
|
|
{
|
|
"epoch": 0.1840442954066911,
|
|
"grad_norm": 0.23684827983379364,
|
|
"learning_rate": 4.949909613763778e-05,
|
|
"loss": 1.8625202178955078,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.43995,
|
|
"step": 590,
|
|
"tokens/total": 3868288,
|
|
"tokens/train_per_sec_per_gpu": 91.98,
|
|
"tokens/trainable": 2488800
|
|
},
|
|
{
|
|
"epoch": 0.18716369024409266,
|
|
"grad_norm": 0.21346761286258698,
|
|
"learning_rate": 4.947363390814645e-05,
|
|
"loss": 1.7853128433227539,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.96144,
|
|
"step": 600,
|
|
"tokens/total": 3933952,
|
|
"tokens/train_per_sec_per_gpu": 64.59,
|
|
"tokens/trainable": 2530801
|
|
},
|
|
{
|
|
"epoch": 0.1902830850814942,
|
|
"grad_norm": 0.21022866666316986,
|
|
"learning_rate": 4.944754734533702e-05,
|
|
"loss": 1.8581354141235351,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.41177,
|
|
"step": 610,
|
|
"tokens/total": 3997824,
|
|
"tokens/train_per_sec_per_gpu": 94.27,
|
|
"tokens/trainable": 2573724
|
|
},
|
|
{
|
|
"epoch": 0.19340247991889573,
|
|
"grad_norm": 0.20024116337299347,
|
|
"learning_rate": 4.942083711468935e-05,
|
|
"loss": 1.8690828323364257,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.48235,
|
|
"step": 620,
|
|
"tokens/total": 4061696,
|
|
"tokens/train_per_sec_per_gpu": 74.39,
|
|
"tokens/trainable": 2614352
|
|
},
|
|
{
|
|
"epoch": 0.1965218747562973,
|
|
"grad_norm": 0.20344480872154236,
|
|
"learning_rate": 4.9393503897593356e-05,
|
|
"loss": 1.8246026992797852,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.20033,
|
|
"step": 630,
|
|
"tokens/total": 4129408,
|
|
"tokens/train_per_sec_per_gpu": 55.38,
|
|
"tokens/trainable": 2657033
|
|
},
|
|
{
|
|
"epoch": 0.19964126959369882,
|
|
"grad_norm": 0.20136019587516785,
|
|
"learning_rate": 4.936554839133158e-05,
|
|
"loss": 1.8994573593139648,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.68227,
|
|
"step": 640,
|
|
"tokens/total": 4195712,
|
|
"tokens/train_per_sec_per_gpu": 53.79,
|
|
"tokens/trainable": 2700119
|
|
},
|
|
{
|
|
"epoch": 0.20276066443110036,
|
|
"grad_norm": 0.17751505970954895,
|
|
"learning_rate": 4.933697130906145e-05,
|
|
"loss": 1.8436349868774413,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.31947,
|
|
"step": 650,
|
|
"tokens/total": 4262784,
|
|
"tokens/train_per_sec_per_gpu": 72.48,
|
|
"tokens/trainable": 2743198
|
|
},
|
|
{
|
|
"epoch": 0.20588005926850192,
|
|
"grad_norm": 0.19823996722698212,
|
|
"learning_rate": 4.930777337979711e-05,
|
|
"loss": 1.8444677352905274,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.32473,
|
|
"step": 660,
|
|
"tokens/total": 4329984,
|
|
"tokens/train_per_sec_per_gpu": 80.08,
|
|
"tokens/trainable": 2785644
|
|
},
|
|
{
|
|
"epoch": 0.20899945410590345,
|
|
"grad_norm": 0.21063409745693207,
|
|
"learning_rate": 4.927795534839073e-05,
|
|
"loss": 1.8350608825683594,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.26552,
|
|
"step": 670,
|
|
"tokens/total": 4396672,
|
|
"tokens/train_per_sec_per_gpu": 43.1,
|
|
"tokens/trainable": 2827993
|
|
},
|
|
{
|
|
"epoch": 0.212118848943305,
|
|
"grad_norm": 0.22310243546962738,
|
|
"learning_rate": 4.9247517975513614e-05,
|
|
"loss": 1.8371290206909179,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.27849,
|
|
"step": 680,
|
|
"tokens/total": 4462976,
|
|
"tokens/train_per_sec_per_gpu": 77.59,
|
|
"tokens/trainable": 2871361
|
|
},
|
|
{
|
|
"epoch": 0.21523824378070655,
|
|
"grad_norm": 0.19987933337688446,
|
|
"learning_rate": 4.921646203763671e-05,
|
|
"loss": 1.8426168441772461,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.31304,
|
|
"step": 690,
|
|
"tokens/total": 4530304,
|
|
"tokens/train_per_sec_per_gpu": 69.73,
|
|
"tokens/trainable": 2914430
|
|
},
|
|
{
|
|
"epoch": 0.21835763861810809,
|
|
"grad_norm": 0.18348121643066406,
|
|
"learning_rate": 4.918478832701085e-05,
|
|
"loss": 1.8888889312744142,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.61202,
|
|
"step": 700,
|
|
"tokens/total": 4595840,
|
|
"tokens/train_per_sec_per_gpu": 115.62,
|
|
"tokens/trainable": 2954964
|
|
},
|
|
{
|
|
"epoch": 0.22147703345550962,
|
|
"grad_norm": 0.17456401884555817,
|
|
"learning_rate": 4.915249765164652e-05,
|
|
"loss": 1.8271587371826172,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.2162,
|
|
"step": 710,
|
|
"tokens/total": 4662144,
|
|
"tokens/train_per_sec_per_gpu": 45.71,
|
|
"tokens/trainable": 2998778
|
|
},
|
|
{
|
|
"epoch": 0.22459642829291118,
|
|
"grad_norm": 0.19512979686260223,
|
|
"learning_rate": 4.9119590835293256e-05,
|
|
"loss": 1.8284429550170898,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.22419,
|
|
"step": 720,
|
|
"tokens/total": 4726016,
|
|
"tokens/train_per_sec_per_gpu": 71.34,
|
|
"tokens/trainable": 3038853
|
|
},
|
|
{
|
|
"epoch": 0.22771582313031272,
|
|
"grad_norm": 0.20984387397766113,
|
|
"learning_rate": 4.908606871741862e-05,
|
|
"loss": 1.8802831649780274,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.55536,
|
|
"step": 730,
|
|
"tokens/total": 4791296,
|
|
"tokens/train_per_sec_per_gpu": 35.98,
|
|
"tokens/trainable": 3081502
|
|
},
|
|
{
|
|
"epoch": 0.23083521796771425,
|
|
"grad_norm": 0.1907588541507721,
|
|
"learning_rate": 4.905193215318679e-05,
|
|
"loss": 1.884172248840332,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.5809,
|
|
"step": 740,
|
|
"tokens/total": 4856704,
|
|
"tokens/train_per_sec_per_gpu": 78.53,
|
|
"tokens/trainable": 3122850
|
|
},
|
|
{
|
|
"epoch": 0.2339546128051158,
|
|
"grad_norm": 0.1766246259212494,
|
|
"learning_rate": 4.901718201343675e-05,
|
|
"loss": 1.832427978515625,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.24904,
|
|
"step": 750,
|
|
"tokens/total": 4921600,
|
|
"tokens/train_per_sec_per_gpu": 71.52,
|
|
"tokens/trainable": 3164173
|
|
},
|
|
{
|
|
"epoch": 0.23707400764251735,
|
|
"grad_norm": 0.19759753346443176,
|
|
"learning_rate": 4.898181918466006e-05,
|
|
"loss": 1.8128562927246095,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.12793,
|
|
"step": 760,
|
|
"tokens/total": 4988416,
|
|
"tokens/train_per_sec_per_gpu": 55.64,
|
|
"tokens/trainable": 3207973
|
|
},
|
|
{
|
|
"epoch": 0.24019340247991888,
|
|
"grad_norm": 0.15614399313926697,
|
|
"learning_rate": 4.894584456897827e-05,
|
|
"loss": 1.8343976974487304,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.26136,
|
|
"step": 770,
|
|
"tokens/total": 5055104,
|
|
"tokens/train_per_sec_per_gpu": 86.18,
|
|
"tokens/trainable": 3251009
|
|
},
|
|
{
|
|
"epoch": 0.24331279731732045,
|
|
"grad_norm": 0.18144932389259338,
|
|
"learning_rate": 4.890925908411986e-05,
|
|
"loss": 1.8365650177001953,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.27495,
|
|
"step": 780,
|
|
"tokens/total": 5120384,
|
|
"tokens/train_per_sec_per_gpu": 51.23,
|
|
"tokens/trainable": 3294517
|
|
},
|
|
{
|
|
"epoch": 0.24643219215472198,
|
|
"grad_norm": 0.19318589568138123,
|
|
"learning_rate": 4.887206366339688e-05,
|
|
"loss": 1.8659175872802733,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.46186,
|
|
"step": 790,
|
|
"tokens/total": 5185920,
|
|
"tokens/train_per_sec_per_gpu": 81.88,
|
|
"tokens/trainable": 3336348
|
|
},
|
|
{
|
|
"epoch": 0.24955158699212354,
|
|
"grad_norm": 0.2216137796640396,
|
|
"learning_rate": 4.883425925568109e-05,
|
|
"loss": 1.8160842895507812,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.14774,
|
|
"step": 800,
|
|
"tokens/total": 5248896,
|
|
"tokens/train_per_sec_per_gpu": 67.02,
|
|
"tokens/trainable": 3376802
|
|
},
|
|
{
|
|
"epoch": 0.2526709818295251,
|
|
"grad_norm": 0.1774853616952896,
|
|
"learning_rate": 4.879584682537981e-05,
|
|
"loss": 1.8577735900878907,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.40945,
|
|
"step": 810,
|
|
"tokens/total": 5313536,
|
|
"tokens/train_per_sec_per_gpu": 93.95,
|
|
"tokens/trainable": 3418699
|
|
},
|
|
{
|
|
"epoch": 0.2557903766669266,
|
|
"grad_norm": 0.21706761419773102,
|
|
"learning_rate": 4.8756827352411264e-05,
|
|
"loss": 1.7832977294921875,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.94944,
|
|
"step": 820,
|
|
"tokens/total": 5380992,
|
|
"tokens/train_per_sec_per_gpu": 77.86,
|
|
"tokens/trainable": 3461481
|
|
},
|
|
{
|
|
"epoch": 0.25890977150432815,
|
|
"grad_norm": 0.18638113141059875,
|
|
"learning_rate": 4.871720183217961e-05,
|
|
"loss": 1.7760698318481445,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.9066,
|
|
"step": 830,
|
|
"tokens/total": 5445504,
|
|
"tokens/train_per_sec_per_gpu": 89.08,
|
|
"tokens/trainable": 3504032
|
|
},
|
|
{
|
|
"epoch": 0.2620291663417297,
|
|
"grad_norm": 0.17489278316497803,
|
|
"learning_rate": 4.867697127554956e-05,
|
|
"loss": 1.794443702697754,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.01613,
|
|
"step": 840,
|
|
"tokens/total": 5509632,
|
|
"tokens/train_per_sec_per_gpu": 50.99,
|
|
"tokens/trainable": 3545740
|
|
},
|
|
{
|
|
"epoch": 0.26514856117913127,
|
|
"grad_norm": 0.1717980057001114,
|
|
"learning_rate": 4.863613670882053e-05,
|
|
"loss": 1.809376335144043,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.10664,
|
|
"step": 850,
|
|
"tokens/total": 5575424,
|
|
"tokens/train_per_sec_per_gpu": 54.47,
|
|
"tokens/trainable": 3588900
|
|
},
|
|
{
|
|
"epoch": 0.2682679560165328,
|
|
"grad_norm": 0.2062896341085434,
|
|
"learning_rate": 4.859469917370057e-05,
|
|
"loss": 1.8082466125488281,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.09974,
|
|
"step": 860,
|
|
"tokens/total": 5640192,
|
|
"tokens/train_per_sec_per_gpu": 86.91,
|
|
"tokens/trainable": 3630694
|
|
},
|
|
{
|
|
"epoch": 0.27138735085393434,
|
|
"grad_norm": 0.20132805407047272,
|
|
"learning_rate": 4.8552659727279654e-05,
|
|
"loss": 1.7593723297119142,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.80879,
|
|
"step": 870,
|
|
"tokens/total": 5705344,
|
|
"tokens/train_per_sec_per_gpu": 69.38,
|
|
"tokens/trainable": 3671586
|
|
},
|
|
{
|
|
"epoch": 0.2745067456913359,
|
|
"grad_norm": 0.1575271636247635,
|
|
"learning_rate": 4.851001944200283e-05,
|
|
"loss": 1.8791692733764649,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.54806,
|
|
"step": 880,
|
|
"tokens/total": 5773440,
|
|
"tokens/train_per_sec_per_gpu": 94.66,
|
|
"tokens/trainable": 3715195
|
|
},
|
|
{
|
|
"epoch": 0.2776261405287374,
|
|
"grad_norm": 0.18341796100139618,
|
|
"learning_rate": 4.8466779405642795e-05,
|
|
"loss": 1.7878997802734375,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.97689,
|
|
"step": 890,
|
|
"tokens/total": 5841664,
|
|
"tokens/train_per_sec_per_gpu": 86.18,
|
|
"tokens/trainable": 3759560
|
|
},
|
|
{
|
|
"epoch": 0.28074553536613894,
|
|
"grad_norm": 0.17901389300823212,
|
|
"learning_rate": 4.8422940721272146e-05,
|
|
"loss": 1.7819393157958985,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.94137,
|
|
"step": 900,
|
|
"tokens/total": 5909760,
|
|
"tokens/train_per_sec_per_gpu": 44.99,
|
|
"tokens/trainable": 3803514
|
|
},
|
|
{
|
|
"epoch": 0.28386493020354053,
|
|
"grad_norm": 0.17867782711982727,
|
|
"learning_rate": 4.837850450723531e-05,
|
|
"loss": 1.8327096939086913,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.2508,
|
|
"step": 910,
|
|
"tokens/total": 5976576,
|
|
"tokens/train_per_sec_per_gpu": 106.29,
|
|
"tokens/trainable": 3847107
|
|
},
|
|
{
|
|
"epoch": 0.28698432504094207,
|
|
"grad_norm": 0.1634192168712616,
|
|
"learning_rate": 4.83334718971199e-05,
|
|
"loss": 1.8085763931274415,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.10175,
|
|
"step": 920,
|
|
"tokens/total": 6042112,
|
|
"tokens/train_per_sec_per_gpu": 64.12,
|
|
"tokens/trainable": 3890709
|
|
},
|
|
{
|
|
"epoch": 0.2901037198783436,
|
|
"grad_norm": 0.19030778110027313,
|
|
"learning_rate": 4.82878440397279e-05,
|
|
"loss": 1.8400047302246094,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.29657,
|
|
"step": 930,
|
|
"tokens/total": 6107136,
|
|
"tokens/train_per_sec_per_gpu": 71.59,
|
|
"tokens/trainable": 3931488
|
|
},
|
|
{
|
|
"epoch": 0.29322311471574514,
|
|
"grad_norm": 0.21266056597232819,
|
|
"learning_rate": 4.8241622099046316e-05,
|
|
"loss": 1.7370513916015624,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.68057,
|
|
"step": 940,
|
|
"tokens/total": 6171392,
|
|
"tokens/train_per_sec_per_gpu": 84.96,
|
|
"tokens/trainable": 3972117
|
|
},
|
|
{
|
|
"epoch": 0.29634250955314667,
|
|
"grad_norm": 0.21191294491291046,
|
|
"learning_rate": 4.8194807254217444e-05,
|
|
"loss": 1.8417661666870118,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.30767,
|
|
"step": 950,
|
|
"tokens/total": 6237056,
|
|
"tokens/train_per_sec_per_gpu": 33.76,
|
|
"tokens/trainable": 4014575
|
|
},
|
|
{
|
|
"epoch": 0.29946190439054826,
|
|
"grad_norm": 0.1874547004699707,
|
|
"learning_rate": 4.814740069950889e-05,
|
|
"loss": 1.8018758773803711,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.06101,
|
|
"step": 960,
|
|
"tokens/total": 6300416,
|
|
"tokens/train_per_sec_per_gpu": 42.36,
|
|
"tokens/trainable": 4055044
|
|
},
|
|
{
|
|
"epoch": 0.3025812992279498,
|
|
"grad_norm": 0.17511016130447388,
|
|
"learning_rate": 4.809940364428299e-05,
|
|
"loss": 1.7425220489501954,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.71173,
|
|
"step": 970,
|
|
"tokens/total": 6365696,
|
|
"tokens/train_per_sec_per_gpu": 91.61,
|
|
"tokens/trainable": 4096123
|
|
},
|
|
{
|
|
"epoch": 0.30570069406535133,
|
|
"grad_norm": 0.17287850379943848,
|
|
"learning_rate": 4.805081731296605e-05,
|
|
"loss": 1.7090280532836915,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.52359,
|
|
"step": 980,
|
|
"tokens/total": 6431872,
|
|
"tokens/train_per_sec_per_gpu": 63.03,
|
|
"tokens/trainable": 4140195
|
|
},
|
|
{
|
|
"epoch": 0.30882008890275287,
|
|
"grad_norm": 0.17550387978553772,
|
|
"learning_rate": 4.800164294501704e-05,
|
|
"loss": 1.8226003646850586,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.18793,
|
|
"step": 990,
|
|
"tokens/total": 6497920,
|
|
"tokens/train_per_sec_per_gpu": 93.0,
|
|
"tokens/trainable": 4181785
|
|
},
|
|
{
|
|
"epoch": 0.3119394837401544,
|
|
"grad_norm": 0.16732849180698395,
|
|
"learning_rate": 4.795188179489603e-05,
|
|
"loss": 1.7610651016235352,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.81863,
|
|
"step": 1000,
|
|
"tokens/total": 6562176,
|
|
"tokens/train_per_sec_per_gpu": 90.18,
|
|
"tokens/trainable": 4223241
|
|
},
|
|
{
|
|
"epoch": 0.31505887857755593,
|
|
"grad_norm": 0.17500464618206024,
|
|
"learning_rate": 4.7901535132032146e-05,
|
|
"loss": 1.8542400360107423,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.38684,
|
|
"step": 1010,
|
|
"tokens/total": 6627968,
|
|
"tokens/train_per_sec_per_gpu": 84.77,
|
|
"tokens/trainable": 4265553
|
|
},
|
|
{
|
|
"epoch": 0.3181782734149575,
|
|
"grad_norm": 0.1539732813835144,
|
|
"learning_rate": 4.785060424079124e-05,
|
|
"loss": 1.731656265258789,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.65,
|
|
"step": 1020,
|
|
"tokens/total": 6692096,
|
|
"tokens/train_per_sec_per_gpu": 67.47,
|
|
"tokens/trainable": 4308469
|
|
},
|
|
{
|
|
"epoch": 0.32129766825235906,
|
|
"grad_norm": 0.16461169719696045,
|
|
"learning_rate": 4.779909042044304e-05,
|
|
"loss": 1.7810606002807616,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.93615,
|
|
"step": 1030,
|
|
"tokens/total": 6757376,
|
|
"tokens/train_per_sec_per_gpu": 51.58,
|
|
"tokens/trainable": 4350675
|
|
},
|
|
{
|
|
"epoch": 0.3244170630897606,
|
|
"grad_norm": 0.18127188086509705,
|
|
"learning_rate": 4.77469949851281e-05,
|
|
"loss": 1.7395605087280273,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.69484,
|
|
"step": 1040,
|
|
"tokens/total": 6824576,
|
|
"tokens/train_per_sec_per_gpu": 103.46,
|
|
"tokens/trainable": 4393277
|
|
},
|
|
{
|
|
"epoch": 0.32753645792716213,
|
|
"grad_norm": 0.1589565873146057,
|
|
"learning_rate": 4.769431926382421e-05,
|
|
"loss": 1.7943479537963867,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.01555,
|
|
"step": 1050,
|
|
"tokens/total": 6893184,
|
|
"tokens/train_per_sec_per_gpu": 43.69,
|
|
"tokens/trainable": 4438122
|
|
},
|
|
{
|
|
"epoch": 0.33065585276456366,
|
|
"grad_norm": 0.17581714689731598,
|
|
"learning_rate": 4.76410646003125e-05,
|
|
"loss": 1.8064342498779298,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.0887,
|
|
"step": 1060,
|
|
"tokens/total": 6959104,
|
|
"tokens/train_per_sec_per_gpu": 65.76,
|
|
"tokens/trainable": 4480622
|
|
},
|
|
{
|
|
"epoch": 0.3337752476019652,
|
|
"grad_norm": 0.19487641751766205,
|
|
"learning_rate": 4.758723235314322e-05,
|
|
"loss": 1.826577377319336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.21259,
|
|
"step": 1070,
|
|
"tokens/total": 7024512,
|
|
"tokens/train_per_sec_per_gpu": 61.04,
|
|
"tokens/trainable": 4523612
|
|
},
|
|
{
|
|
"epoch": 0.3368946424393668,
|
|
"grad_norm": 0.1828070729970932,
|
|
"learning_rate": 4.753282389560099e-05,
|
|
"loss": 1.7523822784423828,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.76833,
|
|
"step": 1080,
|
|
"tokens/total": 7088256,
|
|
"tokens/train_per_sec_per_gpu": 88.56,
|
|
"tokens/trainable": 4564035
|
|
},
|
|
{
|
|
"epoch": 0.3400140372767683,
|
|
"grad_norm": 0.16003485023975372,
|
|
"learning_rate": 4.7477840615669846e-05,
|
|
"loss": 1.7929000854492188,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.00685,
|
|
"step": 1090,
|
|
"tokens/total": 7152768,
|
|
"tokens/train_per_sec_per_gpu": 88.49,
|
|
"tokens/trainable": 4605557
|
|
},
|
|
{
|
|
"epoch": 0.34313343211416986,
|
|
"grad_norm": 0.17932865023612976,
|
|
"learning_rate": 4.742228391599777e-05,
|
|
"loss": 1.834946060180664,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.2648,
|
|
"step": 1100,
|
|
"tokens/total": 7216896,
|
|
"tokens/train_per_sec_per_gpu": 92.6,
|
|
"tokens/trainable": 4644798
|
|
},
|
|
{
|
|
"epoch": 0.3462528269515714,
|
|
"grad_norm": 0.1757347732782364,
|
|
"learning_rate": 4.736615521386095e-05,
|
|
"loss": 1.8846118927001954,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.5838,
|
|
"step": 1110,
|
|
"tokens/total": 7281920,
|
|
"tokens/train_per_sec_per_gpu": 91.82,
|
|
"tokens/trainable": 4686380
|
|
},
|
|
{
|
|
"epoch": 0.3493722217889729,
|
|
"grad_norm": 0.1802482157945633,
|
|
"learning_rate": 4.7309455941127644e-05,
|
|
"loss": 1.8051477432250977,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.08087,
|
|
"step": 1120,
|
|
"tokens/total": 7347456,
|
|
"tokens/train_per_sec_per_gpu": 80.11,
|
|
"tokens/trainable": 4727786
|
|
},
|
|
{
|
|
"epoch": 0.35249161662637446,
|
|
"grad_norm": 0.15785469114780426,
|
|
"learning_rate": 4.725218754422156e-05,
|
|
"loss": 1.776127815246582,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.90694,
|
|
"step": 1130,
|
|
"tokens/total": 7412864,
|
|
"tokens/train_per_sec_per_gpu": 65.2,
|
|
"tokens/trainable": 4769730
|
|
},
|
|
{
|
|
"epoch": 0.35561101146377605,
|
|
"grad_norm": 0.15445466339588165,
|
|
"learning_rate": 4.7194351484085065e-05,
|
|
"loss": 1.7816373825073242,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.93957,
|
|
"step": 1140,
|
|
"tokens/total": 7479552,
|
|
"tokens/train_per_sec_per_gpu": 58.15,
|
|
"tokens/trainable": 4813563
|
|
},
|
|
{
|
|
"epoch": 0.3587304063011776,
|
|
"grad_norm": 0.15774467587471008,
|
|
"learning_rate": 4.713594923614185e-05,
|
|
"loss": 1.7695735931396483,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.86835,
|
|
"step": 1150,
|
|
"tokens/total": 7545984,
|
|
"tokens/train_per_sec_per_gpu": 74.64,
|
|
"tokens/trainable": 4857564
|
|
},
|
|
{
|
|
"epoch": 0.3618498011385791,
|
|
"grad_norm": 0.1751730591058731,
|
|
"learning_rate": 4.707698229025932e-05,
|
|
"loss": 1.7517837524414062,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.76488,
|
|
"step": 1160,
|
|
"tokens/total": 7609472,
|
|
"tokens/train_per_sec_per_gpu": 77.5,
|
|
"tokens/trainable": 4898810
|
|
},
|
|
{
|
|
"epoch": 0.36496919597598065,
|
|
"grad_norm": 0.1625043898820877,
|
|
"learning_rate": 4.701745215071055e-05,
|
|
"loss": 1.8207901000976563,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.17674,
|
|
"step": 1170,
|
|
"tokens/total": 7675264,
|
|
"tokens/train_per_sec_per_gpu": 77.68,
|
|
"tokens/trainable": 4940602
|
|
},
|
|
{
|
|
"epoch": 0.3680885908133822,
|
|
"grad_norm": 0.15293572843074799,
|
|
"learning_rate": 4.695736033613597e-05,
|
|
"loss": 1.786800003051758,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.97032,
|
|
"step": 1180,
|
|
"tokens/total": 7741568,
|
|
"tokens/train_per_sec_per_gpu": 76.17,
|
|
"tokens/trainable": 4983164
|
|
},
|
|
{
|
|
"epoch": 0.3712079856507837,
|
|
"grad_norm": 0.20069563388824463,
|
|
"learning_rate": 4.6896708379504563e-05,
|
|
"loss": 1.6845293045043945,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.38991,
|
|
"step": 1190,
|
|
"tokens/total": 7807872,
|
|
"tokens/train_per_sec_per_gpu": 71.51,
|
|
"tokens/trainable": 5024759
|
|
},
|
|
{
|
|
"epoch": 0.3743273804881853,
|
|
"grad_norm": 0.17199957370758057,
|
|
"learning_rate": 4.6835497828074806e-05,
|
|
"loss": 1.7984563827514648,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.04032,
|
|
"step": 1200,
|
|
"tokens/total": 7873920,
|
|
"tokens/train_per_sec_per_gpu": 74.38,
|
|
"tokens/trainable": 5065930
|
|
},
|
|
{
|
|
"epoch": 0.37744677532558685,
|
|
"grad_norm": 0.19014553725719452,
|
|
"learning_rate": 4.677373024335517e-05,
|
|
"loss": 1.787830924987793,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.97647,
|
|
"step": 1210,
|
|
"tokens/total": 7935232,
|
|
"tokens/train_per_sec_per_gpu": 50.89,
|
|
"tokens/trainable": 5105628
|
|
},
|
|
{
|
|
"epoch": 0.3805661701629884,
|
|
"grad_norm": 0.1500748097896576,
|
|
"learning_rate": 4.67114072010643e-05,
|
|
"loss": 1.792111587524414,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.00211,
|
|
"step": 1220,
|
|
"tokens/total": 8002176,
|
|
"tokens/train_per_sec_per_gpu": 49.71,
|
|
"tokens/trainable": 5149488
|
|
},
|
|
{
|
|
"epoch": 0.3836855650003899,
|
|
"grad_norm": 0.15705767273902893,
|
|
"learning_rate": 4.6648530291090795e-05,
|
|
"loss": 1.7737476348876953,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.8929,
|
|
"step": 1230,
|
|
"tokens/total": 8065280,
|
|
"tokens/train_per_sec_per_gpu": 58.65,
|
|
"tokens/trainable": 5190357
|
|
},
|
|
{
|
|
"epoch": 0.38680495983779145,
|
|
"grad_norm": 0.2005138248205185,
|
|
"learning_rate": 4.658510111745268e-05,
|
|
"loss": 1.723834228515625,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.60598,
|
|
"step": 1240,
|
|
"tokens/total": 8130432,
|
|
"tokens/train_per_sec_per_gpu": 55.45,
|
|
"tokens/trainable": 5233108
|
|
},
|
|
{
|
|
"epoch": 0.389924354675193,
|
|
"grad_norm": 0.17651793360710144,
|
|
"learning_rate": 4.6521121298256464e-05,
|
|
"loss": 1.7616628646850585,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.82211,
|
|
"step": 1250,
|
|
"tokens/total": 8196480,
|
|
"tokens/train_per_sec_per_gpu": 93.38,
|
|
"tokens/trainable": 5275335
|
|
},
|
|
{
|
|
"epoch": 0.3930437495125946,
|
|
"grad_norm": 0.15281912684440613,
|
|
"learning_rate": 4.645659246565589e-05,
|
|
"loss": 1.7928432464599608,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.00651,
|
|
"step": 1260,
|
|
"tokens/total": 8261120,
|
|
"tokens/train_per_sec_per_gpu": 57.75,
|
|
"tokens/trainable": 5318062
|
|
},
|
|
{
|
|
"epoch": 0.3961631443499961,
|
|
"grad_norm": 0.1695193350315094,
|
|
"learning_rate": 4.639151626581024e-05,
|
|
"loss": 1.8626853942871093,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.44101,
|
|
"step": 1270,
|
|
"tokens/total": 8326912,
|
|
"tokens/train_per_sec_per_gpu": 64.56,
|
|
"tokens/trainable": 5361032
|
|
},
|
|
{
|
|
"epoch": 0.39928253918739764,
|
|
"grad_norm": 0.1824774146080017,
|
|
"learning_rate": 4.6325894358842425e-05,
|
|
"loss": 1.716079330444336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.56268,
|
|
"step": 1280,
|
|
"tokens/total": 8392832,
|
|
"tokens/train_per_sec_per_gpu": 52.11,
|
|
"tokens/trainable": 5402939
|
|
},
|
|
{
|
|
"epoch": 0.4024019340247992,
|
|
"grad_norm": 0.18175636231899261,
|
|
"learning_rate": 4.6259728418796545e-05,
|
|
"loss": 1.8381244659423828,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.28474,
|
|
"step": 1290,
|
|
"tokens/total": 8456320,
|
|
"tokens/train_per_sec_per_gpu": 64.67,
|
|
"tokens/trainable": 5444319
|
|
},
|
|
{
|
|
"epoch": 0.4055213288622007,
|
|
"grad_norm": 0.1625058501958847,
|
|
"learning_rate": 4.6193020133595254e-05,
|
|
"loss": 1.8380538940429687,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.2843,
|
|
"step": 1300,
|
|
"tokens/total": 8521344,
|
|
"tokens/train_per_sec_per_gpu": 93.77,
|
|
"tokens/trainable": 5485752
|
|
},
|
|
{
|
|
"epoch": 0.4086407236996023,
|
|
"grad_norm": 0.16854329407215118,
|
|
"learning_rate": 4.6125771204996656e-05,
|
|
"loss": 1.7914077758789062,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.99789,
|
|
"step": 1310,
|
|
"tokens/total": 8588544,
|
|
"tokens/train_per_sec_per_gpu": 54.51,
|
|
"tokens/trainable": 5528271
|
|
},
|
|
{
|
|
"epoch": 0.41176011853700384,
|
|
"grad_norm": 0.16071408987045288,
|
|
"learning_rate": 4.605798334855094e-05,
|
|
"loss": 1.723423385620117,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.60368,
|
|
"step": 1320,
|
|
"tokens/total": 8654720,
|
|
"tokens/train_per_sec_per_gpu": 66.24,
|
|
"tokens/trainable": 5570985
|
|
},
|
|
{
|
|
"epoch": 0.4148795133744054,
|
|
"grad_norm": 0.17922136187553406,
|
|
"learning_rate": 4.598965829355654e-05,
|
|
"loss": 1.7501956939697265,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.75573,
|
|
"step": 1330,
|
|
"tokens/total": 8719744,
|
|
"tokens/train_per_sec_per_gpu": 81.7,
|
|
"tokens/trainable": 5611348
|
|
},
|
|
{
|
|
"epoch": 0.4179989082118069,
|
|
"grad_norm": 0.17143522202968597,
|
|
"learning_rate": 4.592079778301609e-05,
|
|
"loss": 1.7220258712768555,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.59585,
|
|
"step": 1340,
|
|
"tokens/total": 8785152,
|
|
"tokens/train_per_sec_per_gpu": 63.36,
|
|
"tokens/trainable": 5652978
|
|
},
|
|
{
|
|
"epoch": 0.42111830304920844,
|
|
"grad_norm": 0.1701611876487732,
|
|
"learning_rate": 4.5851403573591936e-05,
|
|
"loss": 1.6611724853515626,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.26548,
|
|
"step": 1350,
|
|
"tokens/total": 8850176,
|
|
"tokens/train_per_sec_per_gpu": 62.96,
|
|
"tokens/trainable": 5694521
|
|
},
|
|
{
|
|
"epoch": 0.42423769788661,
|
|
"grad_norm": 0.2894500494003296,
|
|
"learning_rate": 4.578147743556132e-05,
|
|
"loss": 1.74639835357666,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.73391,
|
|
"step": 1360,
|
|
"tokens/total": 8916608,
|
|
"tokens/train_per_sec_per_gpu": 94.5,
|
|
"tokens/trainable": 5737173
|
|
},
|
|
{
|
|
"epoch": 0.42735709272401157,
|
|
"grad_norm": 0.16340133547782898,
|
|
"learning_rate": 4.57110211527712e-05,
|
|
"loss": 1.7146156311035157,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.55454,
|
|
"step": 1370,
|
|
"tokens/total": 8981120,
|
|
"tokens/train_per_sec_per_gpu": 78.54,
|
|
"tokens/trainable": 5778366
|
|
},
|
|
{
|
|
"epoch": 0.4304764875614131,
|
|
"grad_norm": 0.16280069947242737,
|
|
"learning_rate": 4.56400365225928e-05,
|
|
"loss": 1.7889955520629883,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.98344,
|
|
"step": 1380,
|
|
"tokens/total": 9047296,
|
|
"tokens/train_per_sec_per_gpu": 79.68,
|
|
"tokens/trainable": 5819227
|
|
},
|
|
{
|
|
"epoch": 0.43359588239881464,
|
|
"grad_norm": 0.17111781239509583,
|
|
"learning_rate": 4.556852535587569e-05,
|
|
"loss": 1.720347023010254,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.58647,
|
|
"step": 1390,
|
|
"tokens/total": 9114880,
|
|
"tokens/train_per_sec_per_gpu": 71.67,
|
|
"tokens/trainable": 5862101
|
|
},
|
|
{
|
|
"epoch": 0.43671527723621617,
|
|
"grad_norm": 0.162201389670372,
|
|
"learning_rate": 4.549648947690163e-05,
|
|
"loss": 1.7788646697998047,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.92313,
|
|
"step": 1400,
|
|
"tokens/total": 9177472,
|
|
"tokens/train_per_sec_per_gpu": 70.12,
|
|
"tokens/trainable": 5902156
|
|
},
|
|
{
|
|
"epoch": 0.4398346720736177,
|
|
"grad_norm": 0.1679544299840927,
|
|
"learning_rate": 4.542393072333803e-05,
|
|
"loss": 1.6937917709350585,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.44007,
|
|
"step": 1410,
|
|
"tokens/total": 9241984,
|
|
"tokens/train_per_sec_per_gpu": 75.64,
|
|
"tokens/trainable": 5942964
|
|
},
|
|
{
|
|
"epoch": 0.44295406691101924,
|
|
"grad_norm": 0.18267333507537842,
|
|
"learning_rate": 4.5350850946191045e-05,
|
|
"loss": 1.734885025024414,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.66828,
|
|
"step": 1420,
|
|
"tokens/total": 9306624,
|
|
"tokens/train_per_sec_per_gpu": 64.75,
|
|
"tokens/trainable": 5985190
|
|
},
|
|
{
|
|
"epoch": 0.44607346174842083,
|
|
"grad_norm": 0.14984647929668427,
|
|
"learning_rate": 4.5277252009758405e-05,
|
|
"loss": 1.7400466918945312,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.69761,
|
|
"step": 1430,
|
|
"tokens/total": 9372160,
|
|
"tokens/train_per_sec_per_gpu": 61.45,
|
|
"tokens/trainable": 6026845
|
|
},
|
|
{
|
|
"epoch": 0.44919285658582236,
|
|
"grad_norm": 0.1772312968969345,
|
|
"learning_rate": 4.520313579158182e-05,
|
|
"loss": 1.6693408966064454,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.30867,
|
|
"step": 1440,
|
|
"tokens/total": 9436928,
|
|
"tokens/train_per_sec_per_gpu": 76.28,
|
|
"tokens/trainable": 6068036
|
|
},
|
|
{
|
|
"epoch": 0.4523122514232239,
|
|
"grad_norm": 0.18042142689228058,
|
|
"learning_rate": 4.512850418239905e-05,
|
|
"loss": 1.7526473999023438,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.76986,
|
|
"step": 1450,
|
|
"tokens/total": 9504256,
|
|
"tokens/train_per_sec_per_gpu": 74.25,
|
|
"tokens/trainable": 6111880
|
|
},
|
|
{
|
|
"epoch": 0.45543164626062543,
|
|
"grad_norm": 0.17128664255142212,
|
|
"learning_rate": 4.505335908609576e-05,
|
|
"loss": 1.7291893005371093,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.63608,
|
|
"step": 1460,
|
|
"tokens/total": 9570432,
|
|
"tokens/train_per_sec_per_gpu": 47.99,
|
|
"tokens/trainable": 6154942
|
|
},
|
|
{
|
|
"epoch": 0.45855104109802697,
|
|
"grad_norm": 0.1666496992111206,
|
|
"learning_rate": 4.4977702419656856e-05,
|
|
"loss": 1.6212188720703125,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.05925,
|
|
"step": 1470,
|
|
"tokens/total": 9636480,
|
|
"tokens/train_per_sec_per_gpu": 47.73,
|
|
"tokens/trainable": 6197024
|
|
},
|
|
{
|
|
"epoch": 0.4616704359354285,
|
|
"grad_norm": 0.1635151356458664,
|
|
"learning_rate": 4.490153611311767e-05,
|
|
"loss": 1.7073596954345702,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.51438,
|
|
"step": 1480,
|
|
"tokens/total": 9701120,
|
|
"tokens/train_per_sec_per_gpu": 48.06,
|
|
"tokens/trainable": 6238097
|
|
},
|
|
{
|
|
"epoch": 0.4647898307728301,
|
|
"grad_norm": 0.15569257736206055,
|
|
"learning_rate": 4.482486210951463e-05,
|
|
"loss": 1.7654897689819335,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.84443,
|
|
"step": 1490,
|
|
"tokens/total": 9765376,
|
|
"tokens/train_per_sec_per_gpu": 109.43,
|
|
"tokens/trainable": 6279308
|
|
},
|
|
{
|
|
"epoch": 0.4679092256102316,
|
|
"grad_norm": 0.16915340721607208,
|
|
"learning_rate": 4.4747682364835796e-05,
|
|
"loss": 1.7672908782958985,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.85497,
|
|
"step": 1500,
|
|
"tokens/total": 9830528,
|
|
"tokens/train_per_sec_per_gpu": 67.8,
|
|
"tokens/trainable": 6320332
|
|
},
|
|
{
|
|
"epoch": 0.47102862044763316,
|
|
"grad_norm": 0.15137460827827454,
|
|
"learning_rate": 4.4669998847970876e-05,
|
|
"loss": 1.7700065612792968,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.87089,
|
|
"step": 1510,
|
|
"tokens/total": 9894144,
|
|
"tokens/train_per_sec_per_gpu": 51.85,
|
|
"tokens/trainable": 6362957
|
|
},
|
|
{
|
|
"epoch": 0.4741480152850347,
|
|
"grad_norm": 0.16958214342594147,
|
|
"learning_rate": 4.459181354066103e-05,
|
|
"loss": 1.6836748123168945,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.38531,
|
|
"step": 1520,
|
|
"tokens/total": 9960448,
|
|
"tokens/train_per_sec_per_gpu": 45.91,
|
|
"tokens/trainable": 6405668
|
|
},
|
|
{
|
|
"epoch": 0.47726741012243623,
|
|
"grad_norm": 0.15751244127750397,
|
|
"learning_rate": 4.451312843744834e-05,
|
|
"loss": 1.7873207092285157,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.97343,
|
|
"step": 1530,
|
|
"tokens/total": 10025728,
|
|
"tokens/train_per_sec_per_gpu": 67.17,
|
|
"tokens/trainable": 6448456
|
|
},
|
|
{
|
|
"epoch": 0.48038680495983777,
|
|
"grad_norm": 0.16995222866535187,
|
|
"learning_rate": 4.443394554562486e-05,
|
|
"loss": 1.6978713989257812,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.46231,
|
|
"step": 1540,
|
|
"tokens/total": 10089856,
|
|
"tokens/train_per_sec_per_gpu": 91.19,
|
|
"tokens/trainable": 6489370
|
|
},
|
|
{
|
|
"epoch": 0.48350619979723936,
|
|
"grad_norm": 0.1579769253730774,
|
|
"learning_rate": 4.435426688518152e-05,
|
|
"loss": 1.7317926406860351,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.65077,
|
|
"step": 1550,
|
|
"tokens/total": 10155136,
|
|
"tokens/train_per_sec_per_gpu": 69.35,
|
|
"tokens/trainable": 6531298
|
|
},
|
|
{
|
|
"epoch": 0.4866255946346409,
|
|
"grad_norm": 0.17956697940826416,
|
|
"learning_rate": 4.4274094488756456e-05,
|
|
"loss": 1.728812026977539,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.63396,
|
|
"step": 1560,
|
|
"tokens/total": 10222080,
|
|
"tokens/train_per_sec_per_gpu": 70.73,
|
|
"tokens/trainable": 6574329
|
|
},
|
|
{
|
|
"epoch": 0.4897449894720424,
|
|
"grad_norm": 0.15349198877811432,
|
|
"learning_rate": 4.419343040158328e-05,
|
|
"loss": 1.646613883972168,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.18938,
|
|
"step": 1570,
|
|
"tokens/total": 10289408,
|
|
"tokens/train_per_sec_per_gpu": 68.22,
|
|
"tokens/trainable": 6618959
|
|
},
|
|
{
|
|
"epoch": 0.49286438430944396,
|
|
"grad_norm": 0.1683163195848465,
|
|
"learning_rate": 4.411227668143884e-05,
|
|
"loss": 1.748611068725586,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.74662,
|
|
"step": 1580,
|
|
"tokens/total": 10351360,
|
|
"tokens/train_per_sec_per_gpu": 49.88,
|
|
"tokens/trainable": 6658417
|
|
},
|
|
{
|
|
"epoch": 0.4959837791468455,
|
|
"grad_norm": 0.15357564389705658,
|
|
"learning_rate": 4.403063539859073e-05,
|
|
"loss": 1.7452756881713867,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.72748,
|
|
"step": 1590,
|
|
"tokens/total": 10416128,
|
|
"tokens/train_per_sec_per_gpu": 113.78,
|
|
"tokens/trainable": 6700005
|
|
},
|
|
{
|
|
"epoch": 0.4991031739842471,
|
|
"grad_norm": 0.18974904716014862,
|
|
"learning_rate": 4.39485086357445e-05,
|
|
"loss": 1.7341825485229492,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.6643,
|
|
"step": 1600,
|
|
"tokens/total": 10482432,
|
|
"tokens/train_per_sec_per_gpu": 37.54,
|
|
"tokens/trainable": 6741832
|
|
},
|
|
{
|
|
"epoch": 0.5022225688216486,
|
|
"grad_norm": 0.16073961555957794,
|
|
"learning_rate": 4.3865898487990505e-05,
|
|
"loss": 1.6739139556884766,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.333,
|
|
"step": 1610,
|
|
"tokens/total": 10548736,
|
|
"tokens/train_per_sec_per_gpu": 58.56,
|
|
"tokens/trainable": 6785631
|
|
},
|
|
{
|
|
"epoch": 0.5053419636590502,
|
|
"grad_norm": 0.15501181781291962,
|
|
"learning_rate": 4.378280706275046e-05,
|
|
"loss": 1.7250839233398438,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.61299,
|
|
"step": 1620,
|
|
"tokens/total": 10612096,
|
|
"tokens/train_per_sec_per_gpu": 32.24,
|
|
"tokens/trainable": 6826335
|
|
},
|
|
{
|
|
"epoch": 0.5084613584964517,
|
|
"grad_norm": 0.1500595659017563,
|
|
"learning_rate": 4.3699236479723684e-05,
|
|
"loss": 1.7123062133789062,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.54173,
|
|
"step": 1630,
|
|
"tokens/total": 10678016,
|
|
"tokens/train_per_sec_per_gpu": 87.24,
|
|
"tokens/trainable": 6868657
|
|
},
|
|
{
|
|
"epoch": 0.5115807533338532,
|
|
"grad_norm": 0.17212671041488647,
|
|
"learning_rate": 4.361518887083304e-05,
|
|
"loss": 1.6867807388305665,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.40206,
|
|
"step": 1640,
|
|
"tokens/total": 10743040,
|
|
"tokens/train_per_sec_per_gpu": 39.41,
|
|
"tokens/trainable": 6909910
|
|
},
|
|
{
|
|
"epoch": 0.5147001481712548,
|
|
"grad_norm": 0.16413834691047668,
|
|
"learning_rate": 4.353066638017052e-05,
|
|
"loss": 1.6804365158081054,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.3679,
|
|
"step": 1650,
|
|
"tokens/total": 10809600,
|
|
"tokens/train_per_sec_per_gpu": 108.18,
|
|
"tokens/trainable": 6952846
|
|
},
|
|
{
|
|
"epoch": 0.5178195430086563,
|
|
"grad_norm": 0.15592701733112335,
|
|
"learning_rate": 4.344567116394256e-05,
|
|
"loss": 1.7383615493774414,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.68802,
|
|
"step": 1660,
|
|
"tokens/total": 10876160,
|
|
"tokens/train_per_sec_per_gpu": 56.11,
|
|
"tokens/trainable": 6994359
|
|
},
|
|
{
|
|
"epoch": 0.5209389378460578,
|
|
"grad_norm": 0.16743551194667816,
|
|
"learning_rate": 4.3360205390415035e-05,
|
|
"loss": 1.8441978454589845,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.32303,
|
|
"step": 1670,
|
|
"tokens/total": 10942464,
|
|
"tokens/train_per_sec_per_gpu": 97.78,
|
|
"tokens/trainable": 7036292
|
|
},
|
|
{
|
|
"epoch": 0.5240583326834594,
|
|
"grad_norm": 0.16545693576335907,
|
|
"learning_rate": 4.3274271239857966e-05,
|
|
"loss": 1.770683479309082,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.87487,
|
|
"step": 1680,
|
|
"tokens/total": 11009920,
|
|
"tokens/train_per_sec_per_gpu": 87.1,
|
|
"tokens/trainable": 7079149
|
|
},
|
|
{
|
|
"epoch": 0.527177727520861,
|
|
"grad_norm": 0.14743508398532867,
|
|
"learning_rate": 4.318787090448986e-05,
|
|
"loss": 1.7586397171020507,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.80454,
|
|
"step": 1690,
|
|
"tokens/total": 11073024,
|
|
"tokens/train_per_sec_per_gpu": 70.03,
|
|
"tokens/trainable": 7119591
|
|
},
|
|
{
|
|
"epoch": 0.5302971223582625,
|
|
"grad_norm": 0.1614120751619339,
|
|
"learning_rate": 4.31010065884218e-05,
|
|
"loss": 1.7202964782714845,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.58618,
|
|
"step": 1700,
|
|
"tokens/total": 11140224,
|
|
"tokens/train_per_sec_per_gpu": 89.56,
|
|
"tokens/trainable": 7162987
|
|
},
|
|
{
|
|
"epoch": 0.5334165171956641,
|
|
"grad_norm": 0.14227625727653503,
|
|
"learning_rate": 4.301368050760126e-05,
|
|
"loss": 1.7092233657836915,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.52467,
|
|
"step": 1710,
|
|
"tokens/total": 11205504,
|
|
"tokens/train_per_sec_per_gpu": 61.45,
|
|
"tokens/trainable": 7205568
|
|
},
|
|
{
|
|
"epoch": 0.5365359120330656,
|
|
"grad_norm": 0.18265847861766815,
|
|
"learning_rate": 4.2925894889755506e-05,
|
|
"loss": 1.8467853546142579,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.33941,
|
|
"step": 1720,
|
|
"tokens/total": 11272192,
|
|
"tokens/train_per_sec_per_gpu": 36.94,
|
|
"tokens/trainable": 7247566
|
|
},
|
|
{
|
|
"epoch": 0.5396553068704671,
|
|
"grad_norm": 0.1411670595407486,
|
|
"learning_rate": 4.28376519743348e-05,
|
|
"loss": 1.7059291839599608,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.5065,
|
|
"step": 1730,
|
|
"tokens/total": 11336704,
|
|
"tokens/train_per_sec_per_gpu": 102.0,
|
|
"tokens/trainable": 7289603
|
|
},
|
|
{
|
|
"epoch": 0.5427747017078687,
|
|
"grad_norm": 0.1627739816904068,
|
|
"learning_rate": 4.27489540124553e-05,
|
|
"loss": 1.6554450988769531,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.23541,
|
|
"step": 1740,
|
|
"tokens/total": 11405312,
|
|
"tokens/train_per_sec_per_gpu": 48.54,
|
|
"tokens/trainable": 7333598
|
|
},
|
|
{
|
|
"epoch": 0.5458940965452702,
|
|
"grad_norm": 0.14822152256965637,
|
|
"learning_rate": 4.265980326684156e-05,
|
|
"loss": 1.6983177185058593,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.46475,
|
|
"step": 1750,
|
|
"tokens/total": 11472000,
|
|
"tokens/train_per_sec_per_gpu": 79.42,
|
|
"tokens/trainable": 7375438
|
|
},
|
|
{
|
|
"epoch": 0.5490134913826717,
|
|
"grad_norm": 0.17363330721855164,
|
|
"learning_rate": 4.257020201176886e-05,
|
|
"loss": 1.6864757537841797,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.40041,
|
|
"step": 1760,
|
|
"tokens/total": 11538560,
|
|
"tokens/train_per_sec_per_gpu": 71.4,
|
|
"tokens/trainable": 7418000
|
|
},
|
|
{
|
|
"epoch": 0.5521328862200733,
|
|
"grad_norm": 0.17044202983379364,
|
|
"learning_rate": 4.2480152533005215e-05,
|
|
"loss": 1.6587753295898438,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.25287,
|
|
"step": 1770,
|
|
"tokens/total": 11603328,
|
|
"tokens/train_per_sec_per_gpu": 78.83,
|
|
"tokens/trainable": 7460516
|
|
},
|
|
{
|
|
"epoch": 0.5552522810574748,
|
|
"grad_norm": 0.16858325898647308,
|
|
"learning_rate": 4.2389657127752966e-05,
|
|
"loss": 1.6045354843139648,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.97555,
|
|
"step": 1780,
|
|
"tokens/total": 11667968,
|
|
"tokens/train_per_sec_per_gpu": 34.0,
|
|
"tokens/trainable": 7501083
|
|
},
|
|
{
|
|
"epoch": 0.5583716758948764,
|
|
"grad_norm": 0.1591467261314392,
|
|
"learning_rate": 4.22987181045903e-05,
|
|
"loss": 1.7684541702270509,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.86179,
|
|
"step": 1790,
|
|
"tokens/total": 11732864,
|
|
"tokens/train_per_sec_per_gpu": 60.2,
|
|
"tokens/trainable": 7543964
|
|
},
|
|
{
|
|
"epoch": 0.5614910707322779,
|
|
"grad_norm": 0.15314599871635437,
|
|
"learning_rate": 4.2207337783412234e-05,
|
|
"loss": 1.6323293685913085,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.11578,
|
|
"step": 1800,
|
|
"tokens/total": 11799936,
|
|
"tokens/train_per_sec_per_gpu": 89.74,
|
|
"tokens/trainable": 7587628
|
|
},
|
|
{
|
|
"epoch": 0.5646104655696795,
|
|
"grad_norm": 0.1673821359872818,
|
|
"learning_rate": 4.2115518495371555e-05,
|
|
"loss": 1.7283849716186523,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.63155,
|
|
"step": 1810,
|
|
"tokens/total": 11866240,
|
|
"tokens/train_per_sec_per_gpu": 74.86,
|
|
"tokens/trainable": 7631435
|
|
},
|
|
{
|
|
"epoch": 0.5677298604070811,
|
|
"grad_norm": 0.1649511158466339,
|
|
"learning_rate": 4.2023262582819224e-05,
|
|
"loss": 1.6711223602294922,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.31813,
|
|
"step": 1820,
|
|
"tokens/total": 11930240,
|
|
"tokens/train_per_sec_per_gpu": 36.03,
|
|
"tokens/trainable": 7673091
|
|
},
|
|
{
|
|
"epoch": 0.5708492552444826,
|
|
"grad_norm": 0.1715686023235321,
|
|
"learning_rate": 4.193057239924475e-05,
|
|
"loss": 1.6728643417358398,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.32741,
|
|
"step": 1830,
|
|
"tokens/total": 11994880,
|
|
"tokens/train_per_sec_per_gpu": 45.46,
|
|
"tokens/trainable": 7714489
|
|
},
|
|
{
|
|
"epoch": 0.5739686500818841,
|
|
"grad_norm": 0.14329132437705994,
|
|
"learning_rate": 4.183745030921605e-05,
|
|
"loss": 1.6592113494873046,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.25516,
|
|
"step": 1840,
|
|
"tokens/total": 12061056,
|
|
"tokens/train_per_sec_per_gpu": 74.39,
|
|
"tokens/trainable": 7758111
|
|
},
|
|
{
|
|
"epoch": 0.5770880449192857,
|
|
"grad_norm": 0.18552425503730774,
|
|
"learning_rate": 4.1743898688319186e-05,
|
|
"loss": 1.7040969848632812,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.49642,
|
|
"step": 1850,
|
|
"tokens/total": 12126080,
|
|
"tokens/train_per_sec_per_gpu": 47.43,
|
|
"tokens/trainable": 7800458
|
|
},
|
|
{
|
|
"epoch": 0.5802074397566872,
|
|
"grad_norm": 0.1472683548927307,
|
|
"learning_rate": 4.1649919923097734e-05,
|
|
"loss": 1.6853845596313477,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.39453,
|
|
"step": 1860,
|
|
"tokens/total": 12191488,
|
|
"tokens/train_per_sec_per_gpu": 67.63,
|
|
"tokens/trainable": 7842092
|
|
},
|
|
{
|
|
"epoch": 0.5833268345940887,
|
|
"grad_norm": 0.1650121510028839,
|
|
"learning_rate": 4.1555516410991915e-05,
|
|
"loss": 1.6709466934204102,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.3172,
|
|
"step": 1870,
|
|
"tokens/total": 12255232,
|
|
"tokens/train_per_sec_per_gpu": 76.13,
|
|
"tokens/trainable": 7881660
|
|
},
|
|
{
|
|
"epoch": 0.5864462294314903,
|
|
"grad_norm": 0.18134747445583344,
|
|
"learning_rate": 4.146069056027745e-05,
|
|
"loss": 1.7022741317749024,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.48641,
|
|
"step": 1880,
|
|
"tokens/total": 12319232,
|
|
"tokens/train_per_sec_per_gpu": 47.92,
|
|
"tokens/trainable": 7923695
|
|
},
|
|
{
|
|
"epoch": 0.5895656242688918,
|
|
"grad_norm": 0.16381002962589264,
|
|
"learning_rate": 4.1365444790004065e-05,
|
|
"loss": 1.8045862197875977,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 6.07746,
|
|
"step": 1890,
|
|
"tokens/total": 12385280,
|
|
"tokens/train_per_sec_per_gpu": 58.75,
|
|
"tokens/trainable": 7964583
|
|
},
|
|
{
|
|
"epoch": 0.5926850191062933,
|
|
"grad_norm": 0.1534644365310669,
|
|
"learning_rate": 4.126978152993387e-05,
|
|
"loss": 1.6712440490722655,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.31878,
|
|
"step": 1900,
|
|
"tokens/total": 12449792,
|
|
"tokens/train_per_sec_per_gpu": 45.33,
|
|
"tokens/trainable": 8007788
|
|
},
|
|
{
|
|
"epoch": 0.5958044139436949,
|
|
"grad_norm": 0.15505635738372803,
|
|
"learning_rate": 4.1173703220479285e-05,
|
|
"loss": 1.7241756439208984,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.6079,
|
|
"step": 1910,
|
|
"tokens/total": 12515840,
|
|
"tokens/train_per_sec_per_gpu": 68.22,
|
|
"tokens/trainable": 8050631
|
|
},
|
|
{
|
|
"epoch": 0.5989238087810965,
|
|
"grad_norm": 0.15778960287570953,
|
|
"learning_rate": 4.107721231264086e-05,
|
|
"loss": 1.6882822036743164,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.41018,
|
|
"step": 1920,
|
|
"tokens/total": 12581760,
|
|
"tokens/train_per_sec_per_gpu": 30.11,
|
|
"tokens/trainable": 8093841
|
|
},
|
|
{
|
|
"epoch": 0.6020432036184981,
|
|
"grad_norm": 0.14546938240528107,
|
|
"learning_rate": 4.0980311267944664e-05,
|
|
"loss": 1.7195354461669923,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.58193,
|
|
"step": 1930,
|
|
"tokens/total": 12645632,
|
|
"tokens/train_per_sec_per_gpu": 97.39,
|
|
"tokens/trainable": 8135061
|
|
},
|
|
{
|
|
"epoch": 0.6051625984558996,
|
|
"grad_norm": 0.1503245085477829,
|
|
"learning_rate": 4.088300255837961e-05,
|
|
"loss": 1.6573673248291017,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.24548,
|
|
"step": 1940,
|
|
"tokens/total": 12712192,
|
|
"tokens/train_per_sec_per_gpu": 48.31,
|
|
"tokens/trainable": 8177817
|
|
},
|
|
{
|
|
"epoch": 0.6082819932933011,
|
|
"grad_norm": 0.16121330857276917,
|
|
"learning_rate": 4.078528866633427e-05,
|
|
"loss": 1.708116340637207,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.51856,
|
|
"step": 1950,
|
|
"tokens/total": 12777984,
|
|
"tokens/train_per_sec_per_gpu": 80.58,
|
|
"tokens/trainable": 8218869
|
|
},
|
|
{
|
|
"epoch": 0.6114013881307027,
|
|
"grad_norm": 0.16770608723163605,
|
|
"learning_rate": 4.0687172084533606e-05,
|
|
"loss": 1.665958786010742,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.29074,
|
|
"step": 1960,
|
|
"tokens/total": 12841728,
|
|
"tokens/train_per_sec_per_gpu": 62.41,
|
|
"tokens/trainable": 8258766
|
|
},
|
|
{
|
|
"epoch": 0.6145207829681042,
|
|
"grad_norm": 0.16197043657302856,
|
|
"learning_rate": 4.058865531597541e-05,
|
|
"loss": 1.753585433959961,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.77527,
|
|
"step": 1970,
|
|
"tokens/total": 12906240,
|
|
"tokens/train_per_sec_per_gpu": 27.36,
|
|
"tokens/trainable": 8300338
|
|
},
|
|
{
|
|
"epoch": 0.6176401778055057,
|
|
"grad_norm": 0.15220114588737488,
|
|
"learning_rate": 4.048974087386638e-05,
|
|
"loss": 1.5955570220947266,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.93108,
|
|
"step": 1980,
|
|
"tokens/total": 12972544,
|
|
"tokens/train_per_sec_per_gpu": 57.57,
|
|
"tokens/trainable": 8341980
|
|
},
|
|
{
|
|
"epoch": 0.6207595726429073,
|
|
"grad_norm": 0.14449943602085114,
|
|
"learning_rate": 4.039043128155806e-05,
|
|
"loss": 1.7109687805175782,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.53432,
|
|
"step": 1990,
|
|
"tokens/total": 13041024,
|
|
"tokens/train_per_sec_per_gpu": 75.98,
|
|
"tokens/trainable": 8385252
|
|
},
|
|
{
|
|
"epoch": 0.6238789674803088,
|
|
"grad_norm": 0.1620180904865265,
|
|
"learning_rate": 4.029072907248245e-05,
|
|
"loss": 1.6685823440551757,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.30464,
|
|
"step": 2000,
|
|
"tokens/total": 13108608,
|
|
"tokens/train_per_sec_per_gpu": 65.48,
|
|
"tokens/trainable": 8427824
|
|
},
|
|
{
|
|
"epoch": 0.6269983623177103,
|
|
"grad_norm": 0.16538818180561066,
|
|
"learning_rate": 4.019063679008737e-05,
|
|
"loss": 1.7287483215332031,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.6336,
|
|
"step": 2010,
|
|
"tokens/total": 13172480,
|
|
"tokens/train_per_sec_per_gpu": 83.18,
|
|
"tokens/trainable": 8469025
|
|
},
|
|
{
|
|
"epoch": 0.6301177571551119,
|
|
"grad_norm": 0.16408658027648926,
|
|
"learning_rate": 4.009015698777161e-05,
|
|
"loss": 1.7020183563232423,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.48501,
|
|
"step": 2020,
|
|
"tokens/total": 13237760,
|
|
"tokens/train_per_sec_per_gpu": 53.26,
|
|
"tokens/trainable": 8510333
|
|
},
|
|
{
|
|
"epoch": 0.6332371519925134,
|
|
"grad_norm": 0.15589982271194458,
|
|
"learning_rate": 3.998929222881973e-05,
|
|
"loss": 1.6746822357177735,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.3371,
|
|
"step": 2030,
|
|
"tokens/total": 13305472,
|
|
"tokens/train_per_sec_per_gpu": 83.47,
|
|
"tokens/trainable": 8554035
|
|
},
|
|
{
|
|
"epoch": 0.636356546829915,
|
|
"grad_norm": 0.1548449695110321,
|
|
"learning_rate": 3.9888045086336745e-05,
|
|
"loss": 1.6616172790527344,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.26782,
|
|
"step": 2040,
|
|
"tokens/total": 13369472,
|
|
"tokens/train_per_sec_per_gpu": 48.44,
|
|
"tokens/trainable": 8594085
|
|
},
|
|
{
|
|
"epoch": 0.6394759416673166,
|
|
"grad_norm": 0.15159164369106293,
|
|
"learning_rate": 3.978641814318241e-05,
|
|
"loss": 1.647285270690918,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.19286,
|
|
"step": 2050,
|
|
"tokens/total": 13435904,
|
|
"tokens/train_per_sec_per_gpu": 46.9,
|
|
"tokens/trainable": 8636637
|
|
},
|
|
{
|
|
"epoch": 0.6425953365047181,
|
|
"grad_norm": 0.16363564133644104,
|
|
"learning_rate": 3.968441399190538e-05,
|
|
"loss": 1.6983324050903321,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.46483,
|
|
"step": 2060,
|
|
"tokens/total": 13499520,
|
|
"tokens/train_per_sec_per_gpu": 79.42,
|
|
"tokens/trainable": 8678946
|
|
},
|
|
{
|
|
"epoch": 0.6457147313421197,
|
|
"grad_norm": 0.14427447319030762,
|
|
"learning_rate": 3.958203523467705e-05,
|
|
"loss": 1.6332363128662108,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.12042,
|
|
"step": 2070,
|
|
"tokens/total": 13564544,
|
|
"tokens/train_per_sec_per_gpu": 71.84,
|
|
"tokens/trainable": 8722197
|
|
},
|
|
{
|
|
"epoch": 0.6488341261795212,
|
|
"grad_norm": 0.145554319024086,
|
|
"learning_rate": 3.9479284483225184e-05,
|
|
"loss": 1.6738370895385741,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.33259,
|
|
"step": 2080,
|
|
"tokens/total": 13630720,
|
|
"tokens/train_per_sec_per_gpu": 78.03,
|
|
"tokens/trainable": 8764934
|
|
},
|
|
{
|
|
"epoch": 0.6519535210169227,
|
|
"grad_norm": 0.18742021918296814,
|
|
"learning_rate": 3.937616435876731e-05,
|
|
"loss": 1.7018247604370118,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.48395,
|
|
"step": 2090,
|
|
"tokens/total": 13697024,
|
|
"tokens/train_per_sec_per_gpu": 64.19,
|
|
"tokens/trainable": 8807807
|
|
},
|
|
{
|
|
"epoch": 0.6550729158543243,
|
|
"grad_norm": 0.14957238733768463,
|
|
"learning_rate": 3.927267749194378e-05,
|
|
"loss": 1.7438583374023438,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.71937,
|
|
"step": 2100,
|
|
"tokens/total": 13764096,
|
|
"tokens/train_per_sec_per_gpu": 80.33,
|
|
"tokens/trainable": 8851387
|
|
},
|
|
{
|
|
"epoch": 0.6581923106917258,
|
|
"grad_norm": 0.15080009400844574,
|
|
"learning_rate": 3.9168826522750735e-05,
|
|
"loss": 1.6331974029541017,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.12022,
|
|
"step": 2110,
|
|
"tokens/total": 13830016,
|
|
"tokens/train_per_sec_per_gpu": 65.4,
|
|
"tokens/trainable": 8893559
|
|
},
|
|
{
|
|
"epoch": 0.6613117055291273,
|
|
"grad_norm": 0.1466902792453766,
|
|
"learning_rate": 3.906461410047273e-05,
|
|
"loss": 1.6812294006347657,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.37216,
|
|
"step": 2120,
|
|
"tokens/total": 13898240,
|
|
"tokens/train_per_sec_per_gpu": 59.55,
|
|
"tokens/trainable": 8938095
|
|
},
|
|
{
|
|
"epoch": 0.6644311003665289,
|
|
"grad_norm": 0.15279127657413483,
|
|
"learning_rate": 3.8960042883615136e-05,
|
|
"loss": 1.7047332763671874,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.49992,
|
|
"step": 2130,
|
|
"tokens/total": 13963776,
|
|
"tokens/train_per_sec_per_gpu": 97.82,
|
|
"tokens/trainable": 8979656
|
|
},
|
|
{
|
|
"epoch": 0.6675504952039304,
|
|
"grad_norm": 0.15974970161914825,
|
|
"learning_rate": 3.885511553983634e-05,
|
|
"loss": 1.7363504409790038,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.67659,
|
|
"step": 2140,
|
|
"tokens/total": 14026240,
|
|
"tokens/train_per_sec_per_gpu": 81.77,
|
|
"tokens/trainable": 9019307
|
|
},
|
|
{
|
|
"epoch": 0.6706698900413319,
|
|
"grad_norm": 0.14892204105854034,
|
|
"learning_rate": 3.8749834745879696e-05,
|
|
"loss": 1.7014211654663085,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.48173,
|
|
"step": 2150,
|
|
"tokens/total": 14091392,
|
|
"tokens/train_per_sec_per_gpu": 49.43,
|
|
"tokens/trainable": 9061493
|
|
},
|
|
{
|
|
"epoch": 0.6737892848787336,
|
|
"grad_norm": 0.15679015219211578,
|
|
"learning_rate": 3.864420318750521e-05,
|
|
"loss": 1.6146787643432616,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.02627,
|
|
"step": 2160,
|
|
"tokens/total": 14158464,
|
|
"tokens/train_per_sec_per_gpu": 90.26,
|
|
"tokens/trainable": 9104301
|
|
},
|
|
{
|
|
"epoch": 0.6769086797161351,
|
|
"grad_norm": 0.14478443562984467,
|
|
"learning_rate": 3.853822355942107e-05,
|
|
"loss": 1.60101318359375,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.95805,
|
|
"step": 2170,
|
|
"tokens/total": 14226048,
|
|
"tokens/train_per_sec_per_gpu": 79.33,
|
|
"tokens/trainable": 9149404
|
|
},
|
|
{
|
|
"epoch": 0.6800280745535366,
|
|
"grad_norm": 0.16292953491210938,
|
|
"learning_rate": 3.8431898565214855e-05,
|
|
"loss": 1.6186635971069336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.04634,
|
|
"step": 2180,
|
|
"tokens/total": 14290816,
|
|
"tokens/train_per_sec_per_gpu": 86.89,
|
|
"tokens/trainable": 9191247
|
|
},
|
|
{
|
|
"epoch": 0.6831474693909382,
|
|
"grad_norm": 0.15199823677539825,
|
|
"learning_rate": 3.8325230917284605e-05,
|
|
"loss": 1.722325325012207,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.59753,
|
|
"step": 2190,
|
|
"tokens/total": 14357376,
|
|
"tokens/train_per_sec_per_gpu": 38.1,
|
|
"tokens/trainable": 9234842
|
|
},
|
|
{
|
|
"epoch": 0.6862668642283397,
|
|
"grad_norm": 0.161533921957016,
|
|
"learning_rate": 3.821822333676959e-05,
|
|
"loss": 1.637253761291504,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.14103,
|
|
"step": 2200,
|
|
"tokens/total": 14423296,
|
|
"tokens/train_per_sec_per_gpu": 53.81,
|
|
"tokens/trainable": 9276105
|
|
},
|
|
{
|
|
"epoch": 0.6893862590657412,
|
|
"grad_norm": 0.5114003419876099,
|
|
"learning_rate": 3.8110878553480914e-05,
|
|
"loss": 1.6784002304077148,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.35698,
|
|
"step": 2210,
|
|
"tokens/total": 14489088,
|
|
"tokens/train_per_sec_per_gpu": 27.08,
|
|
"tokens/trainable": 9317331
|
|
},
|
|
{
|
|
"epoch": 0.6925056539031428,
|
|
"grad_norm": 0.15215803682804108,
|
|
"learning_rate": 3.80031993058319e-05,
|
|
"loss": 1.6420883178710937,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.16595,
|
|
"step": 2220,
|
|
"tokens/total": 14555520,
|
|
"tokens/train_per_sec_per_gpu": 51.74,
|
|
"tokens/trainable": 9360606
|
|
},
|
|
{
|
|
"epoch": 0.6956250487405443,
|
|
"grad_norm": 0.1517273336648941,
|
|
"learning_rate": 3.789518834076818e-05,
|
|
"loss": 1.71380615234375,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.55005,
|
|
"step": 2230,
|
|
"tokens/total": 14622720,
|
|
"tokens/train_per_sec_per_gpu": 60.89,
|
|
"tokens/trainable": 9404074
|
|
},
|
|
{
|
|
"epoch": 0.6987444435779459,
|
|
"grad_norm": 0.1303180605173111,
|
|
"learning_rate": 3.778684841369766e-05,
|
|
"loss": 1.7221036911010743,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.59629,
|
|
"step": 2240,
|
|
"tokens/total": 14691328,
|
|
"tokens/train_per_sec_per_gpu": 72.57,
|
|
"tokens/trainable": 9449211
|
|
},
|
|
{
|
|
"epoch": 0.7018638384153474,
|
|
"grad_norm": 0.14437110722064972,
|
|
"learning_rate": 3.7678182288420206e-05,
|
|
"loss": 1.7238471984863282,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.60605,
|
|
"step": 2250,
|
|
"tokens/total": 14758016,
|
|
"tokens/train_per_sec_per_gpu": 94.88,
|
|
"tokens/trainable": 9495241
|
|
},
|
|
{
|
|
"epoch": 0.7049832332527489,
|
|
"grad_norm": 0.14504596590995789,
|
|
"learning_rate": 3.756919273705715e-05,
|
|
"loss": 1.748298454284668,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.74482,
|
|
"step": 2260,
|
|
"tokens/total": 14822656,
|
|
"tokens/train_per_sec_per_gpu": 74.77,
|
|
"tokens/trainable": 9537400
|
|
},
|
|
{
|
|
"epoch": 0.7081026280901506,
|
|
"grad_norm": 0.14621959626674652,
|
|
"learning_rate": 3.7459882539980544e-05,
|
|
"loss": 1.6361745834350585,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.13549,
|
|
"step": 2270,
|
|
"tokens/total": 14889344,
|
|
"tokens/train_per_sec_per_gpu": 60.82,
|
|
"tokens/trainable": 9580474
|
|
},
|
|
{
|
|
"epoch": 0.7112220229275521,
|
|
"grad_norm": 0.15423372387886047,
|
|
"learning_rate": 3.73502544857423e-05,
|
|
"loss": 1.6519367218017578,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.21707,
|
|
"step": 2280,
|
|
"tokens/total": 14954624,
|
|
"tokens/train_per_sec_per_gpu": 57.73,
|
|
"tokens/trainable": 9623344
|
|
},
|
|
{
|
|
"epoch": 0.7143414177649536,
|
|
"grad_norm": 0.15068814158439636,
|
|
"learning_rate": 3.7240311371002986e-05,
|
|
"loss": 1.6429204940795898,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.17025,
|
|
"step": 2290,
|
|
"tokens/total": 15021056,
|
|
"tokens/train_per_sec_per_gpu": 86.63,
|
|
"tokens/trainable": 9668246
|
|
},
|
|
{
|
|
"epoch": 0.7174608126023552,
|
|
"grad_norm": 0.17056693136692047,
|
|
"learning_rate": 3.713005600046048e-05,
|
|
"loss": 1.5880664825439452,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.89428,
|
|
"step": 2300,
|
|
"tokens/total": 15088000,
|
|
"tokens/train_per_sec_per_gpu": 60.85,
|
|
"tokens/trainable": 9711658
|
|
},
|
|
{
|
|
"epoch": 0.7205802074397567,
|
|
"grad_norm": 0.14314928650856018,
|
|
"learning_rate": 3.701949118677848e-05,
|
|
"loss": 1.684300994873047,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.38868,
|
|
"step": 2310,
|
|
"tokens/total": 15154432,
|
|
"tokens/train_per_sec_per_gpu": 55.7,
|
|
"tokens/trainable": 9755224
|
|
},
|
|
{
|
|
"epoch": 0.7236996022771582,
|
|
"grad_norm": 0.16593588888645172,
|
|
"learning_rate": 3.690861975051471e-05,
|
|
"loss": 1.6433883666992188,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.17267,
|
|
"step": 2320,
|
|
"tokens/total": 15218048,
|
|
"tokens/train_per_sec_per_gpu": 67.56,
|
|
"tokens/trainable": 9795281
|
|
},
|
|
{
|
|
"epoch": 0.7268189971145598,
|
|
"grad_norm": 0.16335247457027435,
|
|
"learning_rate": 3.6797444520048944e-05,
|
|
"loss": 1.64578857421875,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.1851,
|
|
"step": 2330,
|
|
"tokens/total": 15284480,
|
|
"tokens/train_per_sec_per_gpu": 110.67,
|
|
"tokens/trainable": 9838854
|
|
},
|
|
{
|
|
"epoch": 0.7299383919519613,
|
|
"grad_norm": 0.16695326566696167,
|
|
"learning_rate": 3.668596833151092e-05,
|
|
"loss": 1.5413406372070313,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.67085,
|
|
"step": 2340,
|
|
"tokens/total": 15350784,
|
|
"tokens/train_per_sec_per_gpu": 62.36,
|
|
"tokens/trainable": 9882180
|
|
},
|
|
{
|
|
"epoch": 0.7330577867893628,
|
|
"grad_norm": 0.1589886099100113,
|
|
"learning_rate": 3.657419402870794e-05,
|
|
"loss": 1.6398090362548827,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.15419,
|
|
"step": 2350,
|
|
"tokens/total": 15414656,
|
|
"tokens/train_per_sec_per_gpu": 125.58,
|
|
"tokens/trainable": 9922701
|
|
},
|
|
{
|
|
"epoch": 0.7361771816267644,
|
|
"grad_norm": 0.14891579747200012,
|
|
"learning_rate": 3.646212446305232e-05,
|
|
"loss": 1.619992446899414,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.05305,
|
|
"step": 2360,
|
|
"tokens/total": 15479296,
|
|
"tokens/train_per_sec_per_gpu": 86.24,
|
|
"tokens/trainable": 9966452
|
|
},
|
|
{
|
|
"epoch": 0.7392965764641659,
|
|
"grad_norm": 0.16890475153923035,
|
|
"learning_rate": 3.634976249348867e-05,
|
|
"loss": 1.686701202392578,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.40163,
|
|
"step": 2370,
|
|
"tokens/total": 15543168,
|
|
"tokens/train_per_sec_per_gpu": 37.47,
|
|
"tokens/trainable": 10007859
|
|
},
|
|
{
|
|
"epoch": 0.7424159713015674,
|
|
"grad_norm": 0.15340855717658997,
|
|
"learning_rate": 3.623711098642097e-05,
|
|
"loss": 1.6851417541503906,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.39322,
|
|
"step": 2380,
|
|
"tokens/total": 15610880,
|
|
"tokens/train_per_sec_per_gpu": 54.85,
|
|
"tokens/trainable": 10052552
|
|
},
|
|
{
|
|
"epoch": 0.7455353661389691,
|
|
"grad_norm": 0.15826597809791565,
|
|
"learning_rate": 3.6124172815639384e-05,
|
|
"loss": 1.65814266204834,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.24955,
|
|
"step": 2390,
|
|
"tokens/total": 15678080,
|
|
"tokens/train_per_sec_per_gpu": 89.33,
|
|
"tokens/trainable": 10095951
|
|
},
|
|
{
|
|
"epoch": 0.7486547609763706,
|
|
"grad_norm": 0.15100733935832977,
|
|
"learning_rate": 3.6010950862247056e-05,
|
|
"loss": 1.6023536682128907,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.9647,
|
|
"step": 2400,
|
|
"tokens/total": 15744256,
|
|
"tokens/train_per_sec_per_gpu": 64.33,
|
|
"tokens/trainable": 10139634
|
|
},
|
|
{
|
|
"epoch": 0.7517741558137722,
|
|
"grad_norm": 0.14118531346321106,
|
|
"learning_rate": 3.589744801458651e-05,
|
|
"loss": 1.7136390686035157,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.54912,
|
|
"step": 2410,
|
|
"tokens/total": 15808768,
|
|
"tokens/train_per_sec_per_gpu": 62.05,
|
|
"tokens/trainable": 10181495
|
|
},
|
|
{
|
|
"epoch": 0.7548935506511737,
|
|
"grad_norm": 0.14783544838428497,
|
|
"learning_rate": 3.578366716816603e-05,
|
|
"loss": 1.678952407836914,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.35994,
|
|
"step": 2420,
|
|
"tokens/total": 15874304,
|
|
"tokens/train_per_sec_per_gpu": 61.44,
|
|
"tokens/trainable": 10222361
|
|
},
|
|
{
|
|
"epoch": 0.7580129454885752,
|
|
"grad_norm": 0.16890275478363037,
|
|
"learning_rate": 3.5669611225585757e-05,
|
|
"loss": 1.612213134765625,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.0139,
|
|
"step": 2430,
|
|
"tokens/total": 15940096,
|
|
"tokens/train_per_sec_per_gpu": 89.51,
|
|
"tokens/trainable": 10264099
|
|
},
|
|
{
|
|
"epoch": 0.7611323403259768,
|
|
"grad_norm": 0.1930585652589798,
|
|
"learning_rate": 3.5555283096463665e-05,
|
|
"loss": 1.5885966300964356,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.89687,
|
|
"step": 2440,
|
|
"tokens/total": 16004992,
|
|
"tokens/train_per_sec_per_gpu": 81.34,
|
|
"tokens/trainable": 10306742
|
|
},
|
|
{
|
|
"epoch": 0.7642517351633783,
|
|
"grad_norm": 0.22118310630321503,
|
|
"learning_rate": 3.544068569736132e-05,
|
|
"loss": 1.710581398010254,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.53218,
|
|
"step": 2450,
|
|
"tokens/total": 16069504,
|
|
"tokens/train_per_sec_per_gpu": 50.85,
|
|
"tokens/trainable": 10349367
|
|
},
|
|
{
|
|
"epoch": 0.7673711300007798,
|
|
"grad_norm": 0.149618461728096,
|
|
"learning_rate": 3.5325821951709506e-05,
|
|
"loss": 1.5801680564880372,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.85577,
|
|
"step": 2460,
|
|
"tokens/total": 16135552,
|
|
"tokens/train_per_sec_per_gpu": 93.36,
|
|
"tokens/trainable": 10392831
|
|
},
|
|
{
|
|
"epoch": 0.7704905248381814,
|
|
"grad_norm": 0.17005199193954468,
|
|
"learning_rate": 3.52106947897336e-05,
|
|
"loss": 1.6157630920410155,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.03173,
|
|
"step": 2470,
|
|
"tokens/total": 16202368,
|
|
"tokens/train_per_sec_per_gpu": 37.08,
|
|
"tokens/trainable": 10436978
|
|
},
|
|
{
|
|
"epoch": 0.7736099196755829,
|
|
"grad_norm": 0.16539916396141052,
|
|
"learning_rate": 3.5095307148378876e-05,
|
|
"loss": 1.6135873794555664,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.02079,
|
|
"step": 2480,
|
|
"tokens/total": 16270080,
|
|
"tokens/train_per_sec_per_gpu": 49.99,
|
|
"tokens/trainable": 10479767
|
|
},
|
|
{
|
|
"epoch": 0.7767293145129844,
|
|
"grad_norm": 0.15634194016456604,
|
|
"learning_rate": 3.497966197123552e-05,
|
|
"loss": 1.7117055892944335,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.5384,
|
|
"step": 2490,
|
|
"tokens/total": 16336768,
|
|
"tokens/train_per_sec_per_gpu": 53.83,
|
|
"tokens/trainable": 10523484
|
|
},
|
|
{
|
|
"epoch": 0.779848709350386,
|
|
"grad_norm": 0.15308453142642975,
|
|
"learning_rate": 3.486376220846359e-05,
|
|
"loss": 1.6526248931884766,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.22067,
|
|
"step": 2500,
|
|
"tokens/total": 16402816,
|
|
"tokens/train_per_sec_per_gpu": 73.7,
|
|
"tokens/trainable": 10565133
|
|
},
|
|
{
|
|
"epoch": 0.7829681041877876,
|
|
"grad_norm": 0.1443682312965393,
|
|
"learning_rate": 3.474761081671774e-05,
|
|
"loss": 1.7210195541381836,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.59023,
|
|
"step": 2510,
|
|
"tokens/total": 16469504,
|
|
"tokens/train_per_sec_per_gpu": 35.9,
|
|
"tokens/trainable": 10606608
|
|
},
|
|
{
|
|
"epoch": 0.7860874990251892,
|
|
"grad_norm": 0.1387036144733429,
|
|
"learning_rate": 3.463121075907178e-05,
|
|
"loss": 1.6668996810913086,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.29572,
|
|
"step": 2520,
|
|
"tokens/total": 16533632,
|
|
"tokens/train_per_sec_per_gpu": 56.11,
|
|
"tokens/trainable": 10648100
|
|
},
|
|
{
|
|
"epoch": 0.7892068938625907,
|
|
"grad_norm": 0.1458800584077835,
|
|
"learning_rate": 3.451456500494311e-05,
|
|
"loss": 1.689809799194336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.41845,
|
|
"step": 2530,
|
|
"tokens/total": 16599424,
|
|
"tokens/train_per_sec_per_gpu": 71.76,
|
|
"tokens/trainable": 10691467
|
|
},
|
|
{
|
|
"epoch": 0.7923262886999922,
|
|
"grad_norm": 0.153626948595047,
|
|
"learning_rate": 3.439767653001694e-05,
|
|
"loss": 1.6851633071899415,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.39333,
|
|
"step": 2540,
|
|
"tokens/total": 16667520,
|
|
"tokens/train_per_sec_per_gpu": 68.94,
|
|
"tokens/trainable": 10734798
|
|
},
|
|
{
|
|
"epoch": 0.7954456835373938,
|
|
"grad_norm": 0.1487371027469635,
|
|
"learning_rate": 3.428054831617042e-05,
|
|
"loss": 1.5997867584228516,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.95198,
|
|
"step": 2550,
|
|
"tokens/total": 16732544,
|
|
"tokens/train_per_sec_per_gpu": 70.12,
|
|
"tokens/trainable": 10777893
|
|
},
|
|
{
|
|
"epoch": 0.7985650783747953,
|
|
"grad_norm": 0.14688651263713837,
|
|
"learning_rate": 3.416318335139653e-05,
|
|
"loss": 1.6785497665405273,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.35778,
|
|
"step": 2560,
|
|
"tokens/total": 16795904,
|
|
"tokens/train_per_sec_per_gpu": 74.44,
|
|
"tokens/trainable": 10817921
|
|
},
|
|
{
|
|
"epoch": 0.8016844732121968,
|
|
"grad_norm": 0.15249019861221313,
|
|
"learning_rate": 3.4045584629727865e-05,
|
|
"loss": 1.6235479354858398,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.07105,
|
|
"step": 2570,
|
|
"tokens/total": 16864896,
|
|
"tokens/train_per_sec_per_gpu": 72.86,
|
|
"tokens/trainable": 10862416
|
|
},
|
|
{
|
|
"epoch": 0.8048038680495984,
|
|
"grad_norm": 0.1517254263162613,
|
|
"learning_rate": 3.392775515116028e-05,
|
|
"loss": 1.6850980758666991,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.39298,
|
|
"step": 2580,
|
|
"tokens/total": 16931328,
|
|
"tokens/train_per_sec_per_gpu": 40.86,
|
|
"tokens/trainable": 10903943
|
|
},
|
|
{
|
|
"epoch": 0.8079232628869999,
|
|
"grad_norm": 0.15624631941318512,
|
|
"learning_rate": 3.380969792157635e-05,
|
|
"loss": 1.5908714294433595,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.90802,
|
|
"step": 2590,
|
|
"tokens/total": 16995968,
|
|
"tokens/train_per_sec_per_gpu": 55.2,
|
|
"tokens/trainable": 10944851
|
|
},
|
|
{
|
|
"epoch": 0.8110426577244014,
|
|
"grad_norm": 0.1472957283258438,
|
|
"learning_rate": 3.369141595266863e-05,
|
|
"loss": 1.6641403198242188,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.28113,
|
|
"step": 2600,
|
|
"tokens/total": 17061632,
|
|
"tokens/train_per_sec_per_gpu": 41.22,
|
|
"tokens/trainable": 10986052
|
|
},
|
|
{
|
|
"epoch": 0.814162052561803,
|
|
"grad_norm": 0.15229563415050507,
|
|
"learning_rate": 3.357291226186293e-05,
|
|
"loss": 1.5812364578247071,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.86096,
|
|
"step": 2610,
|
|
"tokens/total": 17129984,
|
|
"tokens/train_per_sec_per_gpu": 89.44,
|
|
"tokens/trainable": 11031137
|
|
},
|
|
{
|
|
"epoch": 0.8172814473992046,
|
|
"grad_norm": 0.16603821516036987,
|
|
"learning_rate": 3.345418987224126e-05,
|
|
"loss": 1.6037185668945313,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.97148,
|
|
"step": 2620,
|
|
"tokens/total": 17194624,
|
|
"tokens/train_per_sec_per_gpu": 73.82,
|
|
"tokens/trainable": 11071568
|
|
},
|
|
{
|
|
"epoch": 0.8204008422366061,
|
|
"grad_norm": 0.1586580127477646,
|
|
"learning_rate": 3.333525181246475e-05,
|
|
"loss": 1.6416893005371094,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.16389,
|
|
"step": 2630,
|
|
"tokens/total": 17258368,
|
|
"tokens/train_per_sec_per_gpu": 33.04,
|
|
"tokens/trainable": 11113121
|
|
},
|
|
{
|
|
"epoch": 0.8235202370740077,
|
|
"grad_norm": 0.14638447761535645,
|
|
"learning_rate": 3.321610111669635e-05,
|
|
"loss": 1.6287551879882813,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.09753,
|
|
"step": 2640,
|
|
"tokens/total": 17323904,
|
|
"tokens/train_per_sec_per_gpu": 82.68,
|
|
"tokens/trainable": 11155603
|
|
},
|
|
{
|
|
"epoch": 0.8266396319114092,
|
|
"grad_norm": 0.13621027767658234,
|
|
"learning_rate": 3.3096740824523454e-05,
|
|
"loss": 1.6125598907470704,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.01563,
|
|
"step": 2650,
|
|
"tokens/total": 17390848,
|
|
"tokens/train_per_sec_per_gpu": 61.2,
|
|
"tokens/trainable": 11197354
|
|
},
|
|
{
|
|
"epoch": 0.8297590267488107,
|
|
"grad_norm": 0.1590825766324997,
|
|
"learning_rate": 3.297717398088037e-05,
|
|
"loss": 1.6573667526245117,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.24548,
|
|
"step": 2660,
|
|
"tokens/total": 17455104,
|
|
"tokens/train_per_sec_per_gpu": 39.48,
|
|
"tokens/trainable": 11237620
|
|
},
|
|
{
|
|
"epoch": 0.8328784215862123,
|
|
"grad_norm": 0.17578908801078796,
|
|
"learning_rate": 3.28574036359706e-05,
|
|
"loss": 1.6356090545654296,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.13258,
|
|
"step": 2670,
|
|
"tokens/total": 17521408,
|
|
"tokens/train_per_sec_per_gpu": 53.72,
|
|
"tokens/trainable": 11280603
|
|
},
|
|
{
|
|
"epoch": 0.8359978164236138,
|
|
"grad_norm": 0.14806489646434784,
|
|
"learning_rate": 3.273743284518909e-05,
|
|
"loss": 1.6927942276000976,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.43465,
|
|
"step": 2680,
|
|
"tokens/total": 17588096,
|
|
"tokens/train_per_sec_per_gpu": 41.1,
|
|
"tokens/trainable": 11323714
|
|
},
|
|
{
|
|
"epoch": 0.8391172112610153,
|
|
"grad_norm": 0.16140392422676086,
|
|
"learning_rate": 3.261726466904421e-05,
|
|
"loss": 1.669748306274414,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.31083,
|
|
"step": 2690,
|
|
"tokens/total": 17654400,
|
|
"tokens/train_per_sec_per_gpu": 86.83,
|
|
"tokens/trainable": 11366476
|
|
},
|
|
{
|
|
"epoch": 0.8422366060984169,
|
|
"grad_norm": 0.13898003101348877,
|
|
"learning_rate": 3.249690217307976e-05,
|
|
"loss": 1.5336098670959473,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.63488,
|
|
"step": 2700,
|
|
"tokens/total": 17721472,
|
|
"tokens/train_per_sec_per_gpu": 55.29,
|
|
"tokens/trainable": 11411706
|
|
},
|
|
{
|
|
"epoch": 0.8453560009358184,
|
|
"grad_norm": 0.14206239581108093,
|
|
"learning_rate": 3.2376348427796675e-05,
|
|
"loss": 1.6422727584838868,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.1669,
|
|
"step": 2710,
|
|
"tokens/total": 17788416,
|
|
"tokens/train_per_sec_per_gpu": 63.92,
|
|
"tokens/trainable": 11455418
|
|
},
|
|
{
|
|
"epoch": 0.84847539577322,
|
|
"grad_norm": 0.14273086190223694,
|
|
"learning_rate": 3.225560650857479e-05,
|
|
"loss": 1.678399658203125,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.35698,
|
|
"step": 2720,
|
|
"tokens/total": 17853824,
|
|
"tokens/train_per_sec_per_gpu": 75.51,
|
|
"tokens/trainable": 11496149
|
|
},
|
|
{
|
|
"epoch": 0.8515947906106215,
|
|
"grad_norm": 0.1582791954278946,
|
|
"learning_rate": 3.213467949559432e-05,
|
|
"loss": 1.687089729309082,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.40373,
|
|
"step": 2730,
|
|
"tokens/total": 17919872,
|
|
"tokens/train_per_sec_per_gpu": 75.73,
|
|
"tokens/trainable": 11537959
|
|
},
|
|
{
|
|
"epoch": 0.8547141854480231,
|
|
"grad_norm": 0.14645002782344818,
|
|
"learning_rate": 3.201357047375732e-05,
|
|
"loss": 1.6819684982299805,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.37613,
|
|
"step": 2740,
|
|
"tokens/total": 17985152,
|
|
"tokens/train_per_sec_per_gpu": 99.28,
|
|
"tokens/trainable": 11580986
|
|
},
|
|
{
|
|
"epoch": 0.8578335802854247,
|
|
"grad_norm": 0.15471987426280975,
|
|
"learning_rate": 3.1892282532608934e-05,
|
|
"loss": 1.681475830078125,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.37348,
|
|
"step": 2750,
|
|
"tokens/total": 18049536,
|
|
"tokens/train_per_sec_per_gpu": 41.73,
|
|
"tokens/trainable": 11623010
|
|
},
|
|
{
|
|
"epoch": 0.8609529751228262,
|
|
"grad_norm": 0.15031155943870544,
|
|
"learning_rate": 3.177081876625867e-05,
|
|
"loss": 1.6249948501586915,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.07839,
|
|
"step": 2760,
|
|
"tokens/total": 18115072,
|
|
"tokens/train_per_sec_per_gpu": 63.86,
|
|
"tokens/trainable": 11665842
|
|
},
|
|
{
|
|
"epoch": 0.8640723699602277,
|
|
"grad_norm": 0.1480269432067871,
|
|
"learning_rate": 3.164918227330138e-05,
|
|
"loss": 1.6917524337768555,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.42899,
|
|
"step": 2770,
|
|
"tokens/total": 18180608,
|
|
"tokens/train_per_sec_per_gpu": 61.66,
|
|
"tokens/trainable": 11707656
|
|
},
|
|
{
|
|
"epoch": 0.8671917647976293,
|
|
"grad_norm": 0.17693500220775604,
|
|
"learning_rate": 3.1527376156738256e-05,
|
|
"loss": 1.6080030441284179,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.99283,
|
|
"step": 2780,
|
|
"tokens/total": 18245120,
|
|
"tokens/train_per_sec_per_gpu": 63.75,
|
|
"tokens/trainable": 11748480
|
|
},
|
|
{
|
|
"epoch": 0.8703111596350308,
|
|
"grad_norm": 0.1379125565290451,
|
|
"learning_rate": 3.140540352389768e-05,
|
|
"loss": 1.6859777450561524,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.39773,
|
|
"step": 2790,
|
|
"tokens/total": 18309504,
|
|
"tokens/train_per_sec_per_gpu": 92.03,
|
|
"tokens/trainable": 11790023
|
|
},
|
|
{
|
|
"epoch": 0.8734305544724323,
|
|
"grad_norm": 0.12693195044994354,
|
|
"learning_rate": 3.1283267486355905e-05,
|
|
"loss": 1.6941141128540038,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.44182,
|
|
"step": 2800,
|
|
"tokens/total": 18377728,
|
|
"tokens/train_per_sec_per_gpu": 71.86,
|
|
"tokens/trainable": 11834194
|
|
},
|
|
{
|
|
"epoch": 0.8765499493098339,
|
|
"grad_norm": 0.1393202841281891,
|
|
"learning_rate": 3.116097115985775e-05,
|
|
"loss": 1.6291584014892577,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.09958,
|
|
"step": 2810,
|
|
"tokens/total": 18444800,
|
|
"tokens/train_per_sec_per_gpu": 58.06,
|
|
"tokens/trainable": 11877918
|
|
},
|
|
{
|
|
"epoch": 0.8796693441472354,
|
|
"grad_norm": 0.142860546708107,
|
|
"learning_rate": 3.103851766423704e-05,
|
|
"loss": 1.5834574699401855,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.87177,
|
|
"step": 2820,
|
|
"tokens/total": 18512896,
|
|
"tokens/train_per_sec_per_gpu": 84.92,
|
|
"tokens/trainable": 11923152
|
|
},
|
|
{
|
|
"epoch": 0.882788738984637,
|
|
"grad_norm": 0.14666636288166046,
|
|
"learning_rate": 3.091591012333707e-05,
|
|
"loss": 1.6785709381103515,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.35789,
|
|
"step": 2830,
|
|
"tokens/total": 18577024,
|
|
"tokens/train_per_sec_per_gpu": 47.94,
|
|
"tokens/trainable": 11964843
|
|
},
|
|
{
|
|
"epoch": 0.8859081338220385,
|
|
"grad_norm": 0.17379850149154663,
|
|
"learning_rate": 3.079315166493093e-05,
|
|
"loss": 1.553987979888916,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.7303,
|
|
"step": 2840,
|
|
"tokens/total": 18640640,
|
|
"tokens/train_per_sec_per_gpu": 87.45,
|
|
"tokens/trainable": 12006061
|
|
},
|
|
{
|
|
"epoch": 0.88902752865944,
|
|
"grad_norm": 0.14987123012542725,
|
|
"learning_rate": 3.0670245420641616e-05,
|
|
"loss": 1.5898012161254882,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.90277,
|
|
"step": 2850,
|
|
"tokens/total": 18707712,
|
|
"tokens/train_per_sec_per_gpu": 39.4,
|
|
"tokens/trainable": 12048874
|
|
},
|
|
{
|
|
"epoch": 0.8921469234968417,
|
|
"grad_norm": 0.14232513308525085,
|
|
"learning_rate": 3.054719452586227e-05,
|
|
"loss": 1.550221824645996,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.71252,
|
|
"step": 2860,
|
|
"tokens/total": 18771456,
|
|
"tokens/train_per_sec_per_gpu": 65.01,
|
|
"tokens/trainable": 12091334
|
|
},
|
|
{
|
|
"epoch": 0.8952663183342432,
|
|
"grad_norm": 0.14222390949726105,
|
|
"learning_rate": 3.0424002119676097e-05,
|
|
"loss": 1.5611849784851075,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.76446,
|
|
"step": 2870,
|
|
"tokens/total": 18837888,
|
|
"tokens/train_per_sec_per_gpu": 69.81,
|
|
"tokens/trainable": 12134043
|
|
},
|
|
{
|
|
"epoch": 0.8983857131716447,
|
|
"grad_norm": 0.15067677199840546,
|
|
"learning_rate": 3.030067134477634e-05,
|
|
"loss": 1.603945541381836,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.97261,
|
|
"step": 2880,
|
|
"tokens/total": 18904576,
|
|
"tokens/train_per_sec_per_gpu": 68.71,
|
|
"tokens/trainable": 12175436
|
|
},
|
|
{
|
|
"epoch": 0.9015051080090463,
|
|
"grad_norm": 0.15149745345115662,
|
|
"learning_rate": 3.0177205347386074e-05,
|
|
"loss": 1.5672722816467286,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.79355,
|
|
"step": 2890,
|
|
"tokens/total": 18970368,
|
|
"tokens/train_per_sec_per_gpu": 90.89,
|
|
"tokens/trainable": 12217831
|
|
},
|
|
{
|
|
"epoch": 0.9046245028464478,
|
|
"grad_norm": 0.172475203871727,
|
|
"learning_rate": 3.0053607277177975e-05,
|
|
"loss": 1.608574676513672,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.99569,
|
|
"step": 2900,
|
|
"tokens/total": 19034496,
|
|
"tokens/train_per_sec_per_gpu": 29.51,
|
|
"tokens/trainable": 12256952
|
|
},
|
|
{
|
|
"epoch": 0.9077438976838493,
|
|
"grad_norm": 0.14654256403446198,
|
|
"learning_rate": 2.992988028719394e-05,
|
|
"loss": 1.6018085479736328,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.962,
|
|
"step": 2910,
|
|
"tokens/total": 19101568,
|
|
"tokens/train_per_sec_per_gpu": 84.55,
|
|
"tokens/trainable": 12300679
|
|
},
|
|
{
|
|
"epoch": 0.9108632925212509,
|
|
"grad_norm": 0.13963788747787476,
|
|
"learning_rate": 2.980602753376468e-05,
|
|
"loss": 1.5836727142333984,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.87282,
|
|
"step": 2920,
|
|
"tokens/total": 19168640,
|
|
"tokens/train_per_sec_per_gpu": 62.33,
|
|
"tokens/trainable": 12344261
|
|
},
|
|
{
|
|
"epoch": 0.9139826873586524,
|
|
"grad_norm": 0.1477433145046234,
|
|
"learning_rate": 2.9682052176429177e-05,
|
|
"loss": 1.6712848663330078,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.319,
|
|
"step": 2930,
|
|
"tokens/total": 19231744,
|
|
"tokens/train_per_sec_per_gpu": 52.5,
|
|
"tokens/trainable": 12384962
|
|
},
|
|
{
|
|
"epoch": 0.9171020821960539,
|
|
"grad_norm": 0.15684698522090912,
|
|
"learning_rate": 2.9557957377854106e-05,
|
|
"loss": 1.548504638671875,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.70443,
|
|
"step": 2940,
|
|
"tokens/total": 19295744,
|
|
"tokens/train_per_sec_per_gpu": 69.22,
|
|
"tokens/trainable": 12425479
|
|
},
|
|
{
|
|
"epoch": 0.9202214770334555,
|
|
"grad_norm": 0.15225949883460999,
|
|
"learning_rate": 2.9433746303753128e-05,
|
|
"loss": 1.6473196029663086,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.19304,
|
|
"step": 2950,
|
|
"tokens/total": 19363328,
|
|
"tokens/train_per_sec_per_gpu": 78.42,
|
|
"tokens/trainable": 12469489
|
|
},
|
|
{
|
|
"epoch": 0.923340871870857,
|
|
"grad_norm": 0.16261142492294312,
|
|
"learning_rate": 2.930942212280616e-05,
|
|
"loss": 1.638981819152832,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.14992,
|
|
"step": 2960,
|
|
"tokens/total": 19428608,
|
|
"tokens/train_per_sec_per_gpu": 36.81,
|
|
"tokens/trainable": 12511665
|
|
},
|
|
{
|
|
"epoch": 0.9264602667082587,
|
|
"grad_norm": 0.13145232200622559,
|
|
"learning_rate": 2.9184988006578505e-05,
|
|
"loss": 1.7191638946533203,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.57986,
|
|
"step": 2970,
|
|
"tokens/total": 19494144,
|
|
"tokens/train_per_sec_per_gpu": 105.79,
|
|
"tokens/trainable": 12554538
|
|
},
|
|
{
|
|
"epoch": 0.9295796615456602,
|
|
"grad_norm": 0.1353570967912674,
|
|
"learning_rate": 2.9060447129439993e-05,
|
|
"loss": 1.642087745666504,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.16594,
|
|
"step": 2980,
|
|
"tokens/total": 19558912,
|
|
"tokens/train_per_sec_per_gpu": 71.26,
|
|
"tokens/trainable": 12596038
|
|
},
|
|
{
|
|
"epoch": 0.9326990563830617,
|
|
"grad_norm": 0.15926408767700195,
|
|
"learning_rate": 2.8935802668483935e-05,
|
|
"loss": 1.6712854385375977,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.319,
|
|
"step": 2990,
|
|
"tokens/total": 19625088,
|
|
"tokens/train_per_sec_per_gpu": 56.25,
|
|
"tokens/trainable": 12639201
|
|
},
|
|
{
|
|
"epoch": 0.9358184512204633,
|
|
"grad_norm": 0.1459934264421463,
|
|
"learning_rate": 2.8811057803446134e-05,
|
|
"loss": 1.6127311706542968,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.01649,
|
|
"step": 3000,
|
|
"tokens/total": 19691008,
|
|
"tokens/train_per_sec_per_gpu": 105.4,
|
|
"tokens/trainable": 12681847
|
|
},
|
|
{
|
|
"epoch": 0.9389378460578648,
|
|
"grad_norm": 0.14705012738704681,
|
|
"learning_rate": 2.8686215716623756e-05,
|
|
"loss": 1.593970775604248,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.92326,
|
|
"step": 3010,
|
|
"tokens/total": 19755648,
|
|
"tokens/train_per_sec_per_gpu": 77.51,
|
|
"tokens/trainable": 12722838
|
|
},
|
|
{
|
|
"epoch": 0.9420572408952663,
|
|
"grad_norm": 0.15950429439544678,
|
|
"learning_rate": 2.856127959279411e-05,
|
|
"loss": 1.6436496734619142,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.17402,
|
|
"step": 3020,
|
|
"tokens/total": 19820544,
|
|
"tokens/train_per_sec_per_gpu": 46.0,
|
|
"tokens/trainable": 12764446
|
|
},
|
|
{
|
|
"epoch": 0.9451766357326679,
|
|
"grad_norm": 0.15326200425624847,
|
|
"learning_rate": 2.8436252619133454e-05,
|
|
"loss": 1.6361040115356444,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.13512,
|
|
"step": 3030,
|
|
"tokens/total": 19886464,
|
|
"tokens/train_per_sec_per_gpu": 63.52,
|
|
"tokens/trainable": 12806484
|
|
},
|
|
{
|
|
"epoch": 0.9482960305700694,
|
|
"grad_norm": 0.13497261703014374,
|
|
"learning_rate": 2.8311137985135644e-05,
|
|
"loss": 1.6122051239013673,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.01386,
|
|
"step": 3040,
|
|
"tokens/total": 19953920,
|
|
"tokens/train_per_sec_per_gpu": 46.47,
|
|
"tokens/trainable": 12849696
|
|
},
|
|
{
|
|
"epoch": 0.9514154254074709,
|
|
"grad_norm": 0.15848316252231598,
|
|
"learning_rate": 2.8185938882530825e-05,
|
|
"loss": 1.634846305847168,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.12867,
|
|
"step": 3050,
|
|
"tokens/total": 20020864,
|
|
"tokens/train_per_sec_per_gpu": 57.37,
|
|
"tokens/trainable": 12893263
|
|
},
|
|
{
|
|
"epoch": 0.9545348202448725,
|
|
"grad_norm": 0.1317116618156433,
|
|
"learning_rate": 2.806065850520394e-05,
|
|
"loss": 1.6756248474121094,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.34213,
|
|
"step": 3060,
|
|
"tokens/total": 20085888,
|
|
"tokens/train_per_sec_per_gpu": 72.33,
|
|
"tokens/trainable": 12935926
|
|
},
|
|
{
|
|
"epoch": 0.957654215082274,
|
|
"grad_norm": 0.15478383004665375,
|
|
"learning_rate": 2.79353000491133e-05,
|
|
"loss": 1.6063106536865235,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.98439,
|
|
"step": 3070,
|
|
"tokens/total": 20150144,
|
|
"tokens/train_per_sec_per_gpu": 32.75,
|
|
"tokens/trainable": 12975935
|
|
},
|
|
{
|
|
"epoch": 0.9607736099196755,
|
|
"grad_norm": 0.143395334482193,
|
|
"learning_rate": 2.7809866712209043e-05,
|
|
"loss": 1.6624685287475587,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.27231,
|
|
"step": 3080,
|
|
"tokens/total": 20213888,
|
|
"tokens/train_per_sec_per_gpu": 86.53,
|
|
"tokens/trainable": 13017104
|
|
},
|
|
{
|
|
"epoch": 0.9638930047570772,
|
|
"grad_norm": 0.14224320650100708,
|
|
"learning_rate": 2.768436169435155e-05,
|
|
"loss": 1.6242662429809571,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.07469,
|
|
"step": 3090,
|
|
"tokens/total": 20280576,
|
|
"tokens/train_per_sec_per_gpu": 68.36,
|
|
"tokens/trainable": 13058796
|
|
},
|
|
{
|
|
"epoch": 0.9670123995944787,
|
|
"grad_norm": 0.15930965542793274,
|
|
"learning_rate": 2.7558788197229797e-05,
|
|
"loss": 1.5603550910949706,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.76051,
|
|
"step": 3100,
|
|
"tokens/total": 20347008,
|
|
"tokens/train_per_sec_per_gpu": 72.65,
|
|
"tokens/trainable": 13102302
|
|
},
|
|
{
|
|
"epoch": 0.9701317944318802,
|
|
"grad_norm": 0.13159207999706268,
|
|
"learning_rate": 2.7433149424279726e-05,
|
|
"loss": 1.659048080444336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.25431,
|
|
"step": 3110,
|
|
"tokens/total": 20412416,
|
|
"tokens/train_per_sec_per_gpu": 86.4,
|
|
"tokens/trainable": 13145067
|
|
},
|
|
{
|
|
"epoch": 0.9732511892692818,
|
|
"grad_norm": 0.1585746854543686,
|
|
"learning_rate": 2.7307448580602468e-05,
|
|
"loss": 1.683304214477539,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.38331,
|
|
"step": 3120,
|
|
"tokens/total": 20478464,
|
|
"tokens/train_per_sec_per_gpu": 47.64,
|
|
"tokens/trainable": 13187985
|
|
},
|
|
{
|
|
"epoch": 0.9763705841066833,
|
|
"grad_norm": 0.15567518770694733,
|
|
"learning_rate": 2.7181688872882622e-05,
|
|
"loss": 1.6149152755737304,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.02746,
|
|
"step": 3130,
|
|
"tokens/total": 20544256,
|
|
"tokens/train_per_sec_per_gpu": 126.29,
|
|
"tokens/trainable": 13231280
|
|
},
|
|
{
|
|
"epoch": 0.9794899789440848,
|
|
"grad_norm": 0.13839098811149597,
|
|
"learning_rate": 2.705587350930643e-05,
|
|
"loss": 1.5398646354675294,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.66396,
|
|
"step": 3140,
|
|
"tokens/total": 20611584,
|
|
"tokens/train_per_sec_per_gpu": 47.77,
|
|
"tokens/trainable": 13273911
|
|
},
|
|
{
|
|
"epoch": 0.9826093737814864,
|
|
"grad_norm": 0.17286650836467743,
|
|
"learning_rate": 2.693000569947994e-05,
|
|
"loss": 1.6025777816772462,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.96582,
|
|
"step": 3150,
|
|
"tokens/total": 20677376,
|
|
"tokens/train_per_sec_per_gpu": 77.73,
|
|
"tokens/trainable": 13315012
|
|
},
|
|
{
|
|
"epoch": 0.9857287686188879,
|
|
"grad_norm": 0.13749009370803833,
|
|
"learning_rate": 2.680408865434713e-05,
|
|
"loss": 1.5244282722473144,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.59252,
|
|
"step": 3160,
|
|
"tokens/total": 20744960,
|
|
"tokens/train_per_sec_per_gpu": 85.96,
|
|
"tokens/trainable": 13358169
|
|
},
|
|
{
|
|
"epoch": 0.9888481634562895,
|
|
"grad_norm": 0.1448889672756195,
|
|
"learning_rate": 2.667812558610798e-05,
|
|
"loss": 1.6376134872436523,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.14288,
|
|
"step": 3170,
|
|
"tokens/total": 20812032,
|
|
"tokens/train_per_sec_per_gpu": 78.51,
|
|
"tokens/trainable": 13401627
|
|
},
|
|
{
|
|
"epoch": 0.991967558293691,
|
|
"grad_norm": 0.14404967427253723,
|
|
"learning_rate": 2.6552119708136574e-05,
|
|
"loss": 1.7385068893432618,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.68884,
|
|
"step": 3180,
|
|
"tokens/total": 20878976,
|
|
"tokens/train_per_sec_per_gpu": 102.91,
|
|
"tokens/trainable": 13444769
|
|
},
|
|
{
|
|
"epoch": 0.9950869531310925,
|
|
"grad_norm": 0.14747276902198792,
|
|
"learning_rate": 2.6426074234899045e-05,
|
|
"loss": 1.6169326782226563,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.03761,
|
|
"step": 3190,
|
|
"tokens/total": 20947968,
|
|
"tokens/train_per_sec_per_gpu": 81.53,
|
|
"tokens/trainable": 13489336
|
|
},
|
|
{
|
|
"epoch": 0.9982063479684942,
|
|
"grad_norm": 0.14146211743354797,
|
|
"learning_rate": 2.6299992381871642e-05,
|
|
"loss": 1.5174983024597168,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.5608,
|
|
"step": 3200,
|
|
"tokens/total": 21011968,
|
|
"tokens/train_per_sec_per_gpu": 75.27,
|
|
"tokens/trainable": 13529857
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"eval_loss": 1.5506293773651123,
|
|
"eval_ppl": 4.71444,
|
|
"eval_runtime": 39.3171,
|
|
"eval_samples_per_second": 68.672,
|
|
"eval_steps_per_second": 34.336,
|
|
"memory/device_reserved (GiB)": 4.49,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"step": 3206
|
|
},
|
|
{
|
|
"epoch": 1.0012477579349606,
|
|
"grad_norm": 0.14874127507209778,
|
|
"learning_rate": 2.6173877365458678e-05,
|
|
"loss": 1.6652931213378905,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.28722,
|
|
"step": 3210,
|
|
"tokens/total": 21075584,
|
|
"tokens/train_per_sec_per_gpu": 110.38,
|
|
"tokens/trainable": 13570789
|
|
},
|
|
{
|
|
"epoch": 1.004367152772362,
|
|
"grad_norm": 0.1414535492658615,
|
|
"learning_rate": 2.6047732402910485e-05,
|
|
"loss": 1.5204573631286622,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.57432,
|
|
"step": 3220,
|
|
"tokens/total": 21141504,
|
|
"tokens/train_per_sec_per_gpu": 43.84,
|
|
"tokens/trainable": 13613171
|
|
},
|
|
{
|
|
"epoch": 1.0074865476097636,
|
|
"grad_norm": 0.13766086101531982,
|
|
"learning_rate": 2.592156071224133e-05,
|
|
"loss": 1.6278236389160157,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.09278,
|
|
"step": 3230,
|
|
"tokens/total": 21205888,
|
|
"tokens/train_per_sec_per_gpu": 66.01,
|
|
"tokens/trainable": 13654112
|
|
},
|
|
{
|
|
"epoch": 1.0106059424471652,
|
|
"grad_norm": 0.1572219431400299,
|
|
"learning_rate": 2.579536551214732e-05,
|
|
"loss": 1.5664448738098145,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78959,
|
|
"step": 3240,
|
|
"tokens/total": 21271552,
|
|
"tokens/train_per_sec_per_gpu": 64.88,
|
|
"tokens/trainable": 13697298
|
|
},
|
|
{
|
|
"epoch": 1.0137253372845667,
|
|
"grad_norm": 0.14219284057617188,
|
|
"learning_rate": 2.5669150021924297e-05,
|
|
"loss": 1.5632357597351074,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.77424,
|
|
"step": 3250,
|
|
"tokens/total": 21338368,
|
|
"tokens/train_per_sec_per_gpu": 61.11,
|
|
"tokens/trainable": 13741557
|
|
},
|
|
{
|
|
"epoch": 1.0168447321219682,
|
|
"grad_norm": 0.15051282942295074,
|
|
"learning_rate": 2.554291746138573e-05,
|
|
"loss": 1.6328500747680663,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.11844,
|
|
"step": 3260,
|
|
"tokens/total": 21404288,
|
|
"tokens/train_per_sec_per_gpu": 79.47,
|
|
"tokens/trainable": 13784290
|
|
},
|
|
{
|
|
"epoch": 1.0199641269593698,
|
|
"grad_norm": 0.16105814278125763,
|
|
"learning_rate": 2.5416671050780534e-05,
|
|
"loss": 1.5128487586975097,
|
|
"memory/device_reserved (GiB)": 4.31,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.53964,
|
|
"step": 3270,
|
|
"tokens/total": 21468032,
|
|
"tokens/train_per_sec_per_gpu": 69.3,
|
|
"tokens/trainable": 13825742
|
|
},
|
|
{
|
|
"epoch": 1.0230835217967715,
|
|
"grad_norm": 0.15369705855846405,
|
|
"learning_rate": 2.529041401071097e-05,
|
|
"loss": 1.6480125427246093,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.19664,
|
|
"step": 3280,
|
|
"tokens/total": 21533440,
|
|
"tokens/train_per_sec_per_gpu": 83.12,
|
|
"tokens/trainable": 13867230
|
|
},
|
|
{
|
|
"epoch": 1.026202916634173,
|
|
"grad_norm": 0.14982105791568756,
|
|
"learning_rate": 2.516414956205045e-05,
|
|
"loss": 1.524454402923584,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.59264,
|
|
"step": 3290,
|
|
"tokens/total": 21598592,
|
|
"tokens/train_per_sec_per_gpu": 28.15,
|
|
"tokens/trainable": 13910066
|
|
},
|
|
{
|
|
"epoch": 1.0293223114715746,
|
|
"grad_norm": 0.15444625914096832,
|
|
"learning_rate": 2.5037880925861372e-05,
|
|
"loss": 1.62322998046875,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.06944,
|
|
"step": 3300,
|
|
"tokens/total": 21666048,
|
|
"tokens/train_per_sec_per_gpu": 60.75,
|
|
"tokens/trainable": 13952392
|
|
},
|
|
{
|
|
"epoch": 1.0324417063089761,
|
|
"grad_norm": 0.1340450644493103,
|
|
"learning_rate": 2.4911611323312975e-05,
|
|
"loss": 1.5519920349121095,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.72086,
|
|
"step": 3310,
|
|
"tokens/total": 21734144,
|
|
"tokens/train_per_sec_per_gpu": 81.4,
|
|
"tokens/trainable": 13995768
|
|
},
|
|
{
|
|
"epoch": 1.0355611011463777,
|
|
"grad_norm": 0.14638938009738922,
|
|
"learning_rate": 2.4785343975599154e-05,
|
|
"loss": 1.608949851989746,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.99756,
|
|
"step": 3320,
|
|
"tokens/total": 21798144,
|
|
"tokens/train_per_sec_per_gpu": 65.31,
|
|
"tokens/trainable": 14036210
|
|
},
|
|
{
|
|
"epoch": 1.0386804959837792,
|
|
"grad_norm": 0.1542874276638031,
|
|
"learning_rate": 2.4659082103856268e-05,
|
|
"loss": 1.6571678161621093,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.24444,
|
|
"step": 3330,
|
|
"tokens/total": 21864192,
|
|
"tokens/train_per_sec_per_gpu": 70.94,
|
|
"tokens/trainable": 14078893
|
|
},
|
|
{
|
|
"epoch": 1.0417998908211807,
|
|
"grad_norm": 0.1390984207391739,
|
|
"learning_rate": 2.4532828929080987e-05,
|
|
"loss": 1.6411628723144531,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.16117,
|
|
"step": 3340,
|
|
"tokens/total": 21931136,
|
|
"tokens/train_per_sec_per_gpu": 65.66,
|
|
"tokens/trainable": 14119723
|
|
},
|
|
{
|
|
"epoch": 1.0449192856585823,
|
|
"grad_norm": 0.1401313990354538,
|
|
"learning_rate": 2.440658767204812e-05,
|
|
"loss": 1.5741381645202637,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.82658,
|
|
"step": 3350,
|
|
"tokens/total": 21998464,
|
|
"tokens/train_per_sec_per_gpu": 61.07,
|
|
"tokens/trainable": 14164541
|
|
},
|
|
{
|
|
"epoch": 1.0480386804959838,
|
|
"grad_norm": 0.1524425745010376,
|
|
"learning_rate": 2.4280361553228435e-05,
|
|
"loss": 1.6065494537353515,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.98558,
|
|
"step": 3360,
|
|
"tokens/total": 22064896,
|
|
"tokens/train_per_sec_per_gpu": 96.84,
|
|
"tokens/trainable": 14207280
|
|
},
|
|
{
|
|
"epoch": 1.0511580753333853,
|
|
"grad_norm": 0.15332253277301788,
|
|
"learning_rate": 2.4154153792706556e-05,
|
|
"loss": 1.6758455276489257,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.34331,
|
|
"step": 3370,
|
|
"tokens/total": 22127744,
|
|
"tokens/train_per_sec_per_gpu": 61.99,
|
|
"tokens/trainable": 14247959
|
|
},
|
|
{
|
|
"epoch": 1.0542774701707869,
|
|
"grad_norm": 0.16406677663326263,
|
|
"learning_rate": 2.402796761009873e-05,
|
|
"loss": 1.5803353309631347,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.85658,
|
|
"step": 3380,
|
|
"tokens/total": 22194304,
|
|
"tokens/train_per_sec_per_gpu": 34.52,
|
|
"tokens/trainable": 14291430
|
|
},
|
|
{
|
|
"epoch": 1.0573968650081884,
|
|
"grad_norm": 0.15428400039672852,
|
|
"learning_rate": 2.3901806224470773e-05,
|
|
"loss": 1.6375354766845702,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.14248,
|
|
"step": 3390,
|
|
"tokens/total": 22259072,
|
|
"tokens/train_per_sec_per_gpu": 63.01,
|
|
"tokens/trainable": 14333671
|
|
},
|
|
{
|
|
"epoch": 1.06051625984559,
|
|
"grad_norm": 0.14430588483810425,
|
|
"learning_rate": 2.3775672854255906e-05,
|
|
"loss": 1.655337142944336,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.23484,
|
|
"step": 3400,
|
|
"tokens/total": 22323968,
|
|
"tokens/train_per_sec_per_gpu": 85.3,
|
|
"tokens/trainable": 14374401
|
|
},
|
|
{
|
|
"epoch": 1.0636356546829915,
|
|
"grad_norm": 0.16545863449573517,
|
|
"learning_rate": 2.364957071717267e-05,
|
|
"loss": 1.6105226516723632,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.00543,
|
|
"step": 3410,
|
|
"tokens/total": 22390528,
|
|
"tokens/train_per_sec_per_gpu": 54.4,
|
|
"tokens/trainable": 14416359
|
|
},
|
|
{
|
|
"epoch": 1.066755049520393,
|
|
"grad_norm": 0.15084774792194366,
|
|
"learning_rate": 2.3523503030142834e-05,
|
|
"loss": 1.597851276397705,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.9424,
|
|
"step": 3420,
|
|
"tokens/total": 22456576,
|
|
"tokens/train_per_sec_per_gpu": 64.13,
|
|
"tokens/trainable": 14458253
|
|
},
|
|
{
|
|
"epoch": 1.0698744443577946,
|
|
"grad_norm": 0.1472393423318863,
|
|
"learning_rate": 2.3397473009209315e-05,
|
|
"loss": 1.5513733863830566,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.71795,
|
|
"step": 3430,
|
|
"tokens/total": 22524160,
|
|
"tokens/train_per_sec_per_gpu": 52.36,
|
|
"tokens/trainable": 14502069
|
|
},
|
|
{
|
|
"epoch": 1.072993839195196,
|
|
"grad_norm": 0.14517679810523987,
|
|
"learning_rate": 2.327148386945419e-05,
|
|
"loss": 1.648844337463379,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.20097,
|
|
"step": 3440,
|
|
"tokens/total": 22591104,
|
|
"tokens/train_per_sec_per_gpu": 70.16,
|
|
"tokens/trainable": 14544770
|
|
},
|
|
{
|
|
"epoch": 1.0761132340325976,
|
|
"grad_norm": 0.15006546676158905,
|
|
"learning_rate": 2.3145538824916606e-05,
|
|
"loss": 1.519717311859131,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.57093,
|
|
"step": 3450,
|
|
"tokens/total": 22658432,
|
|
"tokens/train_per_sec_per_gpu": 37.44,
|
|
"tokens/trainable": 14587771
|
|
},
|
|
{
|
|
"epoch": 1.0792326288699992,
|
|
"grad_norm": 0.13988368213176727,
|
|
"learning_rate": 2.301964108851083e-05,
|
|
"loss": 1.5446808815002442,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68648,
|
|
"step": 3460,
|
|
"tokens/total": 22724736,
|
|
"tokens/train_per_sec_per_gpu": 73.76,
|
|
"tokens/trainable": 14631836
|
|
},
|
|
{
|
|
"epoch": 1.0823520237074007,
|
|
"grad_norm": 0.15881726145744324,
|
|
"learning_rate": 2.2893793871944276e-05,
|
|
"loss": 1.6225914001464843,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.0662,
|
|
"step": 3470,
|
|
"tokens/total": 22790144,
|
|
"tokens/train_per_sec_per_gpu": 58.86,
|
|
"tokens/trainable": 14672578
|
|
},
|
|
{
|
|
"epoch": 1.0854714185448022,
|
|
"grad_norm": 0.16667784750461578,
|
|
"learning_rate": 2.276800038563557e-05,
|
|
"loss": 1.5335986137390136,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.63483,
|
|
"step": 3480,
|
|
"tokens/total": 22855808,
|
|
"tokens/train_per_sec_per_gpu": 64.38,
|
|
"tokens/trainable": 14716090
|
|
},
|
|
{
|
|
"epoch": 1.0885908133822038,
|
|
"grad_norm": 0.15708185732364655,
|
|
"learning_rate": 2.2642263838632675e-05,
|
|
"loss": 1.6635202407836913,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.27786,
|
|
"step": 3490,
|
|
"tokens/total": 22918912,
|
|
"tokens/train_per_sec_per_gpu": 84.79,
|
|
"tokens/trainable": 14757159
|
|
},
|
|
{
|
|
"epoch": 1.0917102082196053,
|
|
"grad_norm": 0.1565551608800888,
|
|
"learning_rate": 2.2516587438530975e-05,
|
|
"loss": 1.594503402709961,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.92588,
|
|
"step": 3500,
|
|
"tokens/total": 22984960,
|
|
"tokens/train_per_sec_per_gpu": 117.27,
|
|
"tokens/trainable": 14800159
|
|
},
|
|
{
|
|
"epoch": 1.0948296030570068,
|
|
"grad_norm": 0.14390283823013306,
|
|
"learning_rate": 2.2390974391391518e-05,
|
|
"loss": 1.4754847526550292,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.37316,
|
|
"step": 3510,
|
|
"tokens/total": 23050368,
|
|
"tokens/train_per_sec_per_gpu": 65.55,
|
|
"tokens/trainable": 14842010
|
|
},
|
|
{
|
|
"epoch": 1.0979489978944086,
|
|
"grad_norm": 0.16479136049747467,
|
|
"learning_rate": 2.2265427901659155e-05,
|
|
"loss": 1.6057992935180665,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.98184,
|
|
"step": 3520,
|
|
"tokens/total": 23116672,
|
|
"tokens/train_per_sec_per_gpu": 60.82,
|
|
"tokens/trainable": 14884889
|
|
},
|
|
{
|
|
"epoch": 1.1010683927318101,
|
|
"grad_norm": 0.17615869641304016,
|
|
"learning_rate": 2.2139951172080846e-05,
|
|
"loss": 1.5619976997375489,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.76834,
|
|
"step": 3530,
|
|
"tokens/total": 23180800,
|
|
"tokens/train_per_sec_per_gpu": 52.83,
|
|
"tokens/trainable": 14925037
|
|
},
|
|
{
|
|
"epoch": 1.1041877875692117,
|
|
"grad_norm": 0.15070563554763794,
|
|
"learning_rate": 2.201454740362394e-05,
|
|
"loss": 1.582502555847168,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.86712,
|
|
"step": 3540,
|
|
"tokens/total": 23246976,
|
|
"tokens/train_per_sec_per_gpu": 75.52,
|
|
"tokens/trainable": 14967428
|
|
},
|
|
{
|
|
"epoch": 1.1073071824066132,
|
|
"grad_norm": 0.16347666084766388,
|
|
"learning_rate": 2.1889219795394504e-05,
|
|
"loss": 1.6863239288330079,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.39959,
|
|
"step": 3550,
|
|
"tokens/total": 23310720,
|
|
"tokens/train_per_sec_per_gpu": 60.19,
|
|
"tokens/trainable": 15009663
|
|
},
|
|
{
|
|
"epoch": 1.1104265772440147,
|
|
"grad_norm": 0.1451273113489151,
|
|
"learning_rate": 2.176397154455573e-05,
|
|
"loss": 1.5806756973266602,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.85824,
|
|
"step": 3560,
|
|
"tokens/total": 23377152,
|
|
"tokens/train_per_sec_per_gpu": 75.65,
|
|
"tokens/trainable": 15052369
|
|
},
|
|
{
|
|
"epoch": 1.1135459720814163,
|
|
"grad_norm": 0.14389927685260773,
|
|
"learning_rate": 2.1638805846246366e-05,
|
|
"loss": 1.5433822631835938,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68039,
|
|
"step": 3570,
|
|
"tokens/total": 23442304,
|
|
"tokens/train_per_sec_per_gpu": 71.6,
|
|
"tokens/trainable": 15094963
|
|
},
|
|
{
|
|
"epoch": 1.1166653669188178,
|
|
"grad_norm": 0.14995242655277252,
|
|
"learning_rate": 2.1513725893499215e-05,
|
|
"loss": 1.634136962890625,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.12503,
|
|
"step": 3580,
|
|
"tokens/total": 23506816,
|
|
"tokens/train_per_sec_per_gpu": 60.17,
|
|
"tokens/trainable": 15136277
|
|
},
|
|
{
|
|
"epoch": 1.1197847617562193,
|
|
"grad_norm": 0.14957784116268158,
|
|
"learning_rate": 2.138873487715966e-05,
|
|
"loss": 1.5991394996643067,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.94877,
|
|
"step": 3590,
|
|
"tokens/total": 23571584,
|
|
"tokens/train_per_sec_per_gpu": 69.69,
|
|
"tokens/trainable": 15177478
|
|
},
|
|
{
|
|
"epoch": 1.1229041565936209,
|
|
"grad_norm": 0.1398400366306305,
|
|
"learning_rate": 2.126383598580429e-05,
|
|
"loss": 1.569289207458496,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.80323,
|
|
"step": 3600,
|
|
"tokens/total": 23637376,
|
|
"tokens/train_per_sec_per_gpu": 43.02,
|
|
"tokens/trainable": 15220004
|
|
},
|
|
{
|
|
"epoch": 1.1260235514310224,
|
|
"grad_norm": 0.15978693962097168,
|
|
"learning_rate": 2.113903240565953e-05,
|
|
"loss": 1.5500237464904785,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.71158,
|
|
"step": 3610,
|
|
"tokens/total": 23700992,
|
|
"tokens/train_per_sec_per_gpu": 76.72,
|
|
"tokens/trainable": 15259908
|
|
},
|
|
{
|
|
"epoch": 1.129142946268424,
|
|
"grad_norm": 0.1460128277540207,
|
|
"learning_rate": 2.101432732052039e-05,
|
|
"loss": 1.6341989517211915,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.12535,
|
|
"step": 3620,
|
|
"tokens/total": 23762944,
|
|
"tokens/train_per_sec_per_gpu": 85.84,
|
|
"tokens/trainable": 15301318
|
|
},
|
|
{
|
|
"epoch": 1.1322623411058255,
|
|
"grad_norm": 0.14776496589183807,
|
|
"learning_rate": 2.0889723911669215e-05,
|
|
"loss": 1.5522699356079102,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.72218,
|
|
"step": 3630,
|
|
"tokens/total": 23828480,
|
|
"tokens/train_per_sec_per_gpu": 41.07,
|
|
"tokens/trainable": 15341405
|
|
},
|
|
{
|
|
"epoch": 1.135381735943227,
|
|
"grad_norm": 0.15818066895008087,
|
|
"learning_rate": 2.0765225357794534e-05,
|
|
"loss": 1.6373224258422852,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.14138,
|
|
"step": 3640,
|
|
"tokens/total": 23891328,
|
|
"tokens/train_per_sec_per_gpu": 51.57,
|
|
"tokens/trainable": 15381903
|
|
},
|
|
{
|
|
"epoch": 1.1385011307806285,
|
|
"grad_norm": 0.13447289168834686,
|
|
"learning_rate": 2.064083483491002e-05,
|
|
"loss": 1.531868076324463,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.62681,
|
|
"step": 3650,
|
|
"tokens/total": 23957376,
|
|
"tokens/train_per_sec_per_gpu": 77.64,
|
|
"tokens/trainable": 15425600
|
|
},
|
|
{
|
|
"epoch": 1.14162052561803,
|
|
"grad_norm": 0.1304033398628235,
|
|
"learning_rate": 2.0516555516273384e-05,
|
|
"loss": 1.4499412536621095,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.26286,
|
|
"step": 3660,
|
|
"tokens/total": 24023552,
|
|
"tokens/train_per_sec_per_gpu": 74.91,
|
|
"tokens/trainable": 15467852
|
|
},
|
|
{
|
|
"epoch": 1.1447399204554316,
|
|
"grad_norm": 0.13597141206264496,
|
|
"learning_rate": 2.0392390572305482e-05,
|
|
"loss": 1.5855513572692872,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.88198,
|
|
"step": 3670,
|
|
"tokens/total": 24089856,
|
|
"tokens/train_per_sec_per_gpu": 57.07,
|
|
"tokens/trainable": 15510683
|
|
},
|
|
{
|
|
"epoch": 1.1478593152928331,
|
|
"grad_norm": 0.14659976959228516,
|
|
"learning_rate": 2.0268343170509428e-05,
|
|
"loss": 1.6490968704223632,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.20228,
|
|
"step": 3680,
|
|
"tokens/total": 24156544,
|
|
"tokens/train_per_sec_per_gpu": 89.84,
|
|
"tokens/trainable": 15553914
|
|
},
|
|
{
|
|
"epoch": 1.1509787101302347,
|
|
"grad_norm": 0.1268371343612671,
|
|
"learning_rate": 2.0144416475389764e-05,
|
|
"loss": 1.4978747367858887,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.47217,
|
|
"step": 3690,
|
|
"tokens/total": 24223360,
|
|
"tokens/train_per_sec_per_gpu": 91.38,
|
|
"tokens/trainable": 15597478
|
|
},
|
|
{
|
|
"epoch": 1.1540981049676362,
|
|
"grad_norm": 0.1465367078781128,
|
|
"learning_rate": 2.0020613648371776e-05,
|
|
"loss": 1.58581600189209,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.88327,
|
|
"step": 3700,
|
|
"tokens/total": 24286976,
|
|
"tokens/train_per_sec_per_gpu": 73.67,
|
|
"tokens/trainable": 15640462
|
|
},
|
|
{
|
|
"epoch": 1.1572174998050377,
|
|
"grad_norm": 0.1302720606327057,
|
|
"learning_rate": 1.9896937847720796e-05,
|
|
"loss": 1.5498015403747558,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.71054,
|
|
"step": 3710,
|
|
"tokens/total": 24349824,
|
|
"tokens/train_per_sec_per_gpu": 74.33,
|
|
"tokens/trainable": 15680393
|
|
},
|
|
{
|
|
"epoch": 1.1603368946424393,
|
|
"grad_norm": 0.14845749735832214,
|
|
"learning_rate": 1.9773392228461677e-05,
|
|
"loss": 1.5675559997558595,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.79492,
|
|
"step": 3720,
|
|
"tokens/total": 24418048,
|
|
"tokens/train_per_sec_per_gpu": 63.29,
|
|
"tokens/trainable": 15723419
|
|
},
|
|
{
|
|
"epoch": 1.163456289479841,
|
|
"grad_norm": 0.13909345865249634,
|
|
"learning_rate": 1.9649979942298274e-05,
|
|
"loss": 1.5458148956298827,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.69179,
|
|
"step": 3730,
|
|
"tokens/total": 24484224,
|
|
"tokens/train_per_sec_per_gpu": 103.45,
|
|
"tokens/trainable": 15767046
|
|
},
|
|
{
|
|
"epoch": 1.1665756843172423,
|
|
"grad_norm": 0.14253029227256775,
|
|
"learning_rate": 1.9526704137533056e-05,
|
|
"loss": 1.609918975830078,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.00241,
|
|
"step": 3740,
|
|
"tokens/total": 24550656,
|
|
"tokens/train_per_sec_per_gpu": 27.32,
|
|
"tokens/trainable": 15811257
|
|
},
|
|
{
|
|
"epoch": 1.169695079154644,
|
|
"grad_norm": 0.14793558418750763,
|
|
"learning_rate": 1.9403567958986794e-05,
|
|
"loss": 1.553643035888672,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.72867,
|
|
"step": 3750,
|
|
"tokens/total": 24615296,
|
|
"tokens/train_per_sec_per_gpu": 53.84,
|
|
"tokens/trainable": 15852725
|
|
},
|
|
{
|
|
"epoch": 1.1728144739920456,
|
|
"grad_norm": 0.1383708119392395,
|
|
"learning_rate": 1.9280574547918327e-05,
|
|
"loss": 1.5879432678222656,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.89367,
|
|
"step": 3760,
|
|
"tokens/total": 24683264,
|
|
"tokens/train_per_sec_per_gpu": 57.66,
|
|
"tokens/trainable": 15895386
|
|
},
|
|
{
|
|
"epoch": 1.1759338688294472,
|
|
"grad_norm": 0.15714044868946075,
|
|
"learning_rate": 1.9157727041944447e-05,
|
|
"loss": 1.5135435104370116,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.5428,
|
|
"step": 3770,
|
|
"tokens/total": 24749952,
|
|
"tokens/train_per_sec_per_gpu": 45.69,
|
|
"tokens/trainable": 15937120
|
|
},
|
|
{
|
|
"epoch": 1.1790532636668487,
|
|
"grad_norm": 0.15041106939315796,
|
|
"learning_rate": 1.903502857495981e-05,
|
|
"loss": 1.5964231491088867,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.93535,
|
|
"step": 3780,
|
|
"tokens/total": 24813312,
|
|
"tokens/train_per_sec_per_gpu": 96.06,
|
|
"tokens/trainable": 15978153
|
|
},
|
|
{
|
|
"epoch": 1.1821726585042502,
|
|
"grad_norm": 0.1377926468849182,
|
|
"learning_rate": 1.8912482277057067e-05,
|
|
"loss": 1.5159405708312987,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.5537,
|
|
"step": 3790,
|
|
"tokens/total": 24880768,
|
|
"tokens/train_per_sec_per_gpu": 70.36,
|
|
"tokens/trainable": 16022005
|
|
},
|
|
{
|
|
"epoch": 1.1852920533416518,
|
|
"grad_norm": 0.14477963745594025,
|
|
"learning_rate": 1.879009127444693e-05,
|
|
"loss": 1.5720962524414062,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.81673,
|
|
"step": 3800,
|
|
"tokens/total": 24948096,
|
|
"tokens/train_per_sec_per_gpu": 78.04,
|
|
"tokens/trainable": 16064529
|
|
},
|
|
{
|
|
"epoch": 1.1884114481790533,
|
|
"grad_norm": 0.1461605578660965,
|
|
"learning_rate": 1.8667858689378465e-05,
|
|
"loss": 1.5836588859558105,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.87275,
|
|
"step": 3810,
|
|
"tokens/total": 25013632,
|
|
"tokens/train_per_sec_per_gpu": 94.63,
|
|
"tokens/trainable": 16106728
|
|
},
|
|
{
|
|
"epoch": 1.1915308430164548,
|
|
"grad_norm": 0.13886338472366333,
|
|
"learning_rate": 1.8545787640059453e-05,
|
|
"loss": 1.579399585723877,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.85204,
|
|
"step": 3820,
|
|
"tokens/total": 25078912,
|
|
"tokens/train_per_sec_per_gpu": 44.83,
|
|
"tokens/trainable": 16149584
|
|
},
|
|
{
|
|
"epoch": 1.1946502378538564,
|
|
"grad_norm": 0.14671356976032257,
|
|
"learning_rate": 1.842388124057681e-05,
|
|
"loss": 1.5195475578308106,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.57016,
|
|
"step": 3830,
|
|
"tokens/total": 25147264,
|
|
"tokens/train_per_sec_per_gpu": 64.03,
|
|
"tokens/trainable": 16194849
|
|
},
|
|
{
|
|
"epoch": 1.197769632691258,
|
|
"grad_norm": 0.13302017748355865,
|
|
"learning_rate": 1.8302142600817167e-05,
|
|
"loss": 1.5307424545288086,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.62161,
|
|
"step": 3840,
|
|
"tokens/total": 25212544,
|
|
"tokens/train_per_sec_per_gpu": 96.37,
|
|
"tokens/trainable": 16237869
|
|
},
|
|
{
|
|
"epoch": 1.2008890275286594,
|
|
"grad_norm": 0.15138085186481476,
|
|
"learning_rate": 1.8180574826387518e-05,
|
|
"loss": 1.6239282608032226,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.07298,
|
|
"step": 3850,
|
|
"tokens/total": 25280000,
|
|
"tokens/train_per_sec_per_gpu": 59.15,
|
|
"tokens/trainable": 16279840
|
|
},
|
|
{
|
|
"epoch": 1.204008422366061,
|
|
"grad_norm": 0.12642429769039154,
|
|
"learning_rate": 1.8059181018536038e-05,
|
|
"loss": 1.546940517425537,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.69708,
|
|
"step": 3860,
|
|
"tokens/total": 25344512,
|
|
"tokens/train_per_sec_per_gpu": 79.08,
|
|
"tokens/trainable": 16322195
|
|
},
|
|
{
|
|
"epoch": 1.2071278172034625,
|
|
"grad_norm": 0.14075028896331787,
|
|
"learning_rate": 1.7937964274072904e-05,
|
|
"loss": 1.5609694480895997,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.76344,
|
|
"step": 3870,
|
|
"tokens/total": 25409152,
|
|
"tokens/train_per_sec_per_gpu": 80.64,
|
|
"tokens/trainable": 16363261
|
|
},
|
|
{
|
|
"epoch": 1.210247212040864,
|
|
"grad_norm": 0.143815815448761,
|
|
"learning_rate": 1.7816927685291345e-05,
|
|
"loss": 1.5783745765686035,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.84707,
|
|
"step": 3880,
|
|
"tokens/total": 25474048,
|
|
"tokens/train_per_sec_per_gpu": 55.53,
|
|
"tokens/trainable": 16404166
|
|
},
|
|
{
|
|
"epoch": 1.2133666068782656,
|
|
"grad_norm": 0.14737196266651154,
|
|
"learning_rate": 1.7696074339888728e-05,
|
|
"loss": 1.5035325050354005,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.49755,
|
|
"step": 3890,
|
|
"tokens/total": 25539840,
|
|
"tokens/train_per_sec_per_gpu": 81.33,
|
|
"tokens/trainable": 16445498
|
|
},
|
|
{
|
|
"epoch": 1.2164860017156671,
|
|
"grad_norm": 0.14170339703559875,
|
|
"learning_rate": 1.7575407320887807e-05,
|
|
"loss": 1.565513515472412,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78513,
|
|
"step": 3900,
|
|
"tokens/total": 25603712,
|
|
"tokens/train_per_sec_per_gpu": 54.79,
|
|
"tokens/trainable": 16486278
|
|
},
|
|
{
|
|
"epoch": 1.2196053965530687,
|
|
"grad_norm": 0.15115408599376678,
|
|
"learning_rate": 1.7454929706558056e-05,
|
|
"loss": 1.60615234375,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.9836,
|
|
"step": 3910,
|
|
"tokens/total": 25669632,
|
|
"tokens/train_per_sec_per_gpu": 72.83,
|
|
"tokens/trainable": 16528993
|
|
},
|
|
{
|
|
"epoch": 1.2227247913904702,
|
|
"grad_norm": 0.1421128809452057,
|
|
"learning_rate": 1.7334644570337155e-05,
|
|
"loss": 1.6260467529296876,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.08374,
|
|
"step": 3920,
|
|
"tokens/total": 25735936,
|
|
"tokens/train_per_sec_per_gpu": 97.4,
|
|
"tokens/trainable": 16572366
|
|
},
|
|
{
|
|
"epoch": 1.2258441862278717,
|
|
"grad_norm": 0.1421615034341812,
|
|
"learning_rate": 1.7214554980752602e-05,
|
|
"loss": 1.598159408569336,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.94392,
|
|
"step": 3930,
|
|
"tokens/total": 25800704,
|
|
"tokens/train_per_sec_per_gpu": 98.41,
|
|
"tokens/trainable": 16612629
|
|
},
|
|
{
|
|
"epoch": 1.2289635810652733,
|
|
"grad_norm": 0.1387406289577484,
|
|
"learning_rate": 1.709466400134339e-05,
|
|
"loss": 1.5272364616394043,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.60543,
|
|
"step": 3940,
|
|
"tokens/total": 25868800,
|
|
"tokens/train_per_sec_per_gpu": 72.76,
|
|
"tokens/trainable": 16656915
|
|
},
|
|
{
|
|
"epoch": 1.2320829759026748,
|
|
"grad_norm": 0.15196749567985535,
|
|
"learning_rate": 1.697497469058188e-05,
|
|
"loss": 1.5816500663757325,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.86297,
|
|
"step": 3950,
|
|
"tokens/total": 25934336,
|
|
"tokens/train_per_sec_per_gpu": 83.55,
|
|
"tokens/trainable": 16698858
|
|
},
|
|
{
|
|
"epoch": 1.2352023707400765,
|
|
"grad_norm": 0.16110177338123322,
|
|
"learning_rate": 1.6855490101795783e-05,
|
|
"loss": 1.6142173767089845,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.02395,
|
|
"step": 3960,
|
|
"tokens/total": 26000000,
|
|
"tokens/train_per_sec_per_gpu": 53.45,
|
|
"tokens/trainable": 16739874
|
|
},
|
|
{
|
|
"epoch": 1.2383217655774779,
|
|
"grad_norm": 0.14961859583854675,
|
|
"learning_rate": 1.673621328309026e-05,
|
|
"loss": 1.5786381721496583,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.84835,
|
|
"step": 3970,
|
|
"tokens/total": 26065024,
|
|
"tokens/train_per_sec_per_gpu": 74.97,
|
|
"tokens/trainable": 16782780
|
|
},
|
|
{
|
|
"epoch": 1.2414411604148796,
|
|
"grad_norm": 0.13897107541561127,
|
|
"learning_rate": 1.6617147277270156e-05,
|
|
"loss": 1.5589570999145508,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.75386,
|
|
"step": 3980,
|
|
"tokens/total": 26133504,
|
|
"tokens/train_per_sec_per_gpu": 63.14,
|
|
"tokens/trainable": 16826992
|
|
},
|
|
{
|
|
"epoch": 1.2445605552522812,
|
|
"grad_norm": 0.14571769535541534,
|
|
"learning_rate": 1.6498295121762393e-05,
|
|
"loss": 1.5854491233825683,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.88148,
|
|
"step": 3990,
|
|
"tokens/total": 26198656,
|
|
"tokens/train_per_sec_per_gpu": 85.82,
|
|
"tokens/trainable": 16869594
|
|
},
|
|
{
|
|
"epoch": 1.2476799500896827,
|
|
"grad_norm": 0.13441121578216553,
|
|
"learning_rate": 1.6379659848538484e-05,
|
|
"loss": 1.5480908393859862,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.70248,
|
|
"step": 4000,
|
|
"tokens/total": 26260480,
|
|
"tokens/train_per_sec_per_gpu": 69.83,
|
|
"tokens/trainable": 16908262
|
|
},
|
|
{
|
|
"epoch": 1.2507993449270842,
|
|
"grad_norm": 0.16603824496269226,
|
|
"learning_rate": 1.6261244484037164e-05,
|
|
"loss": 1.64593505859375,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.18586,
|
|
"step": 4010,
|
|
"tokens/total": 26327808,
|
|
"tokens/train_per_sec_per_gpu": 51.11,
|
|
"tokens/trainable": 16951844
|
|
},
|
|
{
|
|
"epoch": 1.2539187397644858,
|
|
"grad_norm": 0.1614803522825241,
|
|
"learning_rate": 1.61430520490872e-05,
|
|
"loss": 1.6892860412597657,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.41561,
|
|
"step": 4020,
|
|
"tokens/total": 26393088,
|
|
"tokens/train_per_sec_per_gpu": 95.7,
|
|
"tokens/trainable": 16994106
|
|
},
|
|
{
|
|
"epoch": 1.2570381346018873,
|
|
"grad_norm": 0.1688452661037445,
|
|
"learning_rate": 1.602508555883033e-05,
|
|
"loss": 1.6136943817138671,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.02133,
|
|
"step": 4030,
|
|
"tokens/total": 26456576,
|
|
"tokens/train_per_sec_per_gpu": 47.05,
|
|
"tokens/trainable": 17035848
|
|
},
|
|
{
|
|
"epoch": 1.2601575294392888,
|
|
"grad_norm": 0.14532530307769775,
|
|
"learning_rate": 1.5907348022644352e-05,
|
|
"loss": 1.556704330444336,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.74316,
|
|
"step": 4040,
|
|
"tokens/total": 26521472,
|
|
"tokens/train_per_sec_per_gpu": 76.61,
|
|
"tokens/trainable": 17076200
|
|
},
|
|
{
|
|
"epoch": 1.2632769242766904,
|
|
"grad_norm": 0.15120644867420197,
|
|
"learning_rate": 1.5789842444066317e-05,
|
|
"loss": 1.5015097618103028,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.48846,
|
|
"step": 4050,
|
|
"tokens/total": 26586240,
|
|
"tokens/train_per_sec_per_gpu": 70.33,
|
|
"tokens/trainable": 17116216
|
|
},
|
|
{
|
|
"epoch": 1.266396319114092,
|
|
"grad_norm": 0.14386820793151855,
|
|
"learning_rate": 1.567257182071597e-05,
|
|
"loss": 1.5772655487060547,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.8417,
|
|
"step": 4060,
|
|
"tokens/total": 26653440,
|
|
"tokens/train_per_sec_per_gpu": 63.21,
|
|
"tokens/trainable": 17159752
|
|
},
|
|
{
|
|
"epoch": 1.2695157139514934,
|
|
"grad_norm": 0.156844362616539,
|
|
"learning_rate": 1.5555539144219218e-05,
|
|
"loss": 1.6218643188476562,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.06252,
|
|
"step": 4070,
|
|
"tokens/total": 26720512,
|
|
"tokens/train_per_sec_per_gpu": 91.84,
|
|
"tokens/trainable": 17202776
|
|
},
|
|
{
|
|
"epoch": 1.272635108788895,
|
|
"grad_norm": 0.1610804945230484,
|
|
"learning_rate": 1.5438747400131836e-05,
|
|
"loss": 1.605463981628418,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.98017,
|
|
"step": 4080,
|
|
"tokens/total": 26787840,
|
|
"tokens/train_per_sec_per_gpu": 85.4,
|
|
"tokens/trainable": 17245528
|
|
},
|
|
{
|
|
"epoch": 1.2757545036262965,
|
|
"grad_norm": 0.16594064235687256,
|
|
"learning_rate": 1.5322199567863334e-05,
|
|
"loss": 1.5147664070129394,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.54836,
|
|
"step": 4090,
|
|
"tokens/total": 26851456,
|
|
"tokens/train_per_sec_per_gpu": 86.34,
|
|
"tokens/trainable": 17287186
|
|
},
|
|
{
|
|
"epoch": 1.278873898463698,
|
|
"grad_norm": 0.14431948959827423,
|
|
"learning_rate": 1.5205898620600872e-05,
|
|
"loss": 1.570528221130371,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.80919,
|
|
"step": 4100,
|
|
"tokens/total": 26917632,
|
|
"tokens/train_per_sec_per_gpu": 62.77,
|
|
"tokens/trainable": 17329868
|
|
},
|
|
{
|
|
"epoch": 1.2819932933010996,
|
|
"grad_norm": 0.1509508639574051,
|
|
"learning_rate": 1.5089847525233517e-05,
|
|
"loss": 1.634622573852539,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.12752,
|
|
"step": 4110,
|
|
"tokens/total": 26983552,
|
|
"tokens/train_per_sec_per_gpu": 70.66,
|
|
"tokens/trainable": 17372492
|
|
},
|
|
{
|
|
"epoch": 1.285112688138501,
|
|
"grad_norm": 0.1630357950925827,
|
|
"learning_rate": 1.4974049242276458e-05,
|
|
"loss": 1.5631893157958985,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.77402,
|
|
"step": 4120,
|
|
"tokens/total": 27047040,
|
|
"tokens/train_per_sec_per_gpu": 64.92,
|
|
"tokens/trainable": 17414728
|
|
},
|
|
{
|
|
"epoch": 1.2882320829759026,
|
|
"grad_norm": 0.14399567246437073,
|
|
"learning_rate": 1.4858506725795573e-05,
|
|
"loss": 1.4722051620483398,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.35884,
|
|
"step": 4130,
|
|
"tokens/total": 27113984,
|
|
"tokens/train_per_sec_per_gpu": 37.91,
|
|
"tokens/trainable": 17456692
|
|
},
|
|
{
|
|
"epoch": 1.2913514778133042,
|
|
"grad_norm": 0.16126224398612976,
|
|
"learning_rate": 1.4743222923331986e-05,
|
|
"loss": 1.5545909881591797,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73315,
|
|
"step": 4140,
|
|
"tokens/total": 27180544,
|
|
"tokens/train_per_sec_per_gpu": 66.31,
|
|
"tokens/trainable": 17499296
|
|
},
|
|
{
|
|
"epoch": 1.2944708726507057,
|
|
"grad_norm": 0.15963433682918549,
|
|
"learning_rate": 1.4628200775826928e-05,
|
|
"loss": 1.5668434143066405,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.7915,
|
|
"step": 4150,
|
|
"tokens/total": 27244288,
|
|
"tokens/train_per_sec_per_gpu": 74.85,
|
|
"tokens/trainable": 17539696
|
|
},
|
|
{
|
|
"epoch": 1.2975902674881072,
|
|
"grad_norm": 0.13356998562812805,
|
|
"learning_rate": 1.4513443217546685e-05,
|
|
"loss": 1.6179019927978515,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.0425,
|
|
"step": 4160,
|
|
"tokens/total": 27307520,
|
|
"tokens/train_per_sec_per_gpu": 97.72,
|
|
"tokens/trainable": 17581018
|
|
},
|
|
{
|
|
"epoch": 1.3007096623255088,
|
|
"grad_norm": 0.1583528071641922,
|
|
"learning_rate": 1.4398953176007756e-05,
|
|
"loss": 1.5948526382446289,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.9276,
|
|
"step": 4170,
|
|
"tokens/total": 27372928,
|
|
"tokens/train_per_sec_per_gpu": 52.46,
|
|
"tokens/trainable": 17623250
|
|
},
|
|
{
|
|
"epoch": 1.3038290571629103,
|
|
"grad_norm": 0.13760212063789368,
|
|
"learning_rate": 1.4284733571902186e-05,
|
|
"loss": 1.56932373046875,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.8034,
|
|
"step": 4180,
|
|
"tokens/total": 27438720,
|
|
"tokens/train_per_sec_per_gpu": 79.81,
|
|
"tokens/trainable": 17665028
|
|
},
|
|
{
|
|
"epoch": 1.306948452000312,
|
|
"grad_norm": 0.1587306559085846,
|
|
"learning_rate": 1.4170787319022994e-05,
|
|
"loss": 1.6229925155639648,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.06823,
|
|
"step": 4190,
|
|
"tokens/total": 27501056,
|
|
"tokens/train_per_sec_per_gpu": 93.37,
|
|
"tokens/trainable": 17704582
|
|
},
|
|
{
|
|
"epoch": 1.3100678468377134,
|
|
"grad_norm": 0.14124682545661926,
|
|
"learning_rate": 1.4057117324189966e-05,
|
|
"loss": 1.597140407562256,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.93889,
|
|
"step": 4200,
|
|
"tokens/total": 27567360,
|
|
"tokens/train_per_sec_per_gpu": 83.11,
|
|
"tokens/trainable": 17746468
|
|
},
|
|
{
|
|
"epoch": 1.3131872416751151,
|
|
"grad_norm": 0.14113859832286835,
|
|
"learning_rate": 1.3943726487175341e-05,
|
|
"loss": 1.5019424438476563,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.4904,
|
|
"step": 4210,
|
|
"tokens/total": 27632896,
|
|
"tokens/train_per_sec_per_gpu": 48.8,
|
|
"tokens/trainable": 17787644
|
|
},
|
|
{
|
|
"epoch": 1.3163066365125164,
|
|
"grad_norm": 0.1546744853258133,
|
|
"learning_rate": 1.3830617700629973e-05,
|
|
"loss": 1.598065185546875,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.94346,
|
|
"step": 4220,
|
|
"tokens/total": 27700736,
|
|
"tokens/train_per_sec_per_gpu": 77.92,
|
|
"tokens/trainable": 17830832
|
|
},
|
|
{
|
|
"epoch": 1.3194260313499182,
|
|
"grad_norm": 0.14653222262859344,
|
|
"learning_rate": 1.3717793850009442e-05,
|
|
"loss": 1.610985565185547,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.00774,
|
|
"step": 4230,
|
|
"tokens/total": 27768704,
|
|
"tokens/train_per_sec_per_gpu": 57.68,
|
|
"tokens/trainable": 17874524
|
|
},
|
|
{
|
|
"epoch": 1.3225454261873197,
|
|
"grad_norm": 0.13174936175346375,
|
|
"learning_rate": 1.3605257813500526e-05,
|
|
"loss": 1.5397464752197265,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.66341,
|
|
"step": 4240,
|
|
"tokens/total": 27836032,
|
|
"tokens/train_per_sec_per_gpu": 62.17,
|
|
"tokens/trainable": 17918264
|
|
},
|
|
{
|
|
"epoch": 1.3256648210247213,
|
|
"grad_norm": 0.1453728824853897,
|
|
"learning_rate": 1.3493012461947696e-05,
|
|
"loss": 1.5660764694213867,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78783,
|
|
"step": 4250,
|
|
"tokens/total": 27899904,
|
|
"tokens/train_per_sec_per_gpu": 70.38,
|
|
"tokens/trainable": 17958716
|
|
},
|
|
{
|
|
"epoch": 1.3287842158621228,
|
|
"grad_norm": 0.15021538734436035,
|
|
"learning_rate": 1.3381060658779948e-05,
|
|
"loss": 1.5086420059204102,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.52059,
|
|
"step": 4260,
|
|
"tokens/total": 27966848,
|
|
"tokens/train_per_sec_per_gpu": 58.77,
|
|
"tokens/trainable": 18002284
|
|
},
|
|
{
|
|
"epoch": 1.3319036106995243,
|
|
"grad_norm": 0.14687480032444,
|
|
"learning_rate": 1.3269405259937734e-05,
|
|
"loss": 1.5709407806396485,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.81117,
|
|
"step": 4270,
|
|
"tokens/total": 28031104,
|
|
"tokens/train_per_sec_per_gpu": 53.29,
|
|
"tokens/trainable": 18043242
|
|
},
|
|
{
|
|
"epoch": 1.3350230055369259,
|
|
"grad_norm": 0.13591890037059784,
|
|
"learning_rate": 1.3158049113800063e-05,
|
|
"loss": 1.5904165267944337,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90579,
|
|
"step": 4280,
|
|
"tokens/total": 28095616,
|
|
"tokens/train_per_sec_per_gpu": 59.89,
|
|
"tokens/trainable": 18085584
|
|
},
|
|
{
|
|
"epoch": 1.3381424003743274,
|
|
"grad_norm": 0.15907874703407288,
|
|
"learning_rate": 1.3046995061111911e-05,
|
|
"loss": 1.5902300834655763,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90488,
|
|
"step": 4290,
|
|
"tokens/total": 28161536,
|
|
"tokens/train_per_sec_per_gpu": 38.7,
|
|
"tokens/trainable": 18128140
|
|
},
|
|
{
|
|
"epoch": 1.341261795211729,
|
|
"grad_norm": 0.14094775915145874,
|
|
"learning_rate": 1.2936245934911689e-05,
|
|
"loss": 1.5601555824279785,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.75956,
|
|
"step": 4300,
|
|
"tokens/total": 28225536,
|
|
"tokens/train_per_sec_per_gpu": 93.02,
|
|
"tokens/trainable": 18170140
|
|
},
|
|
{
|
|
"epoch": 1.3443811900491305,
|
|
"grad_norm": 0.13825203478336334,
|
|
"learning_rate": 1.2825804560459025e-05,
|
|
"loss": 1.5528628349304199,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.72498,
|
|
"step": 4310,
|
|
"tokens/total": 28290816,
|
|
"tokens/train_per_sec_per_gpu": 44.0,
|
|
"tokens/trainable": 18212016
|
|
},
|
|
{
|
|
"epoch": 1.347500584886532,
|
|
"grad_norm": 0.12606032192707062,
|
|
"learning_rate": 1.2715673755162628e-05,
|
|
"loss": 1.5940751075744628,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.92377,
|
|
"step": 4320,
|
|
"tokens/total": 28356608,
|
|
"tokens/train_per_sec_per_gpu": 78.35,
|
|
"tokens/trainable": 18255464
|
|
},
|
|
{
|
|
"epoch": 1.3506199797239336,
|
|
"grad_norm": 0.1331586092710495,
|
|
"learning_rate": 1.2605856328508475e-05,
|
|
"loss": 1.6620365142822267,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.27003,
|
|
"step": 4330,
|
|
"tokens/total": 28420992,
|
|
"tokens/train_per_sec_per_gpu": 101.4,
|
|
"tokens/trainable": 18297310
|
|
},
|
|
{
|
|
"epoch": 1.353739374561335,
|
|
"grad_norm": 0.14204779267311096,
|
|
"learning_rate": 1.2496355081988117e-05,
|
|
"loss": 1.5074813842773438,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.51534,
|
|
"step": 4340,
|
|
"tokens/total": 28486016,
|
|
"tokens/train_per_sec_per_gpu": 83.24,
|
|
"tokens/trainable": 18339194
|
|
},
|
|
{
|
|
"epoch": 1.3568587693987366,
|
|
"grad_norm": 0.1484689712524414,
|
|
"learning_rate": 1.2387172809027181e-05,
|
|
"loss": 1.5451210975646972,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68854,
|
|
"step": 4350,
|
|
"tokens/total": 28550400,
|
|
"tokens/train_per_sec_per_gpu": 79.72,
|
|
"tokens/trainable": 18381168
|
|
},
|
|
{
|
|
"epoch": 1.3599781642361382,
|
|
"grad_norm": 0.14702345430850983,
|
|
"learning_rate": 1.2278312294914174e-05,
|
|
"loss": 1.5346933364868165,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.6399,
|
|
"step": 4360,
|
|
"tokens/total": 28616320,
|
|
"tokens/train_per_sec_per_gpu": 100.28,
|
|
"tokens/trainable": 18422468
|
|
},
|
|
{
|
|
"epoch": 1.3630975590735397,
|
|
"grad_norm": 0.14137879014015198,
|
|
"learning_rate": 1.216977631672935e-05,
|
|
"loss": 1.5980229377746582,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.94325,
|
|
"step": 4370,
|
|
"tokens/total": 28683648,
|
|
"tokens/train_per_sec_per_gpu": 93.64,
|
|
"tokens/trainable": 18466336
|
|
},
|
|
{
|
|
"epoch": 1.3662169539109412,
|
|
"grad_norm": 0.14050492644309998,
|
|
"learning_rate": 1.2061567643273947e-05,
|
|
"loss": 1.6203935623168946,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.05508,
|
|
"step": 4380,
|
|
"tokens/total": 28747520,
|
|
"tokens/train_per_sec_per_gpu": 78.71,
|
|
"tokens/trainable": 18506384
|
|
},
|
|
{
|
|
"epoch": 1.3693363487483428,
|
|
"grad_norm": 0.14448665082454681,
|
|
"learning_rate": 1.1953689034999475e-05,
|
|
"loss": 1.5309012413024903,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.62234,
|
|
"step": 4390,
|
|
"tokens/total": 28814080,
|
|
"tokens/train_per_sec_per_gpu": 113.64,
|
|
"tokens/trainable": 18549266
|
|
},
|
|
{
|
|
"epoch": 1.3724557435857443,
|
|
"grad_norm": 0.13639667630195618,
|
|
"learning_rate": 1.184614324393736e-05,
|
|
"loss": 1.5499507904052734,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.71124,
|
|
"step": 4400,
|
|
"tokens/total": 28879488,
|
|
"tokens/train_per_sec_per_gpu": 65.43,
|
|
"tokens/trainable": 18590956
|
|
},
|
|
{
|
|
"epoch": 1.3755751384231458,
|
|
"grad_norm": 0.15962563455104828,
|
|
"learning_rate": 1.1738933013628712e-05,
|
|
"loss": 1.513906764984131,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.54445,
|
|
"step": 4410,
|
|
"tokens/total": 28945408,
|
|
"tokens/train_per_sec_per_gpu": 80.18,
|
|
"tokens/trainable": 18633780
|
|
},
|
|
{
|
|
"epoch": 1.3786945332605476,
|
|
"grad_norm": 0.14171810448169708,
|
|
"learning_rate": 1.1632061079054335e-05,
|
|
"loss": 1.567171859741211,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.79307,
|
|
"step": 4420,
|
|
"tokens/total": 29010304,
|
|
"tokens/train_per_sec_per_gpu": 84.16,
|
|
"tokens/trainable": 18676064
|
|
},
|
|
{
|
|
"epoch": 1.381813928097949,
|
|
"grad_norm": 0.16934113204479218,
|
|
"learning_rate": 1.1525530166564932e-05,
|
|
"loss": 1.5808257102966308,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.85897,
|
|
"step": 4430,
|
|
"tokens/total": 29074176,
|
|
"tokens/train_per_sec_per_gpu": 38.83,
|
|
"tokens/trainable": 18717732
|
|
},
|
|
{
|
|
"epoch": 1.3849333229353507,
|
|
"grad_norm": 0.1351517289876938,
|
|
"learning_rate": 1.1419342993811627e-05,
|
|
"loss": 1.539092445373535,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.66036,
|
|
"step": 4440,
|
|
"tokens/total": 29137536,
|
|
"tokens/train_per_sec_per_gpu": 103.7,
|
|
"tokens/trainable": 18760014
|
|
},
|
|
{
|
|
"epoch": 1.388052717772752,
|
|
"grad_norm": 0.1475447118282318,
|
|
"learning_rate": 1.1313502269676552e-05,
|
|
"loss": 1.4690265655517578,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.345,
|
|
"step": 4450,
|
|
"tokens/total": 29203712,
|
|
"tokens/train_per_sec_per_gpu": 62.69,
|
|
"tokens/trainable": 18803936
|
|
},
|
|
{
|
|
"epoch": 1.3911721126101537,
|
|
"grad_norm": 0.14662675559520721,
|
|
"learning_rate": 1.120801069420378e-05,
|
|
"loss": 1.5062439918518067,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.50976,
|
|
"step": 4460,
|
|
"tokens/total": 29268480,
|
|
"tokens/train_per_sec_per_gpu": 57.01,
|
|
"tokens/trainable": 18845660
|
|
},
|
|
{
|
|
"epoch": 1.3942915074475553,
|
|
"grad_norm": 0.1487852931022644,
|
|
"learning_rate": 1.110287095853047e-05,
|
|
"loss": 1.504355239868164,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.50125,
|
|
"step": 4470,
|
|
"tokens/total": 29335424,
|
|
"tokens/train_per_sec_per_gpu": 44.75,
|
|
"tokens/trainable": 18887666
|
|
},
|
|
{
|
|
"epoch": 1.3974109022849568,
|
|
"grad_norm": 0.159983292222023,
|
|
"learning_rate": 1.0998085744818181e-05,
|
|
"loss": 1.616290283203125,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.03438,
|
|
"step": 4480,
|
|
"tokens/total": 29399552,
|
|
"tokens/train_per_sec_per_gpu": 72.46,
|
|
"tokens/trainable": 18929102
|
|
},
|
|
{
|
|
"epoch": 1.4005302971223583,
|
|
"grad_norm": 0.1599910408258438,
|
|
"learning_rate": 1.0893657726184467e-05,
|
|
"loss": 1.6163484573364257,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.03467,
|
|
"step": 4490,
|
|
"tokens/total": 29465216,
|
|
"tokens/train_per_sec_per_gpu": 70.11,
|
|
"tokens/trainable": 18971704
|
|
},
|
|
{
|
|
"epoch": 1.4036496919597599,
|
|
"grad_norm": 0.14899516105651855,
|
|
"learning_rate": 1.0789589566634642e-05,
|
|
"loss": 1.5897905349731445,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.90272,
|
|
"step": 4500,
|
|
"tokens/total": 29533184,
|
|
"tokens/train_per_sec_per_gpu": 62.98,
|
|
"tokens/trainable": 19016200
|
|
},
|
|
{
|
|
"epoch": 1.4067690867971614,
|
|
"grad_norm": 0.14178679883480072,
|
|
"learning_rate": 1.0685883920993903e-05,
|
|
"loss": 1.5734087944030761,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.82306,
|
|
"step": 4510,
|
|
"tokens/total": 29599360,
|
|
"tokens/train_per_sec_per_gpu": 123.98,
|
|
"tokens/trainable": 19059180
|
|
},
|
|
{
|
|
"epoch": 1.409888481634563,
|
|
"grad_norm": 0.1457754224538803,
|
|
"learning_rate": 1.0582543434839508e-05,
|
|
"loss": 1.640203094482422,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.15622,
|
|
"step": 4520,
|
|
"tokens/total": 29664256,
|
|
"tokens/train_per_sec_per_gpu": 87.5,
|
|
"tokens/trainable": 19100092
|
|
},
|
|
{
|
|
"epoch": 1.4130078764719645,
|
|
"grad_norm": 0.141546368598938,
|
|
"learning_rate": 1.0479570744433381e-05,
|
|
"loss": 1.5249863624572755,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.59508,
|
|
"step": 4530,
|
|
"tokens/total": 29731072,
|
|
"tokens/train_per_sec_per_gpu": 92.25,
|
|
"tokens/trainable": 19142898
|
|
},
|
|
{
|
|
"epoch": 1.416127271309366,
|
|
"grad_norm": 0.1456439197063446,
|
|
"learning_rate": 1.037696847665476e-05,
|
|
"loss": 1.5784012794494628,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.8472,
|
|
"step": 4540,
|
|
"tokens/total": 29796992,
|
|
"tokens/train_per_sec_per_gpu": 75.97,
|
|
"tokens/trainable": 19186068
|
|
},
|
|
{
|
|
"epoch": 1.4192466661467675,
|
|
"grad_norm": 0.13533726334571838,
|
|
"learning_rate": 1.0274739248933281e-05,
|
|
"loss": 1.6080335617065429,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.99298,
|
|
"step": 4550,
|
|
"tokens/total": 29864064,
|
|
"tokens/train_per_sec_per_gpu": 48.33,
|
|
"tokens/trainable": 19229808
|
|
},
|
|
{
|
|
"epoch": 1.422366060984169,
|
|
"grad_norm": 0.15571539103984833,
|
|
"learning_rate": 1.0172885669182155e-05,
|
|
"loss": 1.5664268493652345,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.7895,
|
|
"step": 4560,
|
|
"tokens/total": 29930368,
|
|
"tokens/train_per_sec_per_gpu": 62.7,
|
|
"tokens/trainable": 19271400
|
|
},
|
|
{
|
|
"epoch": 1.4254854558215706,
|
|
"grad_norm": 0.14321453869342804,
|
|
"learning_rate": 1.0071410335731613e-05,
|
|
"loss": 1.439999771118164,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.22069,
|
|
"step": 4570,
|
|
"tokens/total": 29996288,
|
|
"tokens/train_per_sec_per_gpu": 52.84,
|
|
"tokens/trainable": 19313888
|
|
},
|
|
{
|
|
"epoch": 1.4286048506589721,
|
|
"grad_norm": 0.14048445224761963,
|
|
"learning_rate": 9.970315837262698e-06,
|
|
"loss": 1.564855194091797,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78198,
|
|
"step": 4580,
|
|
"tokens/total": 30060288,
|
|
"tokens/train_per_sec_per_gpu": 90.74,
|
|
"tokens/trainable": 19355120
|
|
},
|
|
{
|
|
"epoch": 1.4317242454963737,
|
|
"grad_norm": 0.1300923228263855,
|
|
"learning_rate": 9.86960475274114e-06,
|
|
"loss": 1.5211267471313477,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.57738,
|
|
"step": 4590,
|
|
"tokens/total": 30128512,
|
|
"tokens/train_per_sec_per_gpu": 97.1,
|
|
"tokens/trainable": 19398902
|
|
},
|
|
{
|
|
"epoch": 1.4348436403337752,
|
|
"grad_norm": 0.13100987672805786,
|
|
"learning_rate": 9.76927965135165e-06,
|
|
"loss": 1.5517333030700684,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.71964,
|
|
"step": 4600,
|
|
"tokens/total": 30194304,
|
|
"tokens/train_per_sec_per_gpu": 83.57,
|
|
"tokens/trainable": 19441200
|
|
},
|
|
{
|
|
"epoch": 1.4379630351711767,
|
|
"grad_norm": 0.14557424187660217,
|
|
"learning_rate": 9.669343092432296e-06,
|
|
"loss": 1.568013858795166,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.79711,
|
|
"step": 4610,
|
|
"tokens/total": 30258816,
|
|
"tokens/train_per_sec_per_gpu": 88.57,
|
|
"tokens/trainable": 19482320
|
|
},
|
|
{
|
|
"epoch": 1.4410824300085783,
|
|
"grad_norm": 0.136590838432312,
|
|
"learning_rate": 9.569797625409302e-06,
|
|
"loss": 1.630331802368164,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.10557,
|
|
"step": 4620,
|
|
"tokens/total": 30322944,
|
|
"tokens/train_per_sec_per_gpu": 86.52,
|
|
"tokens/trainable": 19522532
|
|
},
|
|
{
|
|
"epoch": 1.4442018248459798,
|
|
"grad_norm": 0.1384277194738388,
|
|
"learning_rate": 9.470645789731914e-06,
|
|
"loss": 1.5262042045593263,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.60068,
|
|
"step": 4630,
|
|
"tokens/total": 30389248,
|
|
"tokens/train_per_sec_per_gpu": 52.4,
|
|
"tokens/trainable": 19564648
|
|
},
|
|
{
|
|
"epoch": 1.4473212196833813,
|
|
"grad_norm": 0.14514420926570892,
|
|
"learning_rate": 9.371890114807708e-06,
|
|
"loss": 1.5683574676513672,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.79876,
|
|
"step": 4640,
|
|
"tokens/total": 30452864,
|
|
"tokens/train_per_sec_per_gpu": 47.63,
|
|
"tokens/trainable": 19606364
|
|
},
|
|
{
|
|
"epoch": 1.450440614520783,
|
|
"grad_norm": 0.13131366670131683,
|
|
"learning_rate": 9.27353311993799e-06,
|
|
"loss": 1.516063117980957,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.55426,
|
|
"step": 4650,
|
|
"tokens/total": 30519168,
|
|
"tokens/train_per_sec_per_gpu": 64.44,
|
|
"tokens/trainable": 19648308
|
|
},
|
|
{
|
|
"epoch": 1.4535600093581844,
|
|
"grad_norm": 0.13949370384216309,
|
|
"learning_rate": 9.175577314253572e-06,
|
|
"loss": 1.5554485321044922,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73721,
|
|
"step": 4660,
|
|
"tokens/total": 30583936,
|
|
"tokens/train_per_sec_per_gpu": 33.45,
|
|
"tokens/trainable": 19689364
|
|
},
|
|
{
|
|
"epoch": 1.4566794041955862,
|
|
"grad_norm": 0.1425829380750656,
|
|
"learning_rate": 9.078025196650769e-06,
|
|
"loss": 1.5294002532958983,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.61541,
|
|
"step": 4670,
|
|
"tokens/total": 30648704,
|
|
"tokens/train_per_sec_per_gpu": 58.15,
|
|
"tokens/trainable": 19732080
|
|
},
|
|
{
|
|
"epoch": 1.4597987990329875,
|
|
"grad_norm": 0.1424936205148697,
|
|
"learning_rate": 8.980879255727583e-06,
|
|
"loss": 1.5415650367736817,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.6719,
|
|
"step": 4680,
|
|
"tokens/total": 30713088,
|
|
"tokens/train_per_sec_per_gpu": 81.71,
|
|
"tokens/trainable": 19773200
|
|
},
|
|
{
|
|
"epoch": 1.4629181938703892,
|
|
"grad_norm": 0.15161575376987457,
|
|
"learning_rate": 8.884141969720348e-06,
|
|
"loss": 1.6255985260009767,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.08146,
|
|
"step": 4690,
|
|
"tokens/total": 30776448,
|
|
"tokens/train_per_sec_per_gpu": 61.28,
|
|
"tokens/trainable": 19814114
|
|
},
|
|
{
|
|
"epoch": 1.4660375887077906,
|
|
"grad_norm": 0.12817059457302094,
|
|
"learning_rate": 8.787815806440344e-06,
|
|
"loss": 1.5206357955932617,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.57513,
|
|
"step": 4700,
|
|
"tokens/total": 30841984,
|
|
"tokens/train_per_sec_per_gpu": 60.62,
|
|
"tokens/trainable": 19855516
|
|
},
|
|
{
|
|
"epoch": 1.4691569835451923,
|
|
"grad_norm": 0.1400434672832489,
|
|
"learning_rate": 8.691903223210992e-06,
|
|
"loss": 1.477653694152832,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.38265,
|
|
"step": 4710,
|
|
"tokens/total": 30907648,
|
|
"tokens/train_per_sec_per_gpu": 56.22,
|
|
"tokens/trainable": 19899052
|
|
},
|
|
{
|
|
"epoch": 1.4722763783825938,
|
|
"grad_norm": 0.13295724987983704,
|
|
"learning_rate": 8.596406666805052e-06,
|
|
"loss": 1.5872390747070313,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.89023,
|
|
"step": 4720,
|
|
"tokens/total": 30972672,
|
|
"tokens/train_per_sec_per_gpu": 61.6,
|
|
"tokens/trainable": 19941268
|
|
},
|
|
{
|
|
"epoch": 1.4753957732199954,
|
|
"grad_norm": 0.1311759352684021,
|
|
"learning_rate": 8.501328573382295e-06,
|
|
"loss": 1.5646821022033692,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78115,
|
|
"step": 4730,
|
|
"tokens/total": 31041152,
|
|
"tokens/train_per_sec_per_gpu": 86.51,
|
|
"tokens/trainable": 19986580
|
|
},
|
|
{
|
|
"epoch": 1.478515168057397,
|
|
"grad_norm": 0.13007056713104248,
|
|
"learning_rate": 8.40667136842728e-06,
|
|
"loss": 1.6018630981445312,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.96227,
|
|
"step": 4740,
|
|
"tokens/total": 31105152,
|
|
"tokens/train_per_sec_per_gpu": 79.85,
|
|
"tokens/trainable": 20028322
|
|
},
|
|
{
|
|
"epoch": 1.4816345628947984,
|
|
"grad_norm": 0.12904047966003418,
|
|
"learning_rate": 8.312437466687539e-06,
|
|
"loss": 1.5781673431396483,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.84607,
|
|
"step": 4750,
|
|
"tokens/total": 31168000,
|
|
"tokens/train_per_sec_per_gpu": 116.22,
|
|
"tokens/trainable": 20068344
|
|
},
|
|
{
|
|
"epoch": 1.4847539577322,
|
|
"grad_norm": 0.162785142660141,
|
|
"learning_rate": 8.218629272111944e-06,
|
|
"loss": 1.5452213287353516,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68901,
|
|
"step": 4760,
|
|
"tokens/total": 31235328,
|
|
"tokens/train_per_sec_per_gpu": 103.2,
|
|
"tokens/trainable": 20111246
|
|
},
|
|
{
|
|
"epoch": 1.4878733525696015,
|
|
"grad_norm": 0.1633530855178833,
|
|
"learning_rate": 8.125249177789373e-06,
|
|
"loss": 1.6021232604980469,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.96356,
|
|
"step": 4770,
|
|
"tokens/total": 31298048,
|
|
"tokens/train_per_sec_per_gpu": 97.02,
|
|
"tokens/trainable": 20153360
|
|
},
|
|
{
|
|
"epoch": 1.490992747407003,
|
|
"grad_norm": 0.13832278549671173,
|
|
"learning_rate": 8.032299565887704e-06,
|
|
"loss": 1.5490838050842286,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.70716,
|
|
"step": 4780,
|
|
"tokens/total": 31366784,
|
|
"tokens/train_per_sec_per_gpu": 58.71,
|
|
"tokens/trainable": 20198436
|
|
},
|
|
{
|
|
"epoch": 1.4941121422444046,
|
|
"grad_norm": 0.15044964849948883,
|
|
"learning_rate": 7.939782807592976e-06,
|
|
"loss": 1.6331809997558593,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.12014,
|
|
"step": 4790,
|
|
"tokens/total": 31433344,
|
|
"tokens/train_per_sec_per_gpu": 102.23,
|
|
"tokens/trainable": 20242918
|
|
},
|
|
{
|
|
"epoch": 1.4972315370818061,
|
|
"grad_norm": 0.13916781544685364,
|
|
"learning_rate": 7.847701263048988e-06,
|
|
"loss": 1.5643228530883788,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.77944,
|
|
"step": 4800,
|
|
"tokens/total": 31497984,
|
|
"tokens/train_per_sec_per_gpu": 93.41,
|
|
"tokens/trainable": 20285116
|
|
},
|
|
{
|
|
"epoch": 1.5003509319192077,
|
|
"grad_norm": 0.13971014320850372,
|
|
"learning_rate": 7.756057281296986e-06,
|
|
"loss": 1.573653507232666,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.82424,
|
|
"step": 4810,
|
|
"tokens/total": 31563904,
|
|
"tokens/train_per_sec_per_gpu": 62.79,
|
|
"tokens/trainable": 20326796
|
|
},
|
|
{
|
|
"epoch": 1.5034703267566092,
|
|
"grad_norm": 0.13818620145320892,
|
|
"learning_rate": 7.664853200215858e-06,
|
|
"loss": 1.5865263938903809,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.88674,
|
|
"step": 4820,
|
|
"tokens/total": 31630208,
|
|
"tokens/train_per_sec_per_gpu": 78.31,
|
|
"tokens/trainable": 20369632
|
|
},
|
|
{
|
|
"epoch": 1.5065897215940107,
|
|
"grad_norm": 0.13834123313426971,
|
|
"learning_rate": 7.574091346462378e-06,
|
|
"loss": 1.5617074966430664,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.76695,
|
|
"step": 4830,
|
|
"tokens/total": 31695872,
|
|
"tokens/train_per_sec_per_gpu": 76.07,
|
|
"tokens/trainable": 20412680
|
|
},
|
|
{
|
|
"epoch": 1.5097091164314123,
|
|
"grad_norm": 0.14885418117046356,
|
|
"learning_rate": 7.48377403541192e-06,
|
|
"loss": 1.5482691764831542,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.70332,
|
|
"step": 4840,
|
|
"tokens/total": 31761792,
|
|
"tokens/train_per_sec_per_gpu": 75.02,
|
|
"tokens/trainable": 20455212
|
|
},
|
|
{
|
|
"epoch": 1.5128285112688138,
|
|
"grad_norm": 0.15173283219337463,
|
|
"learning_rate": 7.393903571099389e-06,
|
|
"loss": 1.5690330505371093,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.802,
|
|
"step": 4850,
|
|
"tokens/total": 31826432,
|
|
"tokens/train_per_sec_per_gpu": 78.49,
|
|
"tokens/trainable": 20498012
|
|
},
|
|
{
|
|
"epoch": 1.5159479061062155,
|
|
"grad_norm": 0.15235094726085663,
|
|
"learning_rate": 7.304482246160402e-06,
|
|
"loss": 1.570943546295166,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.81119,
|
|
"step": 4860,
|
|
"tokens/total": 31889920,
|
|
"tokens/train_per_sec_per_gpu": 47.26,
|
|
"tokens/trainable": 20538284
|
|
},
|
|
{
|
|
"epoch": 1.5190673009436169,
|
|
"grad_norm": 0.1360926479101181,
|
|
"learning_rate": 7.215512341772868e-06,
|
|
"loss": 1.493645477294922,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.4533,
|
|
"step": 4870,
|
|
"tokens/total": 31957760,
|
|
"tokens/train_per_sec_per_gpu": 62.05,
|
|
"tokens/trainable": 20580800
|
|
},
|
|
{
|
|
"epoch": 1.5221866957810186,
|
|
"grad_norm": 0.14283378422260284,
|
|
"learning_rate": 7.126996127598704e-06,
|
|
"loss": 1.5381041526794434,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.65576,
|
|
"step": 4880,
|
|
"tokens/total": 32023040,
|
|
"tokens/train_per_sec_per_gpu": 99.64,
|
|
"tokens/trainable": 20621112
|
|
},
|
|
{
|
|
"epoch": 1.52530609061842,
|
|
"grad_norm": 0.14295639097690582,
|
|
"learning_rate": 7.038935861726056e-06,
|
|
"loss": 1.5845683097839356,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.87719,
|
|
"step": 4890,
|
|
"tokens/total": 32089472,
|
|
"tokens/train_per_sec_per_gpu": 71.55,
|
|
"tokens/trainable": 20664348
|
|
},
|
|
{
|
|
"epoch": 1.5284254854558217,
|
|
"grad_norm": 0.13357999920845032,
|
|
"learning_rate": 6.951333790611561e-06,
|
|
"loss": 1.5853243827819825,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.88087,
|
|
"step": 4900,
|
|
"tokens/total": 32157184,
|
|
"tokens/train_per_sec_per_gpu": 71.26,
|
|
"tokens/trainable": 20707482
|
|
},
|
|
{
|
|
"epoch": 1.531544880293223,
|
|
"grad_norm": 0.14599433541297913,
|
|
"learning_rate": 6.864192149023149e-06,
|
|
"loss": 1.5060012817382813,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.50867,
|
|
"step": 4910,
|
|
"tokens/total": 32222848,
|
|
"tokens/train_per_sec_per_gpu": 64.63,
|
|
"tokens/trainable": 20748656
|
|
},
|
|
{
|
|
"epoch": 1.5346642751306248,
|
|
"grad_norm": 0.1534305065870285,
|
|
"learning_rate": 6.777513159982951e-06,
|
|
"loss": 1.5884516716003418,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.89616,
|
|
"step": 4920,
|
|
"tokens/total": 32289152,
|
|
"tokens/train_per_sec_per_gpu": 89.91,
|
|
"tokens/trainable": 20792570
|
|
},
|
|
{
|
|
"epoch": 1.537783669968026,
|
|
"grad_norm": 0.15131875872612,
|
|
"learning_rate": 6.6912990347106576e-06,
|
|
"loss": 1.5630720138549805,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.77346,
|
|
"step": 4930,
|
|
"tokens/total": 32356224,
|
|
"tokens/train_per_sec_per_gpu": 48.87,
|
|
"tokens/trainable": 20833696
|
|
},
|
|
{
|
|
"epoch": 1.5409030648054278,
|
|
"grad_norm": 0.1547374278306961,
|
|
"learning_rate": 6.605551972567048e-06,
|
|
"loss": 1.574531364440918,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.82848,
|
|
"step": 4940,
|
|
"tokens/total": 32422528,
|
|
"tokens/train_per_sec_per_gpu": 92.66,
|
|
"tokens/trainable": 20875604
|
|
},
|
|
{
|
|
"epoch": 1.5440224596428291,
|
|
"grad_norm": 0.15408794581890106,
|
|
"learning_rate": 6.520274160997936e-06,
|
|
"loss": 1.6241125106811523,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.07391,
|
|
"step": 4950,
|
|
"tokens/total": 32488832,
|
|
"tokens/train_per_sec_per_gpu": 76.48,
|
|
"tokens/trainable": 20918416
|
|
},
|
|
{
|
|
"epoch": 1.547141854480231,
|
|
"grad_norm": 0.13255000114440918,
|
|
"learning_rate": 6.435467775478341e-06,
|
|
"loss": 1.5074248313903809,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.51509,
|
|
"step": 4960,
|
|
"tokens/total": 32555136,
|
|
"tokens/train_per_sec_per_gpu": 68.45,
|
|
"tokens/trainable": 20960104
|
|
},
|
|
{
|
|
"epoch": 1.5502612493176324,
|
|
"grad_norm": 0.12968680262565613,
|
|
"learning_rate": 6.351134979456972e-06,
|
|
"loss": 1.5170172691345214,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.55861,
|
|
"step": 4970,
|
|
"tokens/total": 32622720,
|
|
"tokens/train_per_sec_per_gpu": 92.66,
|
|
"tokens/trainable": 21004772
|
|
},
|
|
{
|
|
"epoch": 1.553380644155034,
|
|
"grad_norm": 0.12842552363872528,
|
|
"learning_rate": 6.267277924301093e-06,
|
|
"loss": 1.4748305320739745,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.3703,
|
|
"step": 4980,
|
|
"tokens/total": 32691456,
|
|
"tokens/train_per_sec_per_gpu": 96.44,
|
|
"tokens/trainable": 21047884
|
|
},
|
|
{
|
|
"epoch": 1.5565000389924355,
|
|
"grad_norm": 0.13930045068264008,
|
|
"learning_rate": 6.183898749241568e-06,
|
|
"loss": 1.6088586807250977,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.9971,
|
|
"step": 4990,
|
|
"tokens/total": 32756352,
|
|
"tokens/train_per_sec_per_gpu": 88.48,
|
|
"tokens/trainable": 21091332
|
|
},
|
|
{
|
|
"epoch": 1.559619433829837,
|
|
"grad_norm": 0.15511450171470642,
|
|
"learning_rate": 6.100999581318365e-06,
|
|
"loss": 1.5874125480651855,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.89108,
|
|
"step": 5000,
|
|
"tokens/total": 32818304,
|
|
"tokens/train_per_sec_per_gpu": 84.96,
|
|
"tokens/trainable": 21129996
|
|
},
|
|
{
|
|
"epoch": 1.5627388286672386,
|
|
"grad_norm": 0.14723895490169525,
|
|
"learning_rate": 6.01858253532622e-06,
|
|
"loss": 1.550753402709961,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.71502,
|
|
"step": 5010,
|
|
"tokens/total": 32883200,
|
|
"tokens/train_per_sec_per_gpu": 71.93,
|
|
"tokens/trainable": 21172696
|
|
},
|
|
{
|
|
"epoch": 1.56585822350464,
|
|
"grad_norm": 0.1273687481880188,
|
|
"learning_rate": 5.93664971376075e-06,
|
|
"loss": 1.459110927581787,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.30213,
|
|
"step": 5020,
|
|
"tokens/total": 32951424,
|
|
"tokens/train_per_sec_per_gpu": 61.24,
|
|
"tokens/trainable": 21217422
|
|
},
|
|
{
|
|
"epoch": 1.5689776183420416,
|
|
"grad_norm": 0.14541353285312653,
|
|
"learning_rate": 5.855203206764784e-06,
|
|
"loss": 1.7013654708862305,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.48143,
|
|
"step": 5030,
|
|
"tokens/total": 33017600,
|
|
"tokens/train_per_sec_per_gpu": 81.22,
|
|
"tokens/trainable": 21259898
|
|
},
|
|
{
|
|
"epoch": 1.5720970131794432,
|
|
"grad_norm": 0.13759450614452362,
|
|
"learning_rate": 5.774245092075037e-06,
|
|
"loss": 1.534003734588623,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.6367,
|
|
"step": 5040,
|
|
"tokens/total": 33083136,
|
|
"tokens/train_per_sec_per_gpu": 80.89,
|
|
"tokens/trainable": 21300810
|
|
},
|
|
{
|
|
"epoch": 1.5752164080168447,
|
|
"grad_norm": 0.16264495253562927,
|
|
"learning_rate": 5.693777434969142e-06,
|
|
"loss": 1.5818219184875488,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.86381,
|
|
"step": 5050,
|
|
"tokens/total": 33149952,
|
|
"tokens/train_per_sec_per_gpu": 91.81,
|
|
"tokens/trainable": 21343644
|
|
},
|
|
{
|
|
"epoch": 1.5783358028542462,
|
|
"grad_norm": 0.15952593088150024,
|
|
"learning_rate": 5.61380228821291e-06,
|
|
"loss": 1.526211929321289,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.60072,
|
|
"step": 5060,
|
|
"tokens/total": 33213696,
|
|
"tokens/train_per_sec_per_gpu": 68.4,
|
|
"tokens/trainable": 21382332
|
|
},
|
|
{
|
|
"epoch": 1.5814551976916478,
|
|
"grad_norm": 0.14448519051074982,
|
|
"learning_rate": 5.534321692008024e-06,
|
|
"loss": 1.592270851135254,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.9149,
|
|
"step": 5070,
|
|
"tokens/total": 33279104,
|
|
"tokens/train_per_sec_per_gpu": 73.87,
|
|
"tokens/trainable": 21424580
|
|
},
|
|
{
|
|
"epoch": 1.5845745925290493,
|
|
"grad_norm": 0.13160553574562073,
|
|
"learning_rate": 5.455337673939936e-06,
|
|
"loss": 1.6008161544799804,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.95708,
|
|
"step": 5080,
|
|
"tokens/total": 33345024,
|
|
"tokens/train_per_sec_per_gpu": 64.78,
|
|
"tokens/trainable": 21466600
|
|
},
|
|
{
|
|
"epoch": 1.587693987366451,
|
|
"grad_norm": 0.13351571559906006,
|
|
"learning_rate": 5.3768522489261845e-06,
|
|
"loss": 1.5782149314880372,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.8463,
|
|
"step": 5090,
|
|
"tokens/total": 33411840,
|
|
"tokens/train_per_sec_per_gpu": 89.28,
|
|
"tokens/trainable": 21509750
|
|
},
|
|
{
|
|
"epoch": 1.5908133822038524,
|
|
"grad_norm": 0.15875142812728882,
|
|
"learning_rate": 5.298867419164974e-06,
|
|
"loss": 1.5138155937194824,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.54404,
|
|
"step": 5100,
|
|
"tokens/total": 33479936,
|
|
"tokens/train_per_sec_per_gpu": 70.18,
|
|
"tokens/trainable": 21554452
|
|
},
|
|
{
|
|
"epoch": 1.5939327770412541,
|
|
"grad_norm": 0.14681676030158997,
|
|
"learning_rate": 5.221385174084104e-06,
|
|
"loss": 1.5458107948303224,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.69177,
|
|
"step": 5110,
|
|
"tokens/total": 33547264,
|
|
"tokens/train_per_sec_per_gpu": 74.95,
|
|
"tokens/trainable": 21598056
|
|
},
|
|
{
|
|
"epoch": 1.5970521718786554,
|
|
"grad_norm": 0.12346994876861572,
|
|
"learning_rate": 5.144407490290193e-06,
|
|
"loss": 1.488891887664795,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.43218,
|
|
"step": 5120,
|
|
"tokens/total": 33613568,
|
|
"tokens/train_per_sec_per_gpu": 97.03,
|
|
"tokens/trainable": 21640720
|
|
},
|
|
{
|
|
"epoch": 1.6001715667160572,
|
|
"grad_norm": 0.1325811743736267,
|
|
"learning_rate": 5.067936331518314e-06,
|
|
"loss": 1.5809184074401856,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.85942,
|
|
"step": 5130,
|
|
"tokens/total": 33680256,
|
|
"tokens/train_per_sec_per_gpu": 53.96,
|
|
"tokens/trainable": 21684030
|
|
},
|
|
{
|
|
"epoch": 1.6032909615534585,
|
|
"grad_norm": 0.14104141294956207,
|
|
"learning_rate": 4.991973648581818e-06,
|
|
"loss": 1.5370706558227538,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.65095,
|
|
"step": 5140,
|
|
"tokens/total": 33746432,
|
|
"tokens/train_per_sec_per_gpu": 64.28,
|
|
"tokens/trainable": 21724982
|
|
},
|
|
{
|
|
"epoch": 1.6064103563908603,
|
|
"grad_norm": 0.14926715195178986,
|
|
"learning_rate": 4.91652137932265e-06,
|
|
"loss": 1.5518075942993164,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.71999,
|
|
"step": 5150,
|
|
"tokens/total": 33810560,
|
|
"tokens/train_per_sec_per_gpu": 68.51,
|
|
"tokens/trainable": 21767060
|
|
},
|
|
{
|
|
"epoch": 1.6095297512282616,
|
|
"grad_norm": 0.13466720283031464,
|
|
"learning_rate": 4.841581448561841e-06,
|
|
"loss": 1.4738055229187013,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.36582,
|
|
"step": 5160,
|
|
"tokens/total": 33875584,
|
|
"tokens/train_per_sec_per_gpu": 94.62,
|
|
"tokens/trainable": 21808068
|
|
},
|
|
{
|
|
"epoch": 1.6126491460656633,
|
|
"grad_norm": 0.13840611279010773,
|
|
"learning_rate": 4.767155768050461e-06,
|
|
"loss": 1.506776714324951,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.51216,
|
|
"step": 5170,
|
|
"tokens/total": 33939712,
|
|
"tokens/train_per_sec_per_gpu": 92.59,
|
|
"tokens/trainable": 21850434
|
|
},
|
|
{
|
|
"epoch": 1.6157685409030647,
|
|
"grad_norm": 0.14631780982017517,
|
|
"learning_rate": 4.693246236420823e-06,
|
|
"loss": 1.595878505706787,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.93266,
|
|
"step": 5180,
|
|
"tokens/total": 34004352,
|
|
"tokens/train_per_sec_per_gpu": 74.71,
|
|
"tokens/trainable": 21892286
|
|
},
|
|
{
|
|
"epoch": 1.6188879357404664,
|
|
"grad_norm": 0.1493046134710312,
|
|
"learning_rate": 4.619854739138038e-06,
|
|
"loss": 1.6333091735839844,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.12079,
|
|
"step": 5190,
|
|
"tokens/total": 34071552,
|
|
"tokens/train_per_sec_per_gpu": 114.19,
|
|
"tokens/trainable": 21937020
|
|
},
|
|
{
|
|
"epoch": 1.622007330577868,
|
|
"grad_norm": 0.13579541444778442,
|
|
"learning_rate": 4.546983148451952e-06,
|
|
"loss": 1.518681526184082,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.5662,
|
|
"step": 5200,
|
|
"tokens/total": 34139264,
|
|
"tokens/train_per_sec_per_gpu": 100.77,
|
|
"tokens/trainable": 21980932
|
|
},
|
|
{
|
|
"epoch": 1.6251267254152695,
|
|
"grad_norm": 0.1567651480436325,
|
|
"learning_rate": 4.4746333233493394e-06,
|
|
"loss": 1.4987947463989257,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.47629,
|
|
"step": 5210,
|
|
"tokens/total": 34205952,
|
|
"tokens/train_per_sec_per_gpu": 74.87,
|
|
"tokens/trainable": 22023494
|
|
},
|
|
{
|
|
"epoch": 1.628246120252671,
|
|
"grad_norm": 0.14161834120750427,
|
|
"learning_rate": 4.402807109506521e-06,
|
|
"loss": 1.5893315315246581,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90047,
|
|
"step": 5220,
|
|
"tokens/total": 34272512,
|
|
"tokens/train_per_sec_per_gpu": 43.2,
|
|
"tokens/trainable": 22065218
|
|
},
|
|
{
|
|
"epoch": 1.6313655150900725,
|
|
"grad_norm": 0.1377056986093521,
|
|
"learning_rate": 4.331506339242239e-06,
|
|
"loss": 1.5076416969299316,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.51607,
|
|
"step": 5230,
|
|
"tokens/total": 34338816,
|
|
"tokens/train_per_sec_per_gpu": 74.89,
|
|
"tokens/trainable": 22107364
|
|
},
|
|
{
|
|
"epoch": 1.634484909927474,
|
|
"grad_norm": 0.14098578691482544,
|
|
"learning_rate": 4.2607328314709595e-06,
|
|
"loss": 1.535930061340332,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.64564,
|
|
"step": 5240,
|
|
"tokens/total": 34407168,
|
|
"tokens/train_per_sec_per_gpu": 95.67,
|
|
"tokens/trainable": 22152080
|
|
},
|
|
{
|
|
"epoch": 1.6376043047648756,
|
|
"grad_norm": 0.14594222605228424,
|
|
"learning_rate": 4.190488391656436e-06,
|
|
"loss": 1.600592041015625,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.95597,
|
|
"step": 5250,
|
|
"tokens/total": 34470016,
|
|
"tokens/train_per_sec_per_gpu": 56.47,
|
|
"tokens/trainable": 22192602
|
|
},
|
|
{
|
|
"epoch": 1.6407236996022772,
|
|
"grad_norm": 0.14143657684326172,
|
|
"learning_rate": 4.120774811765654e-06,
|
|
"loss": 1.5294102668762206,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.61545,
|
|
"step": 5260,
|
|
"tokens/total": 34535808,
|
|
"tokens/train_per_sec_per_gpu": 66.82,
|
|
"tokens/trainable": 22234892
|
|
},
|
|
{
|
|
"epoch": 1.6438430944396787,
|
|
"grad_norm": 0.1540057361125946,
|
|
"learning_rate": 4.051593870223147e-06,
|
|
"loss": 1.499960231781006,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.48151,
|
|
"step": 5270,
|
|
"tokens/total": 34601344,
|
|
"tokens/train_per_sec_per_gpu": 91.44,
|
|
"tokens/trainable": 22278184
|
|
},
|
|
{
|
|
"epoch": 1.6469624892770802,
|
|
"grad_norm": 0.14380136132240295,
|
|
"learning_rate": 3.982947331865577e-06,
|
|
"loss": 1.5885168075561524,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.89648,
|
|
"step": 5280,
|
|
"tokens/total": 34667008,
|
|
"tokens/train_per_sec_per_gpu": 69.49,
|
|
"tokens/trainable": 22320908
|
|
},
|
|
{
|
|
"epoch": 1.6500818841144818,
|
|
"grad_norm": 0.1258169710636139,
|
|
"learning_rate": 3.914836947896777e-06,
|
|
"loss": 1.5533151626586914,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.72712,
|
|
"step": 5290,
|
|
"tokens/total": 34734464,
|
|
"tokens/train_per_sec_per_gpu": 77.51,
|
|
"tokens/trainable": 22364802
|
|
},
|
|
{
|
|
"epoch": 1.6532012789518833,
|
|
"grad_norm": 0.13773156702518463,
|
|
"learning_rate": 3.847264455843003e-06,
|
|
"loss": 1.5653531074523925,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78436,
|
|
"step": 5300,
|
|
"tokens/total": 34801920,
|
|
"tokens/train_per_sec_per_gpu": 92.33,
|
|
"tokens/trainable": 22408560
|
|
},
|
|
{
|
|
"epoch": 1.6563206737892848,
|
|
"grad_norm": 0.16436602175235748,
|
|
"learning_rate": 3.7802315795086947e-06,
|
|
"loss": 1.6201396942138673,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.0538,
|
|
"step": 5310,
|
|
"tokens/total": 34866432,
|
|
"tokens/train_per_sec_per_gpu": 61.77,
|
|
"tokens/trainable": 22448928
|
|
},
|
|
{
|
|
"epoch": 1.6594400686266866,
|
|
"grad_norm": 0.1402582973241806,
|
|
"learning_rate": 3.713740028932411e-06,
|
|
"loss": 1.5824259757995605,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.86675,
|
|
"step": 5320,
|
|
"tokens/total": 34930048,
|
|
"tokens/train_per_sec_per_gpu": 59.57,
|
|
"tokens/trainable": 22491588
|
|
},
|
|
{
|
|
"epoch": 1.662559463464088,
|
|
"grad_norm": 0.1327056735754013,
|
|
"learning_rate": 3.6477915003432755e-06,
|
|
"loss": 1.5920062065124512,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.9136,
|
|
"step": 5330,
|
|
"tokens/total": 34995456,
|
|
"tokens/train_per_sec_per_gpu": 78.93,
|
|
"tokens/trainable": 22534560
|
|
},
|
|
{
|
|
"epoch": 1.6656788583014897,
|
|
"grad_norm": 0.13299594819545746,
|
|
"learning_rate": 3.5823876761176656e-06,
|
|
"loss": 1.4854873657226562,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.41712,
|
|
"step": 5340,
|
|
"tokens/total": 35061504,
|
|
"tokens/train_per_sec_per_gpu": 89.22,
|
|
"tokens/trainable": 22577080
|
|
},
|
|
{
|
|
"epoch": 1.668798253138891,
|
|
"grad_norm": 0.13764125108718872,
|
|
"learning_rate": 3.517530224736321e-06,
|
|
"loss": 1.5484750747680665,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.70429,
|
|
"step": 5350,
|
|
"tokens/total": 35125248,
|
|
"tokens/train_per_sec_per_gpu": 64.74,
|
|
"tokens/trainable": 22620712
|
|
},
|
|
{
|
|
"epoch": 1.6719176479762927,
|
|
"grad_norm": 0.14515449106693268,
|
|
"learning_rate": 3.453220800741749e-06,
|
|
"loss": 1.557614517211914,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.74748,
|
|
"step": 5360,
|
|
"tokens/total": 35188480,
|
|
"tokens/train_per_sec_per_gpu": 53.41,
|
|
"tokens/trainable": 22660756
|
|
},
|
|
{
|
|
"epoch": 1.675037042813694,
|
|
"grad_norm": 0.13613158464431763,
|
|
"learning_rate": 3.389461044696038e-06,
|
|
"loss": 1.5452713012695312,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68924,
|
|
"step": 5370,
|
|
"tokens/total": 35250304,
|
|
"tokens/train_per_sec_per_gpu": 79.74,
|
|
"tokens/trainable": 22701630
|
|
},
|
|
{
|
|
"epoch": 1.6781564376510958,
|
|
"grad_norm": 0.1321331411600113,
|
|
"learning_rate": 3.3262525831390225e-06,
|
|
"loss": 1.4688044548034669,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.34404,
|
|
"step": 5380,
|
|
"tokens/total": 35315328,
|
|
"tokens/train_per_sec_per_gpu": 42.45,
|
|
"tokens/trainable": 22743536
|
|
},
|
|
{
|
|
"epoch": 1.681275832488497,
|
|
"grad_norm": 0.1552678644657135,
|
|
"learning_rate": 3.263597028546739e-06,
|
|
"loss": 1.5998359680175782,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.95222,
|
|
"step": 5390,
|
|
"tokens/total": 35380096,
|
|
"tokens/train_per_sec_per_gpu": 86.93,
|
|
"tokens/trainable": 22786904
|
|
},
|
|
{
|
|
"epoch": 1.6843952273258989,
|
|
"grad_norm": 0.13293081521987915,
|
|
"learning_rate": 3.2014959792903422e-06,
|
|
"loss": 1.5894336700439453,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90097,
|
|
"step": 5400,
|
|
"tokens/total": 35446272,
|
|
"tokens/train_per_sec_per_gpu": 85.48,
|
|
"tokens/trainable": 22830102
|
|
},
|
|
{
|
|
"epoch": 1.6875146221633002,
|
|
"grad_norm": 0.12856332957744598,
|
|
"learning_rate": 3.1399510195952942e-06,
|
|
"loss": 1.5064952850341797,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.51089,
|
|
"step": 5410,
|
|
"tokens/total": 35515008,
|
|
"tokens/train_per_sec_per_gpu": 90.48,
|
|
"tokens/trainable": 22876000
|
|
},
|
|
{
|
|
"epoch": 1.690634017000702,
|
|
"grad_norm": 0.14628250896930695,
|
|
"learning_rate": 3.0789637195009796e-06,
|
|
"loss": 1.5182460784912108,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.56421,
|
|
"step": 5420,
|
|
"tokens/total": 35583872,
|
|
"tokens/train_per_sec_per_gpu": 90.88,
|
|
"tokens/trainable": 22919522
|
|
},
|
|
{
|
|
"epoch": 1.6937534118381035,
|
|
"grad_norm": 0.14349891245365143,
|
|
"learning_rate": 3.0185356348206155e-06,
|
|
"loss": 1.5152307510375977,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.55047,
|
|
"step": 5430,
|
|
"tokens/total": 35650816,
|
|
"tokens/train_per_sec_per_gpu": 48.64,
|
|
"tokens/trainable": 22961464
|
|
},
|
|
{
|
|
"epoch": 1.696872806675505,
|
|
"grad_norm": 0.1363019496202469,
|
|
"learning_rate": 2.958668307101606e-06,
|
|
"loss": 1.6042192459106446,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.97397,
|
|
"step": 5440,
|
|
"tokens/total": 35713408,
|
|
"tokens/train_per_sec_per_gpu": 57.17,
|
|
"tokens/trainable": 23002748
|
|
},
|
|
{
|
|
"epoch": 1.6999922015129065,
|
|
"grad_norm": 0.13265329599380493,
|
|
"learning_rate": 2.8993632635861883e-06,
|
|
"loss": 1.5074874877929687,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.51537,
|
|
"step": 5450,
|
|
"tokens/total": 35780608,
|
|
"tokens/train_per_sec_per_gpu": 59.04,
|
|
"tokens/trainable": 23045650
|
|
},
|
|
{
|
|
"epoch": 1.703111596350308,
|
|
"grad_norm": 0.14598093926906586,
|
|
"learning_rate": 2.84062201717247e-06,
|
|
"loss": 1.5543317794799805,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73192,
|
|
"step": 5460,
|
|
"tokens/total": 35847808,
|
|
"tokens/train_per_sec_per_gpu": 51.33,
|
|
"tokens/trainable": 23089666
|
|
},
|
|
{
|
|
"epoch": 1.7062309911877096,
|
|
"grad_norm": 0.14332054555416107,
|
|
"learning_rate": 2.7824460663758627e-06,
|
|
"loss": 1.529410457611084,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.61546,
|
|
"step": 5470,
|
|
"tokens/total": 35914112,
|
|
"tokens/train_per_sec_per_gpu": 98.92,
|
|
"tokens/trainable": 23132816
|
|
},
|
|
{
|
|
"epoch": 1.7093503860251111,
|
|
"grad_norm": 0.14680002629756927,
|
|
"learning_rate": 2.7248368952908053e-06,
|
|
"loss": 1.5988737106323243,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.94746,
|
|
"step": 5480,
|
|
"tokens/total": 35979904,
|
|
"tokens/train_per_sec_per_gpu": 80.84,
|
|
"tokens/trainable": 23176280
|
|
},
|
|
{
|
|
"epoch": 1.7124697808625127,
|
|
"grad_norm": 0.13773256540298462,
|
|
"learning_rate": 2.667795973552964e-06,
|
|
"loss": 1.552729034423828,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.72435,
|
|
"step": 5490,
|
|
"tokens/total": 36049024,
|
|
"tokens/train_per_sec_per_gpu": 80.26,
|
|
"tokens/trainable": 23221252
|
|
},
|
|
{
|
|
"epoch": 1.7155891756999142,
|
|
"grad_norm": 0.1369321346282959,
|
|
"learning_rate": 2.6113247563016864e-06,
|
|
"loss": 1.5542631149291992,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.7316,
|
|
"step": 5500,
|
|
"tokens/total": 36115712,
|
|
"tokens/train_per_sec_per_gpu": 55.46,
|
|
"tokens/trainable": 23263712
|
|
},
|
|
{
|
|
"epoch": 1.7187085705373157,
|
|
"grad_norm": 0.1484479159116745,
|
|
"learning_rate": 2.5554246841429206e-06,
|
|
"loss": 1.5419631004333496,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.67376,
|
|
"step": 5510,
|
|
"tokens/total": 36181888,
|
|
"tokens/train_per_sec_per_gpu": 88.14,
|
|
"tokens/trainable": 23305358
|
|
},
|
|
{
|
|
"epoch": 1.7218279653747173,
|
|
"grad_norm": 0.13669778406620026,
|
|
"learning_rate": 2.5000971831124433e-06,
|
|
"loss": 1.5883105278015137,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.89547,
|
|
"step": 5520,
|
|
"tokens/total": 36247424,
|
|
"tokens/train_per_sec_per_gpu": 73.7,
|
|
"tokens/trainable": 23347906
|
|
},
|
|
{
|
|
"epoch": 1.7249473602121188,
|
|
"grad_norm": 0.1643878072500229,
|
|
"learning_rate": 2.445343664639488e-06,
|
|
"loss": 1.4853872299194335,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.41668,
|
|
"step": 5530,
|
|
"tokens/total": 36315136,
|
|
"tokens/train_per_sec_per_gpu": 78.69,
|
|
"tokens/trainable": 23389738
|
|
},
|
|
{
|
|
"epoch": 1.7280667550495203,
|
|
"grad_norm": 0.15027537941932678,
|
|
"learning_rate": 2.3911655255107302e-06,
|
|
"loss": 1.5624693870544433,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.77059,
|
|
"step": 5540,
|
|
"tokens/total": 36379392,
|
|
"tokens/train_per_sec_per_gpu": 63.68,
|
|
"tokens/trainable": 23431980
|
|
},
|
|
{
|
|
"epoch": 1.731186149886922,
|
|
"grad_norm": 0.15651673078536987,
|
|
"learning_rate": 2.3375641478346725e-06,
|
|
"loss": 1.470098876953125,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.34967,
|
|
"step": 5550,
|
|
"tokens/total": 36445696,
|
|
"tokens/train_per_sec_per_gpu": 72.4,
|
|
"tokens/trainable": 23474432
|
|
},
|
|
{
|
|
"epoch": 1.7343055447243234,
|
|
"grad_norm": 0.15287302434444427,
|
|
"learning_rate": 2.2845408990063656e-06,
|
|
"loss": 1.526399803161621,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.60158,
|
|
"step": 5560,
|
|
"tokens/total": 36511616,
|
|
"tokens/train_per_sec_per_gpu": 49.16,
|
|
"tokens/trainable": 23516320
|
|
},
|
|
{
|
|
"epoch": 1.7374249395617252,
|
|
"grad_norm": 0.1470254808664322,
|
|
"learning_rate": 2.2320971316725307e-06,
|
|
"loss": 1.5744261741638184,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.82797,
|
|
"step": 5570,
|
|
"tokens/total": 36579200,
|
|
"tokens/train_per_sec_per_gpu": 59.24,
|
|
"tokens/trainable": 23559560
|
|
},
|
|
{
|
|
"epoch": 1.7405443343991265,
|
|
"grad_norm": 0.14781436324119568,
|
|
"learning_rate": 2.1802341836970856e-06,
|
|
"loss": 1.5240687370300292,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.59087,
|
|
"step": 5580,
|
|
"tokens/total": 36646784,
|
|
"tokens/train_per_sec_per_gpu": 69.51,
|
|
"tokens/trainable": 23604336
|
|
},
|
|
{
|
|
"epoch": 1.7436637292365282,
|
|
"grad_norm": 0.1489586979150772,
|
|
"learning_rate": 2.128953378126955e-06,
|
|
"loss": 1.5688559532165527,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.80115,
|
|
"step": 5590,
|
|
"tokens/total": 36711936,
|
|
"tokens/train_per_sec_per_gpu": 96.74,
|
|
"tokens/trainable": 23646240
|
|
},
|
|
{
|
|
"epoch": 1.7467831240739295,
|
|
"grad_norm": 0.13258112967014313,
|
|
"learning_rate": 2.0782560231583705e-06,
|
|
"loss": 1.5658369064331055,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.78668,
|
|
"step": 5600,
|
|
"tokens/total": 36778240,
|
|
"tokens/train_per_sec_per_gpu": 100.05,
|
|
"tokens/trainable": 23689276
|
|
},
|
|
{
|
|
"epoch": 1.7499025189113313,
|
|
"grad_norm": 0.13804979622364044,
|
|
"learning_rate": 2.0281434121034643e-06,
|
|
"loss": 1.5689093589782714,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.80141,
|
|
"step": 5610,
|
|
"tokens/total": 36846208,
|
|
"tokens/train_per_sec_per_gpu": 67.36,
|
|
"tokens/trainable": 23733184
|
|
},
|
|
{
|
|
"epoch": 1.7530219137487326,
|
|
"grad_norm": 0.14498800039291382,
|
|
"learning_rate": 1.9786168233573045e-06,
|
|
"loss": 1.5179577827453614,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.5629,
|
|
"step": 5620,
|
|
"tokens/total": 36908800,
|
|
"tokens/train_per_sec_per_gpu": 74.45,
|
|
"tokens/trainable": 23773244
|
|
},
|
|
{
|
|
"epoch": 1.7561413085861344,
|
|
"grad_norm": 0.15927951037883759,
|
|
"learning_rate": 1.929677520365261e-06,
|
|
"loss": 1.5996320724487305,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.95121,
|
|
"step": 5630,
|
|
"tokens/total": 36971904,
|
|
"tokens/train_per_sec_per_gpu": 103.26,
|
|
"tokens/trainable": 23813286
|
|
},
|
|
{
|
|
"epoch": 1.7592607034235357,
|
|
"grad_norm": 0.16359494626522064,
|
|
"learning_rate": 1.8813267515907812e-06,
|
|
"loss": 1.4822440147399902,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.40281,
|
|
"step": 5640,
|
|
"tokens/total": 37035648,
|
|
"tokens/train_per_sec_per_gpu": 38.55,
|
|
"tokens/trainable": 23854196
|
|
},
|
|
{
|
|
"epoch": 1.7623800982609374,
|
|
"grad_norm": 0.14311447739601135,
|
|
"learning_rate": 1.8335657504835514e-06,
|
|
"loss": 1.5773542404174805,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.84213,
|
|
"step": 5650,
|
|
"tokens/total": 37098624,
|
|
"tokens/train_per_sec_per_gpu": 65.27,
|
|
"tokens/trainable": 23896456
|
|
},
|
|
{
|
|
"epoch": 1.7654994930983388,
|
|
"grad_norm": 0.1315753310918808,
|
|
"learning_rate": 1.78639573544801e-06,
|
|
"loss": 1.5053918838500977,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.50592,
|
|
"step": 5660,
|
|
"tokens/total": 37167488,
|
|
"tokens/train_per_sec_per_gpu": 96.29,
|
|
"tokens/trainable": 23940426
|
|
},
|
|
{
|
|
"epoch": 1.7686188879357405,
|
|
"grad_norm": 0.14975935220718384,
|
|
"learning_rate": 1.739817909812283e-06,
|
|
"loss": 1.5183167457580566,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.56454,
|
|
"step": 5670,
|
|
"tokens/total": 37231360,
|
|
"tokens/train_per_sec_per_gpu": 88.42,
|
|
"tokens/trainable": 23981236
|
|
},
|
|
{
|
|
"epoch": 1.771738282773142,
|
|
"grad_norm": 0.13258233666419983,
|
|
"learning_rate": 1.6938334617974755e-06,
|
|
"loss": 1.5476274490356445,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.70031,
|
|
"step": 5680,
|
|
"tokens/total": 37296768,
|
|
"tokens/train_per_sec_per_gpu": 90.34,
|
|
"tokens/trainable": 24022612
|
|
},
|
|
{
|
|
"epoch": 1.7748576776105436,
|
|
"grad_norm": 0.13746336102485657,
|
|
"learning_rate": 1.6484435644873785e-06,
|
|
"loss": 1.5870176315307618,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.88915,
|
|
"step": 5690,
|
|
"tokens/total": 37363072,
|
|
"tokens/train_per_sec_per_gpu": 66.28,
|
|
"tokens/trainable": 24064622
|
|
},
|
|
{
|
|
"epoch": 1.7779770724479451,
|
|
"grad_norm": 0.13110321760177612,
|
|
"learning_rate": 1.6036493757985105e-06,
|
|
"loss": 1.600935173034668,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.95767,
|
|
"step": 5700,
|
|
"tokens/total": 37430528,
|
|
"tokens/train_per_sec_per_gpu": 50.43,
|
|
"tokens/trainable": 24108916
|
|
},
|
|
{
|
|
"epoch": 1.7810964672853467,
|
|
"grad_norm": 0.15515461564064026,
|
|
"learning_rate": 1.5594520384506096e-06,
|
|
"loss": 1.5411881446838378,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.67014,
|
|
"step": 5710,
|
|
"tokens/total": 37492864,
|
|
"tokens/train_per_sec_per_gpu": 67.09,
|
|
"tokens/trainable": 24148524
|
|
},
|
|
{
|
|
"epoch": 1.7842158621227482,
|
|
"grad_norm": 0.14537785947322845,
|
|
"learning_rate": 1.5158526799374655e-06,
|
|
"loss": 1.458988857269287,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.30161,
|
|
"step": 5720,
|
|
"tokens/total": 37558400,
|
|
"tokens/train_per_sec_per_gpu": 95.45,
|
|
"tokens/trainable": 24190076
|
|
},
|
|
{
|
|
"epoch": 1.7873352569601497,
|
|
"grad_norm": 0.15173108875751495,
|
|
"learning_rate": 1.4728524124981685e-06,
|
|
"loss": 1.554797649383545,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73413,
|
|
"step": 5730,
|
|
"tokens/total": 37624576,
|
|
"tokens/train_per_sec_per_gpu": 56.01,
|
|
"tokens/trainable": 24232818
|
|
},
|
|
{
|
|
"epoch": 1.7904546517975513,
|
|
"grad_norm": 0.13938990235328674,
|
|
"learning_rate": 1.4304523330887194e-06,
|
|
"loss": 1.5674165725708007,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.79425,
|
|
"step": 5740,
|
|
"tokens/total": 37691520,
|
|
"tokens/train_per_sec_per_gpu": 55.03,
|
|
"tokens/trainable": 24275714
|
|
},
|
|
{
|
|
"epoch": 1.7935740466349528,
|
|
"grad_norm": 0.15122099220752716,
|
|
"learning_rate": 1.3886535233540526e-06,
|
|
"loss": 1.5342636108398438,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.63791,
|
|
"step": 5750,
|
|
"tokens/total": 37758208,
|
|
"tokens/train_per_sec_per_gpu": 81.11,
|
|
"tokens/trainable": 24317012
|
|
},
|
|
{
|
|
"epoch": 1.7966934414723543,
|
|
"grad_norm": 0.14161832630634308,
|
|
"learning_rate": 1.3474570496004563e-06,
|
|
"loss": 1.5800642013549804,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.85527,
|
|
"step": 5760,
|
|
"tokens/total": 37820544,
|
|
"tokens/train_per_sec_per_gpu": 48.33,
|
|
"tokens/trainable": 24356978
|
|
},
|
|
{
|
|
"epoch": 1.7998128363097559,
|
|
"grad_norm": 0.13030937314033508,
|
|
"learning_rate": 1.3068639627683483e-06,
|
|
"loss": 1.554422092437744,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73235,
|
|
"step": 5770,
|
|
"tokens/total": 37887616,
|
|
"tokens/train_per_sec_per_gpu": 102.66,
|
|
"tokens/trainable": 24400030
|
|
},
|
|
{
|
|
"epoch": 1.8029322311471576,
|
|
"grad_norm": 0.13343578577041626,
|
|
"learning_rate": 1.2668752984054788e-06,
|
|
"loss": 1.5433303833007812,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68015,
|
|
"step": 5780,
|
|
"tokens/total": 37953536,
|
|
"tokens/train_per_sec_per_gpu": 48.1,
|
|
"tokens/trainable": 24441712
|
|
},
|
|
{
|
|
"epoch": 1.806051625984559,
|
|
"grad_norm": 0.15799951553344727,
|
|
"learning_rate": 1.2274920766405212e-06,
|
|
"loss": 1.522444248199463,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.58341,
|
|
"step": 5790,
|
|
"tokens/total": 38019072,
|
|
"tokens/train_per_sec_per_gpu": 58.73,
|
|
"tokens/trainable": 24482784
|
|
},
|
|
{
|
|
"epoch": 1.8091710208219607,
|
|
"grad_norm": 0.14375533163547516,
|
|
"learning_rate": 1.1887153021570314e-06,
|
|
"loss": 1.5162397384643556,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.55506,
|
|
"step": 5800,
|
|
"tokens/total": 38084608,
|
|
"tokens/train_per_sec_per_gpu": 55.75,
|
|
"tokens/trainable": 24525104
|
|
},
|
|
{
|
|
"epoch": 1.812290415659362,
|
|
"grad_norm": 0.13492465019226074,
|
|
"learning_rate": 1.1505459641678151e-06,
|
|
"loss": 1.541007900238037,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.66929,
|
|
"step": 5810,
|
|
"tokens/total": 38151424,
|
|
"tokens/train_per_sec_per_gpu": 91.03,
|
|
"tokens/trainable": 24569996
|
|
},
|
|
{
|
|
"epoch": 1.8154098104967638,
|
|
"grad_norm": 0.13006772100925446,
|
|
"learning_rate": 1.1129850363897187e-06,
|
|
"loss": 1.5848455429077148,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.87854,
|
|
"step": 5820,
|
|
"tokens/total": 38215296,
|
|
"tokens/train_per_sec_per_gpu": 28.47,
|
|
"tokens/trainable": 24611504
|
|
},
|
|
{
|
|
"epoch": 1.818529205334165,
|
|
"grad_norm": 0.15029467642307281,
|
|
"learning_rate": 1.0760334770187564e-06,
|
|
"loss": 1.4929359436035157,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.45014,
|
|
"step": 5830,
|
|
"tokens/total": 38282496,
|
|
"tokens/train_per_sec_per_gpu": 56.56,
|
|
"tokens/trainable": 24656498
|
|
},
|
|
{
|
|
"epoch": 1.8216486001715668,
|
|
"grad_norm": 0.1460971236228943,
|
|
"learning_rate": 1.039692228705702e-06,
|
|
"loss": 1.6408479690551758,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.15954,
|
|
"step": 5840,
|
|
"tokens/total": 38344832,
|
|
"tokens/train_per_sec_per_gpu": 56.51,
|
|
"tokens/trainable": 24696188
|
|
},
|
|
{
|
|
"epoch": 1.8247679950089681,
|
|
"grad_norm": 0.14317525923252106,
|
|
"learning_rate": 1.0039622185320009e-06,
|
|
"loss": 1.646989631652832,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.19133,
|
|
"step": 5850,
|
|
"tokens/total": 38411648,
|
|
"tokens/train_per_sec_per_gpu": 62.02,
|
|
"tokens/trainable": 24740292
|
|
},
|
|
{
|
|
"epoch": 1.82788738984637,
|
|
"grad_norm": 0.14138714969158173,
|
|
"learning_rate": 9.688443579861566e-07,
|
|
"loss": 1.5335498809814454,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.6346,
|
|
"step": 5860,
|
|
"tokens/total": 38477696,
|
|
"tokens/train_per_sec_per_gpu": 96.63,
|
|
"tokens/trainable": 24783600
|
|
},
|
|
{
|
|
"epoch": 1.8310067846837712,
|
|
"grad_norm": 0.1352611631155014,
|
|
"learning_rate": 9.343395429404573e-07,
|
|
"loss": 1.529552173614502,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.61611,
|
|
"step": 5870,
|
|
"tokens/total": 38545280,
|
|
"tokens/train_per_sec_per_gpu": 87.69,
|
|
"tokens/trainable": 24826298
|
|
},
|
|
{
|
|
"epoch": 1.834126179521173,
|
|
"grad_norm": 0.13490384817123413,
|
|
"learning_rate": 9.004486536281193e-07,
|
|
"loss": 1.5804509162902831,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.85715,
|
|
"step": 5880,
|
|
"tokens/total": 38611072,
|
|
"tokens/train_per_sec_per_gpu": 100.2,
|
|
"tokens/trainable": 24869144
|
|
},
|
|
{
|
|
"epoch": 1.8372455743585743,
|
|
"grad_norm": 0.14591826498508453,
|
|
"learning_rate": 8.67172554620857e-07,
|
|
"loss": 1.5936957359313966,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.92191,
|
|
"step": 5890,
|
|
"tokens/total": 38675072,
|
|
"tokens/train_per_sec_per_gpu": 71.21,
|
|
"tokens/trainable": 24909268
|
|
},
|
|
{
|
|
"epoch": 1.840364969195976,
|
|
"grad_norm": 0.13786549866199493,
|
|
"learning_rate": 8.345120948067914e-07,
|
|
"loss": 1.4800718307495118,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.39326,
|
|
"step": 5900,
|
|
"tokens/total": 38742784,
|
|
"tokens/train_per_sec_per_gpu": 73.16,
|
|
"tokens/trainable": 24953342
|
|
},
|
|
{
|
|
"epoch": 1.8434843640333776,
|
|
"grad_norm": 0.13986949622631073,
|
|
"learning_rate": 8.024681073688284e-07,
|
|
"loss": 1.5094839096069337,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.5244,
|
|
"step": 5910,
|
|
"tokens/total": 38810496,
|
|
"tokens/train_per_sec_per_gpu": 106.83,
|
|
"tokens/trainable": 24997376
|
|
},
|
|
{
|
|
"epoch": 1.846603758870779,
|
|
"grad_norm": 0.13499020040035248,
|
|
"learning_rate": 7.710414097633734e-07,
|
|
"loss": 1.5841424942016602,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.87511,
|
|
"step": 5920,
|
|
"tokens/total": 38877440,
|
|
"tokens/train_per_sec_per_gpu": 85.64,
|
|
"tokens/trainable": 25040820
|
|
},
|
|
{
|
|
"epoch": 1.8497231537081806,
|
|
"grad_norm": 0.1478787064552307,
|
|
"learning_rate": 7.402328036995116e-07,
|
|
"loss": 1.5825369834899903,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.86729,
|
|
"step": 5930,
|
|
"tokens/total": 38942592,
|
|
"tokens/train_per_sec_per_gpu": 63.34,
|
|
"tokens/trainable": 25081256
|
|
},
|
|
{
|
|
"epoch": 1.8528425485455822,
|
|
"grad_norm": 0.15646716952323914,
|
|
"learning_rate": 7.100430751185188e-07,
|
|
"loss": 1.588590431213379,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.89684,
|
|
"step": 5940,
|
|
"tokens/total": 39004544,
|
|
"tokens/train_per_sec_per_gpu": 65.22,
|
|
"tokens/trainable": 25120132
|
|
},
|
|
{
|
|
"epoch": 1.8559619433829837,
|
|
"grad_norm": 0.20769719779491425,
|
|
"learning_rate": 6.804729941738363e-07,
|
|
"loss": 1.6100143432617187,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.00288,
|
|
"step": 5950,
|
|
"tokens/total": 39073152,
|
|
"tokens/train_per_sec_per_gpu": 82.6,
|
|
"tokens/trainable": 25164614
|
|
},
|
|
{
|
|
"epoch": 1.8590813382203852,
|
|
"grad_norm": 0.12530042231082916,
|
|
"learning_rate": 6.515233152114275e-07,
|
|
"loss": 1.4825664520263673,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.40423,
|
|
"step": 5960,
|
|
"tokens/total": 39141760,
|
|
"tokens/train_per_sec_per_gpu": 74.66,
|
|
"tokens/trainable": 25208248
|
|
},
|
|
{
|
|
"epoch": 1.8622007330577868,
|
|
"grad_norm": 0.1455196887254715,
|
|
"learning_rate": 6.231947767505103e-07,
|
|
"loss": 1.606389045715332,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.98478,
|
|
"step": 5970,
|
|
"tokens/total": 39205376,
|
|
"tokens/train_per_sec_per_gpu": 73.57,
|
|
"tokens/trainable": 25248856
|
|
},
|
|
{
|
|
"epoch": 1.8653201278951883,
|
|
"grad_norm": 0.13254357874393463,
|
|
"learning_rate": 5.95488101464739e-07,
|
|
"loss": 1.555849552154541,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73911,
|
|
"step": 5980,
|
|
"tokens/total": 39269632,
|
|
"tokens/train_per_sec_per_gpu": 90.92,
|
|
"tokens/trainable": 25289896
|
|
},
|
|
{
|
|
"epoch": 1.8684395227325898,
|
|
"grad_norm": 0.16078540682792664,
|
|
"learning_rate": 5.684039961637578e-07,
|
|
"loss": 1.5423778533935546,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.6757,
|
|
"step": 5990,
|
|
"tokens/total": 39333760,
|
|
"tokens/train_per_sec_per_gpu": 52.01,
|
|
"tokens/trainable": 25331266
|
|
},
|
|
{
|
|
"epoch": 1.8715589175699914,
|
|
"grad_norm": 0.1577766090631485,
|
|
"learning_rate": 5.419431517751788e-07,
|
|
"loss": 1.587224006652832,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.89016,
|
|
"step": 6000,
|
|
"tokens/total": 39398016,
|
|
"tokens/train_per_sec_per_gpu": 67.36,
|
|
"tokens/trainable": 25371256
|
|
},
|
|
{
|
|
"epoch": 1.874678312407393,
|
|
"grad_norm": 0.1444595903158188,
|
|
"learning_rate": 5.161062433269442e-07,
|
|
"loss": 1.5748126983642579,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.82984,
|
|
"step": 6010,
|
|
"tokens/total": 39462400,
|
|
"tokens/train_per_sec_per_gpu": 83.36,
|
|
"tokens/trainable": 25412004
|
|
},
|
|
{
|
|
"epoch": 1.8777977072447944,
|
|
"grad_norm": 0.12568695843219757,
|
|
"learning_rate": 4.908939299301112e-07,
|
|
"loss": 1.6056318283081055,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.98101,
|
|
"step": 6020,
|
|
"tokens/total": 39528704,
|
|
"tokens/train_per_sec_per_gpu": 78.24,
|
|
"tokens/trainable": 25454198
|
|
},
|
|
{
|
|
"epoch": 1.8809171020821962,
|
|
"grad_norm": 0.13505080342292786,
|
|
"learning_rate": 4.6630685476204417e-07,
|
|
"loss": 1.5776721954345703,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.84367,
|
|
"step": 6030,
|
|
"tokens/total": 39596160,
|
|
"tokens/train_per_sec_per_gpu": 62.65,
|
|
"tokens/trainable": 25498278
|
|
},
|
|
{
|
|
"epoch": 1.8840364969195975,
|
|
"grad_norm": 0.12592627108097076,
|
|
"learning_rate": 4.423456450499991e-07,
|
|
"loss": 1.5907666206359863,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90751,
|
|
"step": 6040,
|
|
"tokens/total": 39661440,
|
|
"tokens/train_per_sec_per_gpu": 47.63,
|
|
"tokens/trainable": 25540312
|
|
},
|
|
{
|
|
"epoch": 1.8871558917569993,
|
|
"grad_norm": 0.14127464592456818,
|
|
"learning_rate": 4.190109120551178e-07,
|
|
"loss": 1.5563036918640136,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.74126,
|
|
"step": 6050,
|
|
"tokens/total": 39726848,
|
|
"tokens/train_per_sec_per_gpu": 97.25,
|
|
"tokens/trainable": 25582626
|
|
},
|
|
{
|
|
"epoch": 1.8902752865944006,
|
|
"grad_norm": 0.14158284664154053,
|
|
"learning_rate": 3.963032510568537e-07,
|
|
"loss": 1.5212724685668946,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.57805,
|
|
"step": 6060,
|
|
"tokens/total": 39792256,
|
|
"tokens/train_per_sec_per_gpu": 50.81,
|
|
"tokens/trainable": 25624428
|
|
},
|
|
{
|
|
"epoch": 1.8933946814318023,
|
|
"grad_norm": 0.12947708368301392,
|
|
"learning_rate": 3.742232413377705e-07,
|
|
"loss": 1.540519905090332,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.66702,
|
|
"step": 6070,
|
|
"tokens/total": 39859584,
|
|
"tokens/train_per_sec_per_gpu": 63.27,
|
|
"tokens/trainable": 25667682
|
|
},
|
|
{
|
|
"epoch": 1.8965140762692037,
|
|
"grad_norm": 0.1475353091955185,
|
|
"learning_rate": 3.5277144616875925e-07,
|
|
"loss": 1.597586441040039,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.94109,
|
|
"step": 6080,
|
|
"tokens/total": 39924224,
|
|
"tokens/train_per_sec_per_gpu": 95.9,
|
|
"tokens/trainable": 25708688
|
|
},
|
|
{
|
|
"epoch": 1.8996334711066054,
|
|
"grad_norm": 0.13512283563613892,
|
|
"learning_rate": 3.319484127946887e-07,
|
|
"loss": 1.5522504806518556,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.72209,
|
|
"step": 6090,
|
|
"tokens/total": 39991808,
|
|
"tokens/train_per_sec_per_gpu": 35.45,
|
|
"tokens/trainable": 25751824
|
|
},
|
|
{
|
|
"epoch": 1.9027528659440067,
|
|
"grad_norm": 0.15205733478069305,
|
|
"learning_rate": 3.117546724204251e-07,
|
|
"loss": 1.6299901962280274,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.10382,
|
|
"step": 6100,
|
|
"tokens/total": 40056832,
|
|
"tokens/train_per_sec_per_gpu": 71.15,
|
|
"tokens/trainable": 25794490
|
|
},
|
|
{
|
|
"epoch": 1.9058722607814085,
|
|
"grad_norm": 0.12750151753425598,
|
|
"learning_rate": 2.9219074019729563e-07,
|
|
"loss": 1.5442309379577637,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.68437,
|
|
"step": 6110,
|
|
"tokens/total": 40121728,
|
|
"tokens/train_per_sec_per_gpu": 84.18,
|
|
"tokens/trainable": 25836580
|
|
},
|
|
{
|
|
"epoch": 1.9089916556188098,
|
|
"grad_norm": 0.13145914673805237,
|
|
"learning_rate": 2.732571152099378e-07,
|
|
"loss": 1.5119094848632812,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.53538,
|
|
"step": 6120,
|
|
"tokens/total": 40188160,
|
|
"tokens/train_per_sec_per_gpu": 69.69,
|
|
"tokens/trainable": 25880672
|
|
},
|
|
{
|
|
"epoch": 1.9121110504562115,
|
|
"grad_norm": 0.13738512992858887,
|
|
"learning_rate": 2.5495428046356795e-07,
|
|
"loss": 1.620722770690918,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.05674,
|
|
"step": 6130,
|
|
"tokens/total": 40253568,
|
|
"tokens/train_per_sec_per_gpu": 63.52,
|
|
"tokens/trainable": 25923794
|
|
},
|
|
{
|
|
"epoch": 1.915230445293613,
|
|
"grad_norm": 0.1462392657995224,
|
|
"learning_rate": 2.3728270287167176e-07,
|
|
"loss": 1.5197964668273927,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.57129,
|
|
"step": 6140,
|
|
"tokens/total": 40319488,
|
|
"tokens/train_per_sec_per_gpu": 61.7,
|
|
"tokens/trainable": 25965226
|
|
},
|
|
{
|
|
"epoch": 1.9183498401310146,
|
|
"grad_norm": 0.13698002696037292,
|
|
"learning_rate": 2.2024283324406923e-07,
|
|
"loss": 1.590304470062256,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90524,
|
|
"step": 6150,
|
|
"tokens/total": 40386304,
|
|
"tokens/train_per_sec_per_gpu": 72.96,
|
|
"tokens/trainable": 26008212
|
|
},
|
|
{
|
|
"epoch": 1.9214692349684162,
|
|
"grad_norm": 0.1523156762123108,
|
|
"learning_rate": 2.0383510627544345e-07,
|
|
"loss": 1.5485605239868163,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.70469,
|
|
"step": 6160,
|
|
"tokens/total": 40453504,
|
|
"tokens/train_per_sec_per_gpu": 69.43,
|
|
"tokens/trainable": 26052264
|
|
},
|
|
{
|
|
"epoch": 1.9245886298058177,
|
|
"grad_norm": 0.1376800388097763,
|
|
"learning_rate": 1.880599405342187e-07,
|
|
"loss": 1.6177717208862306,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 5.04184,
|
|
"step": 6170,
|
|
"tokens/total": 40518272,
|
|
"tokens/train_per_sec_per_gpu": 58.47,
|
|
"tokens/trainable": 26093908
|
|
},
|
|
{
|
|
"epoch": 1.9277080246432192,
|
|
"grad_norm": 0.14166291058063507,
|
|
"learning_rate": 1.7291773845191928e-07,
|
|
"loss": 1.502146339416504,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.49132,
|
|
"step": 6180,
|
|
"tokens/total": 40582912,
|
|
"tokens/train_per_sec_per_gpu": 84.33,
|
|
"tokens/trainable": 26135322
|
|
},
|
|
{
|
|
"epoch": 1.9308274194806208,
|
|
"grad_norm": 0.13489550352096558,
|
|
"learning_rate": 1.5840888631286632e-07,
|
|
"loss": 1.6604825973510742,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.26185,
|
|
"step": 6190,
|
|
"tokens/total": 40650368,
|
|
"tokens/train_per_sec_per_gpu": 59.92,
|
|
"tokens/trainable": 26179588
|
|
},
|
|
{
|
|
"epoch": 1.9339468143180223,
|
|
"grad_norm": 0.14025822281837463,
|
|
"learning_rate": 1.4453375424435812e-07,
|
|
"loss": 1.5969094276428222,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.93775,
|
|
"step": 6200,
|
|
"tokens/total": 40715776,
|
|
"tokens/train_per_sec_per_gpu": 82.36,
|
|
"tokens/trainable": 26221568
|
|
},
|
|
{
|
|
"epoch": 1.9370662091554238,
|
|
"grad_norm": 0.15891960263252258,
|
|
"learning_rate": 1.3129269620719698e-07,
|
|
"loss": 1.6056411743164063,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.98105,
|
|
"step": 6210,
|
|
"tokens/total": 40778624,
|
|
"tokens/train_per_sec_per_gpu": 82.65,
|
|
"tokens/trainable": 26262420
|
|
},
|
|
{
|
|
"epoch": 1.9401856039928254,
|
|
"grad_norm": 0.15552055835723877,
|
|
"learning_rate": 1.1868604998668265e-07,
|
|
"loss": 1.5012582778930663,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.48733,
|
|
"step": 6220,
|
|
"tokens/total": 40844160,
|
|
"tokens/train_per_sec_per_gpu": 78.07,
|
|
"tokens/trainable": 26305176
|
|
},
|
|
{
|
|
"epoch": 1.943304998830227,
|
|
"grad_norm": 0.14225958287715912,
|
|
"learning_rate": 1.0671413718398582e-07,
|
|
"loss": 1.5261828422546386,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.60058,
|
|
"step": 6230,
|
|
"tokens/total": 40908672,
|
|
"tokens/train_per_sec_per_gpu": 56.33,
|
|
"tokens/trainable": 26347164
|
|
},
|
|
{
|
|
"epoch": 1.9464243936676284,
|
|
"grad_norm": 0.1386898010969162,
|
|
"learning_rate": 9.537726320793805e-08,
|
|
"loss": 1.5625441551208497,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.77094,
|
|
"step": 6240,
|
|
"tokens/total": 40972928,
|
|
"tokens/train_per_sec_per_gpu": 97.97,
|
|
"tokens/trainable": 26388458
|
|
},
|
|
{
|
|
"epoch": 1.94954378850503,
|
|
"grad_norm": 0.14062097668647766,
|
|
"learning_rate": 8.467571726725188e-08,
|
|
"loss": 1.469991970062256,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.3492,
|
|
"step": 6250,
|
|
"tokens/total": 41036800,
|
|
"tokens/train_per_sec_per_gpu": 82.21,
|
|
"tokens/trainable": 26430166
|
|
},
|
|
{
|
|
"epoch": 1.9526631833424317,
|
|
"grad_norm": 0.1507580727338791,
|
|
"learning_rate": 7.460977236314336e-08,
|
|
"loss": 1.6075920104980468,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.99078,
|
|
"step": 6260,
|
|
"tokens/total": 41100032,
|
|
"tokens/train_per_sec_per_gpu": 53.63,
|
|
"tokens/trainable": 26471144
|
|
},
|
|
{
|
|
"epoch": 1.955782578179833,
|
|
"grad_norm": 0.15054067969322205,
|
|
"learning_rate": 6.517968528235163e-08,
|
|
"loss": 1.5907088279724122,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.90723,
|
|
"step": 6270,
|
|
"tokens/total": 41165696,
|
|
"tokens/train_per_sec_per_gpu": 53.12,
|
|
"tokens/trainable": 26511744
|
|
},
|
|
{
|
|
"epoch": 1.9589019730172348,
|
|
"grad_norm": 0.15668627619743347,
|
|
"learning_rate": 5.6385696590605106e-08,
|
|
"loss": 1.507272434234619,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.5144,
|
|
"step": 6280,
|
|
"tokens/total": 41230720,
|
|
"tokens/train_per_sec_per_gpu": 84.16,
|
|
"tokens/trainable": 26553822
|
|
},
|
|
{
|
|
"epoch": 1.962021367854636,
|
|
"grad_norm": 0.1405741274356842,
|
|
"learning_rate": 4.822803062648207e-08,
|
|
"loss": 1.5356951713562013,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.64455,
|
|
"step": 6290,
|
|
"tokens/total": 41294336,
|
|
"tokens/train_per_sec_per_gpu": 56.49,
|
|
"tokens/trainable": 26595328
|
|
},
|
|
{
|
|
"epoch": 1.9651407626920379,
|
|
"grad_norm": 0.16112461686134338,
|
|
"learning_rate": 4.07068954956763e-08,
|
|
"loss": 1.569282054901123,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.8032,
|
|
"step": 6300,
|
|
"tokens/total": 41358848,
|
|
"tokens/train_per_sec_per_gpu": 78.11,
|
|
"tokens/trainable": 26636766
|
|
},
|
|
{
|
|
"epoch": 1.9682601575294392,
|
|
"grad_norm": 0.147075355052948,
|
|
"learning_rate": 3.382248306570135e-08,
|
|
"loss": 1.6198949813842773,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 5.05256,
|
|
"step": 6310,
|
|
"tokens/total": 41425152,
|
|
"tokens/train_per_sec_per_gpu": 36.08,
|
|
"tokens/trainable": 26679824
|
|
},
|
|
{
|
|
"epoch": 1.971379552366841,
|
|
"grad_norm": 0.13877691328525543,
|
|
"learning_rate": 2.757496896098888e-08,
|
|
"loss": 1.5246665000915527,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.59361,
|
|
"step": 6320,
|
|
"tokens/total": 41489792,
|
|
"tokens/train_per_sec_per_gpu": 66.01,
|
|
"tokens/trainable": 26720344
|
|
},
|
|
{
|
|
"epoch": 1.9744989472042422,
|
|
"grad_norm": 0.1317119151353836,
|
|
"learning_rate": 2.19645125584117e-08,
|
|
"loss": 1.5347830772399902,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.64032,
|
|
"step": 6330,
|
|
"tokens/total": 41557504,
|
|
"tokens/train_per_sec_per_gpu": 75.83,
|
|
"tokens/trainable": 26764418
|
|
},
|
|
{
|
|
"epoch": 1.977618342041644,
|
|
"grad_norm": 0.15722636878490448,
|
|
"learning_rate": 1.6991256983212024e-08,
|
|
"loss": 1.5357014656066894,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.64458,
|
|
"step": 6340,
|
|
"tokens/total": 41620480,
|
|
"tokens/train_per_sec_per_gpu": 72.06,
|
|
"tokens/trainable": 26804098
|
|
},
|
|
{
|
|
"epoch": 1.9807377368790453,
|
|
"grad_norm": 0.1385359764099121,
|
|
"learning_rate": 1.2655329105362712e-08,
|
|
"loss": 1.5237247467041015,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.58929,
|
|
"step": 6350,
|
|
"tokens/total": 41687168,
|
|
"tokens/train_per_sec_per_gpu": 82.57,
|
|
"tokens/trainable": 26847666
|
|
},
|
|
{
|
|
"epoch": 1.983857131716447,
|
|
"grad_norm": 0.1311943680047989,
|
|
"learning_rate": 8.956839536314321e-09,
|
|
"loss": 1.4549382209777832,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.28422,
|
|
"step": 6360,
|
|
"tokens/total": 41752960,
|
|
"tokens/train_per_sec_per_gpu": 73.29,
|
|
"tokens/trainable": 26888720
|
|
},
|
|
{
|
|
"epoch": 1.9869765265538486,
|
|
"grad_norm": 0.142974391579628,
|
|
"learning_rate": 5.895882626189009e-09,
|
|
"loss": 1.5607447624206543,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.76237,
|
|
"step": 6370,
|
|
"tokens/total": 41815424,
|
|
"tokens/train_per_sec_per_gpu": 64.73,
|
|
"tokens/trainable": 26929208
|
|
},
|
|
{
|
|
"epoch": 1.9900959213912501,
|
|
"grad_norm": 0.12927938997745514,
|
|
"learning_rate": 3.472536461360254e-09,
|
|
"loss": 1.4408729553222657,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.22438,
|
|
"step": 6380,
|
|
"tokens/total": 41881984,
|
|
"tokens/train_per_sec_per_gpu": 67.29,
|
|
"tokens/trainable": 26972472
|
|
},
|
|
{
|
|
"epoch": 1.9932153162286517,
|
|
"grad_norm": 0.1558847725391388,
|
|
"learning_rate": 1.6868628624711057e-09,
|
|
"loss": 1.5446084022521973,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.68614,
|
|
"step": 6390,
|
|
"tokens/total": 41949056,
|
|
"tokens/train_per_sec_per_gpu": 73.94,
|
|
"tokens/trainable": 27016170
|
|
},
|
|
{
|
|
"epoch": 1.9963347110660532,
|
|
"grad_norm": 0.15741246938705444,
|
|
"learning_rate": 5.389073828493407e-10,
|
|
"loss": 1.5909221649169922,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"ppl": 4.90827,
|
|
"step": 6400,
|
|
"tokens/total": 42013056,
|
|
"tokens/train_per_sec_per_gpu": 77.85,
|
|
"tokens/trainable": 27056634
|
|
},
|
|
{
|
|
"epoch": 1.9994541059034547,
|
|
"grad_norm": 0.13895246386528015,
|
|
"learning_rate": 2.869930735560633e-11,
|
|
"loss": 1.5546998977661133,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.12,
|
|
"memory/max_allocated (GiB)": 4.12,
|
|
"ppl": 4.73367,
|
|
"step": 6410,
|
|
"tokens/total": 42079488,
|
|
"tokens/train_per_sec_per_gpu": 42.58,
|
|
"tokens/trainable": 27098368
|
|
},
|
|
{
|
|
"epoch": 2.0,
|
|
"eval_loss": 1.5107629299163818,
|
|
"eval_ppl": 4.53019,
|
|
"eval_runtime": 42.0036,
|
|
"eval_samples_per_second": 64.28,
|
|
"eval_steps_per_second": 32.14,
|
|
"memory/device_reserved (GiB)": 4.38,
|
|
"memory/max_active (GiB)": 4.11,
|
|
"memory/max_allocated (GiB)": 4.11,
|
|
"step": 6412
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 6412,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 3206,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 2.1480883273728e+16,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|