401 lines
11 KiB
JSON
401 lines
11 KiB
JSON
{
|
|
"best_global_step": 600,
|
|
"best_metric": 1.2342418432235718,
|
|
"best_model_checkpoint": "/scratch/multilingual_model_sft/checkpoint-600",
|
|
"epoch": 2.4,
|
|
"eval_steps": 100,
|
|
"global_step": 600,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"entropy": 0.9756445910781621,
|
|
"epoch": 0.08,
|
|
"grad_norm": 3.6301491260528564,
|
|
"learning_rate": 1e-05,
|
|
"loss": 2.7480094909667967,
|
|
"mean_token_accuracy": 0.5633710252121091,
|
|
"num_tokens": 100509.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"entropy": 1.4356517121195793,
|
|
"epoch": 0.16,
|
|
"grad_norm": 1.0212956666946411,
|
|
"learning_rate": 1.9999902656029183e-05,
|
|
"loss": 1.8655733108520507,
|
|
"mean_token_accuracy": 0.6183225071057677,
|
|
"num_tokens": 194221.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"entropy": 1.4469021618366242,
|
|
"epoch": 0.24,
|
|
"grad_norm": 0.5234550833702087,
|
|
"learning_rate": 1.995710194507367e-05,
|
|
"loss": 1.4041429519653321,
|
|
"mean_token_accuracy": 0.6799660105258226,
|
|
"num_tokens": 292021.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"entropy": 1.3612655106931926,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.5519381165504456,
|
|
"learning_rate": 1.9836810309184468e-05,
|
|
"loss": 1.3517327308654785,
|
|
"mean_token_accuracy": 0.6959601622074842,
|
|
"num_tokens": 383296.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"entropy": 1.3319022830575704,
|
|
"epoch": 0.4,
|
|
"grad_norm": 0.6105360388755798,
|
|
"learning_rate": 1.9639963915348547e-05,
|
|
"loss": 1.309855842590332,
|
|
"mean_token_accuracy": 0.6991457279771567,
|
|
"num_tokens": 480575.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.4,
|
|
"eval_entropy": 1.3079482184516058,
|
|
"eval_loss": 1.3204385042190552,
|
|
"eval_mean_token_accuracy": 0.6943969362311893,
|
|
"eval_num_tokens": 480575.0,
|
|
"eval_runtime": 10.6851,
|
|
"eval_samples_per_second": 26.673,
|
|
"eval_steps_per_second": 3.369,
|
|
"step": 100
|
|
},
|
|
{
|
|
"entropy": 1.3555128309875726,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.6203790307044983,
|
|
"learning_rate": 1.93680947162578e-05,
|
|
"loss": 1.3396360397338867,
|
|
"mean_token_accuracy": 0.7012265287339687,
|
|
"num_tokens": 570090.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"entropy": 1.3061895180493592,
|
|
"epoch": 0.56,
|
|
"grad_norm": 0.4907386302947998,
|
|
"learning_rate": 1.9023318527921213e-05,
|
|
"loss": 1.2728803634643555,
|
|
"mean_token_accuracy": 0.7048281095921993,
|
|
"num_tokens": 667523.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"entropy": 1.3211671005934478,
|
|
"epoch": 0.64,
|
|
"grad_norm": 0.6228592395782471,
|
|
"learning_rate": 1.8608318563373735e-05,
|
|
"loss": 1.2911997795104981,
|
|
"mean_token_accuracy": 0.6987178284674883,
|
|
"num_tokens": 766038.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"entropy": 1.3343474190682172,
|
|
"epoch": 0.72,
|
|
"grad_norm": 0.5544903874397278,
|
|
"learning_rate": 1.8126324550630364e-05,
|
|
"loss": 1.318357276916504,
|
|
"mean_token_accuracy": 0.7034054283052683,
|
|
"num_tokens": 859651.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"entropy": 1.3386923231184482,
|
|
"epoch": 0.8,
|
|
"grad_norm": 0.6091942191123962,
|
|
"learning_rate": 1.7581087597399556e-05,
|
|
"loss": 1.3211461067199708,
|
|
"mean_token_accuracy": 0.6983358260244131,
|
|
"num_tokens": 954094.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.8,
|
|
"eval_entropy": 1.2773916522661846,
|
|
"eval_loss": 1.2775527238845825,
|
|
"eval_mean_token_accuracy": 0.701493599348598,
|
|
"eval_num_tokens": 954094.0,
|
|
"eval_runtime": 9.9476,
|
|
"eval_samples_per_second": 28.65,
|
|
"eval_steps_per_second": 3.619,
|
|
"step": 200
|
|
},
|
|
{
|
|
"entropy": 1.2788743745535611,
|
|
"epoch": 0.88,
|
|
"grad_norm": 0.6575713753700256,
|
|
"learning_rate": 1.69768509981707e-05,
|
|
"loss": 1.2579275131225587,
|
|
"mean_token_accuracy": 0.7133958246558905,
|
|
"num_tokens": 1043359.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"entropy": 1.2770573951303958,
|
|
"epoch": 0.96,
|
|
"grad_norm": 0.6401338577270508,
|
|
"learning_rate": 1.6318317210868936e-05,
|
|
"loss": 1.250250244140625,
|
|
"mean_token_accuracy": 0.7097325529903173,
|
|
"num_tokens": 1139550.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"entropy": 1.2663291092962026,
|
|
"epoch": 1.04,
|
|
"grad_norm": 0.6415250897407532,
|
|
"learning_rate": 1.56106112600807e-05,
|
|
"loss": 1.2160707473754884,
|
|
"mean_token_accuracy": 0.7165820356458426,
|
|
"num_tokens": 1237885.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"entropy": 1.2433586917817592,
|
|
"epoch": 1.12,
|
|
"grad_norm": 0.5600829124450684,
|
|
"learning_rate": 1.485924085166359e-05,
|
|
"loss": 1.2002551078796386,
|
|
"mean_token_accuracy": 0.7145083237439394,
|
|
"num_tokens": 1337347.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"entropy": 1.3045797925442457,
|
|
"epoch": 1.2,
|
|
"grad_norm": 0.6553295850753784,
|
|
"learning_rate": 1.4070053509147777e-05,
|
|
"loss": 1.2605380058288573,
|
|
"mean_token_accuracy": 0.7103811588138342,
|
|
"num_tokens": 1424998.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 1.2,
|
|
"eval_entropy": 1.2426132311423619,
|
|
"eval_loss": 1.2581056356430054,
|
|
"eval_mean_token_accuracy": 0.7045401218864653,
|
|
"eval_num_tokens": 1424998.0,
|
|
"eval_runtime": 9.9679,
|
|
"eval_samples_per_second": 28.592,
|
|
"eval_steps_per_second": 3.612,
|
|
"step": 300
|
|
},
|
|
{
|
|
"entropy": 1.2380406064912677,
|
|
"epoch": 1.28,
|
|
"grad_norm": 0.5739977359771729,
|
|
"learning_rate": 1.3249191065513857e-05,
|
|
"loss": 1.208388614654541,
|
|
"mean_token_accuracy": 0.7182681906968356,
|
|
"num_tokens": 1523052.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"entropy": 1.2422979518771171,
|
|
"epoch": 1.3599999999999999,
|
|
"grad_norm": 0.6654541492462158,
|
|
"learning_rate": 1.2403041864514005e-05,
|
|
"loss": 1.20194673538208,
|
|
"mean_token_accuracy": 0.7169692400842905,
|
|
"num_tokens": 1619061.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"entropy": 1.2528483916074038,
|
|
"epoch": 1.44,
|
|
"grad_norm": 0.5971714854240417,
|
|
"learning_rate": 1.153819104352846e-05,
|
|
"loss": 1.2315282821655273,
|
|
"mean_token_accuracy": 0.7162977565079928,
|
|
"num_tokens": 1710487.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"entropy": 1.2364515990018845,
|
|
"epoch": 1.52,
|
|
"grad_norm": 0.7230240702629089,
|
|
"learning_rate": 1.0661369284880005e-05,
|
|
"loss": 1.1993119239807128,
|
|
"mean_token_accuracy": 0.7190557043999434,
|
|
"num_tokens": 1805041.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"entropy": 1.195871875807643,
|
|
"epoch": 1.6,
|
|
"grad_norm": 0.6772177219390869,
|
|
"learning_rate": 9.779400434448137e-06,
|
|
"loss": 1.1887290954589844,
|
|
"mean_token_accuracy": 0.7250112488865852,
|
|
"num_tokens": 1898912.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 1.6,
|
|
"eval_entropy": 1.218405197064082,
|
|
"eval_loss": 1.2451484203338623,
|
|
"eval_mean_token_accuracy": 0.706767362025049,
|
|
"eval_num_tokens": 1898912.0,
|
|
"eval_runtime": 9.9407,
|
|
"eval_samples_per_second": 28.67,
|
|
"eval_steps_per_second": 3.621,
|
|
"step": 400
|
|
},
|
|
{
|
|
"entropy": 1.2644711177796126,
|
|
"epoch": 1.6800000000000002,
|
|
"grad_norm": 0.6553245782852173,
|
|
"learning_rate": 8.899148395239946e-06,
|
|
"loss": 1.2322025299072266,
|
|
"mean_token_accuracy": 0.7138959027826786,
|
|
"num_tokens": 1997219.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"entropy": 1.245160198956728,
|
|
"epoch": 1.76,
|
|
"grad_norm": 0.6900923848152161,
|
|
"learning_rate": 8.027463709217266e-06,
|
|
"loss": 1.2088998794555663,
|
|
"mean_token_accuracy": 0.7181152984499931,
|
|
"num_tokens": 2089697.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"entropy": 1.2373197358101606,
|
|
"epoch": 1.8399999999999999,
|
|
"grad_norm": 0.6978564858436584,
|
|
"learning_rate": 7.1711302431058795e-06,
|
|
"loss": 1.2115466117858886,
|
|
"mean_token_accuracy": 0.7191348981112242,
|
|
"num_tokens": 2185513.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"entropy": 1.2476738758385182,
|
|
"epoch": 1.92,
|
|
"grad_norm": 0.7304058074951172,
|
|
"learning_rate": 6.33681239310327e-06,
|
|
"loss": 1.223146629333496,
|
|
"mean_token_accuracy": 0.7194057062268258,
|
|
"num_tokens": 2278767.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"entropy": 1.2755128189921379,
|
|
"epoch": 2.0,
|
|
"grad_norm": 0.6474395990371704,
|
|
"learning_rate": 5.531003219363079e-06,
|
|
"loss": 1.2292745590209961,
|
|
"mean_token_accuracy": 0.7124675642699003,
|
|
"num_tokens": 2371346.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 2.0,
|
|
"eval_entropy": 1.214999525083436,
|
|
"eval_loss": 1.2368030548095703,
|
|
"eval_mean_token_accuracy": 0.7081699487235811,
|
|
"eval_num_tokens": 2371346.0,
|
|
"eval_runtime": 9.9336,
|
|
"eval_samples_per_second": 28.69,
|
|
"eval_steps_per_second": 3.624,
|
|
"step": 500
|
|
},
|
|
{
|
|
"entropy": 1.167585114017129,
|
|
"epoch": 2.08,
|
|
"grad_norm": 0.7057409882545471,
|
|
"learning_rate": 4.759973913898578e-06,
|
|
"loss": 1.1290258407592773,
|
|
"mean_token_accuracy": 0.733013367280364,
|
|
"num_tokens": 2466083.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"entropy": 1.2017585724592208,
|
|
"epoch": 2.16,
|
|
"grad_norm": 0.7214968204498291,
|
|
"learning_rate": 4.029724995169951e-06,
|
|
"loss": 1.1627693176269531,
|
|
"mean_token_accuracy": 0.7261429976671934,
|
|
"num_tokens": 2561750.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"entropy": 1.185857030004263,
|
|
"epoch": 2.24,
|
|
"grad_norm": 0.7629018425941467,
|
|
"learning_rate": 3.345939609182558e-06,
|
|
"loss": 1.175245475769043,
|
|
"mean_token_accuracy": 0.7282546646893024,
|
|
"num_tokens": 2658568.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"entropy": 1.236609160900116,
|
|
"epoch": 2.32,
|
|
"grad_norm": 0.6379859447479248,
|
|
"learning_rate": 2.713939300529255e-06,
|
|
"loss": 1.1822381019592285,
|
|
"mean_token_accuracy": 0.7198391534388066,
|
|
"num_tokens": 2757472.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"entropy": 1.2224572278559207,
|
|
"epoch": 2.4,
|
|
"grad_norm": 0.607354462146759,
|
|
"learning_rate": 2.138642597587686e-06,
|
|
"loss": 1.1898154258728026,
|
|
"mean_token_accuracy": 0.7245916619896888,
|
|
"num_tokens": 2850420.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 2.4,
|
|
"eval_entropy": 1.1946484976344638,
|
|
"eval_loss": 1.2342418432235718,
|
|
"eval_mean_token_accuracy": 0.7090677950117323,
|
|
"eval_num_tokens": 2850420.0,
|
|
"eval_runtime": 11.5935,
|
|
"eval_samples_per_second": 24.583,
|
|
"eval_steps_per_second": 3.105,
|
|
"step": 600
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 750,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 100,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 3.371131850832691e+16,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|