270 lines
8.5 KiB
JSON
270 lines
8.5 KiB
JSON
|
{
|
||
|
|
"best_global_step": 146,
|
||
|
|
"best_metric": 23.90784454345703,
|
||
|
|
"best_model_checkpoint": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p1_s42_nhn-b200-q3-seq-07101430-ipo/checkpoint-146",
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 146,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.06864006864006864,
|
||
|
|
"grad_norm": 946.3972141968312,
|
||
|
|
"learning_rate": 3e-07,
|
||
|
|
"logits/chosen": -0.8046875,
|
||
|
|
"logits/rejected": -0.8125,
|
||
|
|
"logps/chosen": -16.0,
|
||
|
|
"logps/rejected": -15.9375,
|
||
|
|
"loss": 24.708133697509766,
|
||
|
|
"rewards/accuracies": 0.5249999761581421,
|
||
|
|
"rewards/chosen": -153.47763061523438,
|
||
|
|
"rewards/margins": 1.1338882446289062,
|
||
|
|
"rewards/rejected": -154.61151123046875,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13728013728013727,
|
||
|
|
"grad_norm": 1114.6296794744821,
|
||
|
|
"learning_rate": 4.988506452457066e-07,
|
||
|
|
"logits/chosen": -0.88671875,
|
||
|
|
"logits/rejected": -0.86328125,
|
||
|
|
"logps/chosen": -15.8125,
|
||
|
|
"logps/rejected": -15.875,
|
||
|
|
"loss": 23.966333389282227,
|
||
|
|
"rewards/accuracies": 0.5062500238418579,
|
||
|
|
"rewards/chosen": -153.10357666015625,
|
||
|
|
"rewards/margins": 1.342376708984375,
|
||
|
|
"rewards/rejected": -154.44595336914062,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2059202059202059,
|
||
|
|
"grad_norm": 889.7596588174916,
|
||
|
|
"learning_rate": 4.860414659112948e-07,
|
||
|
|
"logits/chosen": -0.7734375,
|
||
|
|
"logits/rejected": -0.78125,
|
||
|
|
"logps/chosen": -15.9375,
|
||
|
|
"logps/rejected": -15.875,
|
||
|
|
"loss": 25.885950088500977,
|
||
|
|
"rewards/accuracies": 0.4749999940395355,
|
||
|
|
"rewards/chosen": -154.28179931640625,
|
||
|
|
"rewards/margins": -0.187957763671875,
|
||
|
|
"rewards/rejected": -154.09384155273438,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27456027456027454,
|
||
|
|
"grad_norm": 849.2347387176682,
|
||
|
|
"learning_rate": 4.597220474379125e-07,
|
||
|
|
"logits/chosen": -0.78125,
|
||
|
|
"logits/rejected": -0.79296875,
|
||
|
|
"logps/chosen": -15.875,
|
||
|
|
"logps/rejected": -15.875,
|
||
|
|
"loss": 25.355098724365234,
|
||
|
|
"rewards/accuracies": 0.5562499761581421,
|
||
|
|
"rewards/chosen": -153.62156677246094,
|
||
|
|
"rewards/margins": 0.29400634765625,
|
||
|
|
"rewards/rejected": -153.91555786132812,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3432003432003432,
|
||
|
|
"grad_norm": 841.6754120513388,
|
||
|
|
"learning_rate": 4.2139882699413613e-07,
|
||
|
|
"logits/chosen": -0.7890625,
|
||
|
|
"logits/rejected": -0.78125,
|
||
|
|
"logps/chosen": -15.875,
|
||
|
|
"logps/rejected": -15.8125,
|
||
|
|
"loss": 24.766250610351562,
|
||
|
|
"rewards/accuracies": 0.518750011920929,
|
||
|
|
"rewards/chosen": -152.64520263671875,
|
||
|
|
"rewards/margins": 0.9361648559570312,
|
||
|
|
"rewards/rejected": -153.5813751220703,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4118404118404118,
|
||
|
|
"grad_norm": 881.2956684704218,
|
||
|
|
"learning_rate": 3.7326529999303633e-07,
|
||
|
|
"logits/chosen": -0.77734375,
|
||
|
|
"logits/rejected": -0.78125,
|
||
|
|
"logps/chosen": -15.75,
|
||
|
|
"logps/rejected": -15.6875,
|
||
|
|
"loss": 25.237834930419922,
|
||
|
|
"rewards/accuracies": 0.5562499761581421,
|
||
|
|
"rewards/chosen": -152.76690673828125,
|
||
|
|
"rewards/margins": 0.15995025634765625,
|
||
|
|
"rewards/rejected": -152.92684936523438,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4804804804804805,
|
||
|
|
"grad_norm": 847.5060900325763,
|
||
|
|
"learning_rate": 3.1807647161082797e-07,
|
||
|
|
"logits/chosen": -0.7890625,
|
||
|
|
"logits/rejected": -0.76953125,
|
||
|
|
"logps/chosen": -15.625,
|
||
|
|
"logps/rejected": -15.625,
|
||
|
|
"loss": 24.342920303344727,
|
||
|
|
"rewards/accuracies": 0.5874999761581421,
|
||
|
|
"rewards/chosen": -151.05902099609375,
|
||
|
|
"rewards/margins": 1.297332763671875,
|
||
|
|
"rewards/rejected": -152.35635375976562,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5491205491205491,
|
||
|
|
"grad_norm": 1018.8684011913645,
|
||
|
|
"learning_rate": 2.5899116932879534e-07,
|
||
|
|
"logits/chosen": -0.78515625,
|
||
|
|
"logits/rejected": -0.80078125,
|
||
|
|
"logps/chosen": -15.625,
|
||
|
|
"logps/rejected": -15.625,
|
||
|
|
"loss": 25.037967681884766,
|
||
|
|
"rewards/accuracies": 0.5,
|
||
|
|
"rewards/chosen": -151.04458618164062,
|
||
|
|
"rewards/margins": 0.520965576171875,
|
||
|
|
"rewards/rejected": -151.5655517578125,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6177606177606177,
|
||
|
|
"grad_norm": 845.4769875876663,
|
||
|
|
"learning_rate": 1.993912420112756e-07,
|
||
|
|
"logits/chosen": -0.8359375,
|
||
|
|
"logits/rejected": -0.828125,
|
||
|
|
"logps/chosen": -15.5,
|
||
|
|
"logps/rejected": -15.4375,
|
||
|
|
"loss": 24.371807098388672,
|
||
|
|
"rewards/accuracies": 0.6000000238418579,
|
||
|
|
"rewards/chosen": -149.18389892578125,
|
||
|
|
"rewards/margins": 1.032012939453125,
|
||
|
|
"rewards/rejected": -150.21591186523438,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6864006864006864,
|
||
|
|
"grad_norm": 741.7784259326611,
|
||
|
|
"learning_rate": 1.426879939713322e-07,
|
||
|
|
"logits/chosen": -0.8125,
|
||
|
|
"logits/rejected": -0.796875,
|
||
|
|
"logps/chosen": -15.375,
|
||
|
|
"logps/rejected": -15.4375,
|
||
|
|
"loss": 24.30291748046875,
|
||
|
|
"rewards/accuracies": 0.581250011920929,
|
||
|
|
"rewards/chosen": -148.29949951171875,
|
||
|
|
"rewards/margins": 1.22161865234375,
|
||
|
|
"rewards/rejected": -149.52110290527344,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7550407550407551,
|
||
|
|
"grad_norm": 891.2761904365972,
|
||
|
|
"learning_rate": 9.212693310317479e-08,
|
||
|
|
"logits/chosen": -0.81640625,
|
||
|
|
"logits/rejected": -0.80859375,
|
||
|
|
"logps/chosen": -15.4375,
|
||
|
|
"logps/rejected": -15.4375,
|
||
|
|
"loss": 24.666929244995117,
|
||
|
|
"rewards/accuracies": 0.512499988079071,
|
||
|
|
"rewards/chosen": -148.9534912109375,
|
||
|
|
"rewards/margins": 0.953338623046875,
|
||
|
|
"rewards/rejected": -149.90682983398438,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8236808236808236,
|
||
|
|
"grad_norm": 837.0292977089434,
|
||
|
|
"learning_rate": 5.060200865767605e-08,
|
||
|
|
"logits/chosen": -0.77734375,
|
||
|
|
"logits/rejected": -0.78515625,
|
||
|
|
"logps/chosen": -15.5,
|
||
|
|
"logps/rejected": -15.5,
|
||
|
|
"loss": 24.336681365966797,
|
||
|
|
"rewards/accuracies": 0.550000011920929,
|
||
|
|
"rewards/chosen": -148.3778076171875,
|
||
|
|
"rewards/margins": 1.705291748046875,
|
||
|
|
"rewards/rejected": -150.08309936523438,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8923208923208923,
|
||
|
|
"grad_norm": 912.1427660596082,
|
||
|
|
"learning_rate": 2.0489971081290193e-08,
|
||
|
|
"logits/chosen": -0.79296875,
|
||
|
|
"logits/rejected": -0.77734375,
|
||
|
|
"logps/chosen": -15.3125,
|
||
|
|
"logps/rejected": -15.375,
|
||
|
|
"loss": 24.866641998291016,
|
||
|
|
"rewards/accuracies": 0.512499988079071,
|
||
|
|
"rewards/chosen": -148.61953735351562,
|
||
|
|
"rewards/margins": 0.5936431884765625,
|
||
|
|
"rewards/rejected": -149.2131805419922,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.960960960960961,
|
||
|
|
"grad_norm": 1236.4699022849795,
|
||
|
|
"learning_rate": 3.5143346177878565e-09,
|
||
|
|
"logits/chosen": -0.77734375,
|
||
|
|
"logits/rejected": -0.78125,
|
||
|
|
"logps/chosen": -15.375,
|
||
|
|
"logps/rejected": -15.375,
|
||
|
|
"loss": 24.20675277709961,
|
||
|
|
"rewards/accuracies": 0.5562499761581421,
|
||
|
|
"rewards/chosen": -148.130126953125,
|
||
|
|
"rewards/margins": 1.1678466796875,
|
||
|
|
"rewards/rejected": -149.2979736328125,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_logits/chosen": -0.8125,
|
||
|
|
"eval_logits/rejected": -0.7734375,
|
||
|
|
"eval_logps/chosen": -15.25,
|
||
|
|
"eval_logps/rejected": -15.375,
|
||
|
|
"eval_loss": 23.90784454345703,
|
||
|
|
"eval_rewards/accuracies": 0.5526315569877625,
|
||
|
|
"eval_rewards/chosen": -148.4471893310547,
|
||
|
|
"eval_rewards/margins": 1.3118354082107544,
|
||
|
|
"eval_rewards/rejected": -149.759033203125,
|
||
|
|
"eval_runtime": 70.7414,
|
||
|
|
"eval_samples_per_second": 8.538,
|
||
|
|
"eval_steps_per_second": 1.074,
|
||
|
|
"step": 146
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0,
|
||
|
|
"step": 146,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 24.544400567877783,
|
||
|
|
"train_runtime": 3326.8166,
|
||
|
|
"train_samples_per_second": 5.604,
|
||
|
|
"train_steps_per_second": 0.044
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 146,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 1000,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|