270 lines
8.5 KiB
JSON
270 lines
8.5 KiB
JSON
{
|
|
"best_global_step": 146,
|
|
"best_metric": 2487.621826171875,
|
|
"best_model_checkpoint": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/ipo_avg_beta0p01_s42_nhn-b200-q3-seq-07101430-ipo/checkpoint-146",
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 146,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.06864006864006864,
|
|
"grad_norm": 10027.114290420466,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -0.8125,
|
|
"logits/rejected": -0.8203125,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -15.9375,
|
|
"loss": 2489.71826171875,
|
|
"rewards/accuracies": 0.518750011920929,
|
|
"rewards/chosen": -153.37606811523438,
|
|
"rewards/margins": 1.1065444946289062,
|
|
"rewards/rejected": -154.48260498046875,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.13728013728013727,
|
|
"grad_norm": 8819.949018899022,
|
|
"learning_rate": 4.988506452457066e-07,
|
|
"logits/chosen": -0.8828125,
|
|
"logits/rejected": -0.85546875,
|
|
"logps/chosen": -15.8125,
|
|
"logps/rejected": -15.875,
|
|
"loss": 2487.34423828125,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": -153.25982666015625,
|
|
"rewards/margins": 1.295501708984375,
|
|
"rewards/rejected": -154.55532836914062,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.2059202059202059,
|
|
"grad_norm": 8211.073831209287,
|
|
"learning_rate": 4.860414659112948e-07,
|
|
"logits/chosen": -0.76953125,
|
|
"logits/rejected": -0.77734375,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -15.9375,
|
|
"loss": 2502.94287109375,
|
|
"rewards/accuracies": 0.4937500059604645,
|
|
"rewards/chosen": -154.95367431640625,
|
|
"rewards/margins": -0.227020263671875,
|
|
"rewards/rejected": -154.72665405273438,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.27456027456027454,
|
|
"grad_norm": 8719.803872679653,
|
|
"learning_rate": 4.597220474379125e-07,
|
|
"logits/chosen": -0.76171875,
|
|
"logits/rejected": -0.7734375,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -15.9375,
|
|
"loss": 2498.837158203125,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": -154.68797302246094,
|
|
"rewards/margins": 0.18853759765625,
|
|
"rewards/rejected": -154.87649536132812,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.3432003432003432,
|
|
"grad_norm": 9152.672087584928,
|
|
"learning_rate": 4.2139882699413613e-07,
|
|
"logits/chosen": -0.765625,
|
|
"logits/rejected": -0.74609375,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -16.0,
|
|
"loss": 2489.340087890625,
|
|
"rewards/accuracies": 0.5062500238418579,
|
|
"rewards/chosen": -154.12176513671875,
|
|
"rewards/margins": 1.1353836059570312,
|
|
"rewards/rejected": -155.2571563720703,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.4118404118404118,
|
|
"grad_norm": 14845.378062289448,
|
|
"learning_rate": 3.7326529999303633e-07,
|
|
"logits/chosen": -0.73046875,
|
|
"logits/rejected": -0.734375,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -15.9375,
|
|
"loss": 2498.56591796875,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": -155.47003173828125,
|
|
"rewards/margins": 0.18729400634765625,
|
|
"rewards/rejected": -155.65731811523438,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.4804804804804805,
|
|
"grad_norm": 10089.737237780628,
|
|
"learning_rate": 3.1807647161082797e-07,
|
|
"logits/chosen": -0.72265625,
|
|
"logits/rejected": -0.69140625,
|
|
"logps/chosen": -15.9375,
|
|
"logps/rejected": -15.9375,
|
|
"loss": 2490.2265625,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -154.46917724609375,
|
|
"rewards/margins": 1.055145263671875,
|
|
"rewards/rejected": -155.52432250976562,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.5491205491205491,
|
|
"grad_norm": 13080.69256033714,
|
|
"learning_rate": 2.5899116932879534e-07,
|
|
"logits/chosen": -0.7109375,
|
|
"logits/rejected": -0.71875,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -16.0,
|
|
"loss": 2498.123046875,
|
|
"rewards/accuracies": 0.48124998807907104,
|
|
"rewards/chosen": -155.04067993164062,
|
|
"rewards/margins": 0.243621826171875,
|
|
"rewards/rejected": -155.2843017578125,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.6177606177606177,
|
|
"grad_norm": 10488.56173199215,
|
|
"learning_rate": 1.993912420112756e-07,
|
|
"logits/chosen": -0.7265625,
|
|
"logits/rejected": -0.7109375,
|
|
"logps/chosen": -15.875,
|
|
"logps/rejected": -15.875,
|
|
"loss": 2492.578369140625,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": -153.39874267578125,
|
|
"rewards/margins": 0.793731689453125,
|
|
"rewards/rejected": -154.19247436523438,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.6864006864006864,
|
|
"grad_norm": 12700.337502115342,
|
|
"learning_rate": 1.426879939713322e-07,
|
|
"logits/chosen": -0.69140625,
|
|
"logits/rejected": -0.6875,
|
|
"logps/chosen": -15.875,
|
|
"logps/rejected": -15.875,
|
|
"loss": 2490.487060546875,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": -153.31903076171875,
|
|
"rewards/margins": 1.01458740234375,
|
|
"rewards/rejected": -154.33360290527344,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.7550407550407551,
|
|
"grad_norm": 9077.531663523849,
|
|
"learning_rate": 9.212693310317479e-08,
|
|
"logits/chosen": -0.69140625,
|
|
"logits/rejected": -0.69140625,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -16.0,
|
|
"loss": 2487.44677734375,
|
|
"rewards/accuracies": 0.543749988079071,
|
|
"rewards/chosen": -154.6409912109375,
|
|
"rewards/margins": 1.328338623046875,
|
|
"rewards/rejected": -155.96932983398438,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.8236808236808236,
|
|
"grad_norm": 8718.070043453483,
|
|
"learning_rate": 5.060200865767605e-08,
|
|
"logits/chosen": -0.625,
|
|
"logits/rejected": -0.62890625,
|
|
"logps/chosen": -16.0,
|
|
"logps/rejected": -16.125,
|
|
"loss": 2481.56396484375,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": -153.7762451171875,
|
|
"rewards/margins": 1.955291748046875,
|
|
"rewards/rejected": -155.73153686523438,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.8923208923208923,
|
|
"grad_norm": 11737.493049054548,
|
|
"learning_rate": 2.0489971081290193e-08,
|
|
"logits/chosen": -0.66015625,
|
|
"logits/rejected": -0.66015625,
|
|
"logps/chosen": -15.9375,
|
|
"logps/rejected": -15.9375,
|
|
"loss": 2495.81494140625,
|
|
"rewards/accuracies": 0.5375000238418579,
|
|
"rewards/chosen": -154.38516235351562,
|
|
"rewards/margins": 0.4686431884765625,
|
|
"rewards/rejected": -154.8538055419922,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.960960960960961,
|
|
"grad_norm": 9402.719273365525,
|
|
"learning_rate": 3.5143346177878565e-09,
|
|
"logits/chosen": -0.640625,
|
|
"logits/rejected": -0.62890625,
|
|
"logps/chosen": -15.9375,
|
|
"logps/rejected": -16.0,
|
|
"loss": 2483.26611328125,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -154.122314453125,
|
|
"rewards/margins": 1.7264404296875,
|
|
"rewards/rejected": -155.8487548828125,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"eval_logits/chosen": -0.65234375,
|
|
"eval_logits/rejected": -0.625,
|
|
"eval_logps/chosen": -15.9375,
|
|
"eval_logps/rejected": -16.0,
|
|
"eval_loss": 2487.621826171875,
|
|
"eval_rewards/accuracies": 0.5526315569877625,
|
|
"eval_rewards/chosen": -154.9027862548828,
|
|
"eval_rewards/margins": 1.2624943256378174,
|
|
"eval_rewards/rejected": -156.165283203125,
|
|
"eval_runtime": 69.2925,
|
|
"eval_samples_per_second": 8.717,
|
|
"eval_steps_per_second": 1.097,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 146,
|
|
"total_flos": 0.0,
|
|
"train_loss": 2490.3512815710615,
|
|
"train_runtime": 3284.6395,
|
|
"train_samples_per_second": 5.676,
|
|
"train_steps_per_second": 0.044
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 146,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 1000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|