345 lines
12 KiB
JSON
345 lines
12 KiB
JSON
{
|
|
"best_global_step": 146,
|
|
"best_metric": 0.09535243362188339,
|
|
"best_model_checkpoint": "/NHNHOME/WORKSPACE/26msit001_A/mnpo/revision_qwen3_8b/full_iter1/train/htmnpo_skywork_s42_nhn-b200-q3-seq-07101430-htmnpo/checkpoint-146",
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 146,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.06866952789699571,
|
|
"grad_norm": 658.2609913586693,
|
|
"ht_mnpo/logit": -0.07977066189050674,
|
|
"ht_mnpo/residual": -0.08727066218852997,
|
|
"ht_mnpo/residual_abs": 0.5136820673942566,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -0.81640625,
|
|
"logits/rejected": -0.81640625,
|
|
"logps/chosen": -15.9375,
|
|
"logps/rejected": -15.875,
|
|
"loss": 1.032188057899475,
|
|
"rewards/accuracies": 0.4749999940395355,
|
|
"rewards/chosen": -154.9237518310547,
|
|
"rewards/margins": -0.7977066040039062,
|
|
"rewards/rejected": -154.1260528564453,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.13733905579399142,
|
|
"grad_norm": 486.85604769071176,
|
|
"ht_mnpo/logit": 0.07697143405675888,
|
|
"ht_mnpo/residual": 0.06947144120931625,
|
|
"ht_mnpo/residual_abs": 0.46500977873802185,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.988506452457066e-07,
|
|
"logits/chosen": -0.80859375,
|
|
"logits/rejected": -0.83984375,
|
|
"logps/chosen": -15.875,
|
|
"logps/rejected": -15.875,
|
|
"loss": 0.5586695075035095,
|
|
"rewards/accuracies": 0.543749988079071,
|
|
"rewards/chosen": -153.60064697265625,
|
|
"rewards/margins": 0.76971435546875,
|
|
"rewards/rejected": -154.370361328125,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.20600858369098712,
|
|
"grad_norm": 279.6489736984974,
|
|
"ht_mnpo/logit": 0.08701781928539276,
|
|
"ht_mnpo/residual": 0.07951782643795013,
|
|
"ht_mnpo/residual_abs": 0.48369044065475464,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.860414659112948e-07,
|
|
"logits/chosen": -0.8359375,
|
|
"logits/rejected": -0.8359375,
|
|
"logps/chosen": -15.75,
|
|
"logps/rejected": -15.75,
|
|
"loss": 0.6932045221328735,
|
|
"rewards/accuracies": 0.543749988079071,
|
|
"rewards/chosen": -152.07644653320312,
|
|
"rewards/margins": 0.87017822265625,
|
|
"rewards/rejected": -152.94662475585938,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.27467811158798283,
|
|
"grad_norm": 405.5407274927664,
|
|
"ht_mnpo/logit": 0.11707458645105362,
|
|
"ht_mnpo/residual": 0.1095745787024498,
|
|
"ht_mnpo/residual_abs": 0.40353816747665405,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.597220474379125e-07,
|
|
"logits/chosen": -0.82421875,
|
|
"logits/rejected": -0.82421875,
|
|
"logps/chosen": -15.5,
|
|
"logps/rejected": -15.5,
|
|
"loss": 0.6164440512657166,
|
|
"rewards/accuracies": 0.5375000238418579,
|
|
"rewards/chosen": -149.45849609375,
|
|
"rewards/margins": 1.170745849609375,
|
|
"rewards/rejected": -150.62924194335938,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.34334763948497854,
|
|
"grad_norm": 173.47778729238564,
|
|
"ht_mnpo/logit": 0.09282531589269638,
|
|
"ht_mnpo/residual": 0.08532532304525375,
|
|
"ht_mnpo/residual_abs": 0.3633372485637665,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.2139882699413613e-07,
|
|
"logits/chosen": -0.89453125,
|
|
"logits/rejected": -0.921875,
|
|
"logps/chosen": -14.9375,
|
|
"logps/rejected": -14.9375,
|
|
"loss": 0.38052481412887573,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -144.1920166015625,
|
|
"rewards/margins": 0.928253173828125,
|
|
"rewards/rejected": -145.12026977539062,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.41201716738197425,
|
|
"grad_norm": 220.4191286143336,
|
|
"ht_mnpo/logit": 0.15768662095069885,
|
|
"ht_mnpo/residual": 0.15018662810325623,
|
|
"ht_mnpo/residual_abs": 0.4016141891479492,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.7326529999303633e-07,
|
|
"logits/chosen": -0.89453125,
|
|
"logits/rejected": -0.91796875,
|
|
"logps/chosen": -14.5625,
|
|
"logps/rejected": -14.625,
|
|
"loss": 0.8031173944473267,
|
|
"rewards/accuracies": 0.5687500238418579,
|
|
"rewards/chosen": -140.22152709960938,
|
|
"rewards/margins": 1.5768661499023438,
|
|
"rewards/rejected": -141.7983856201172,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.48068669527896996,
|
|
"grad_norm": 226.8002386346793,
|
|
"ht_mnpo/logit": 0.15667113661766052,
|
|
"ht_mnpo/residual": 0.1491711437702179,
|
|
"ht_mnpo/residual_abs": 0.4558327794075012,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1807647161082797e-07,
|
|
"logits/chosen": -0.890625,
|
|
"logits/rejected": -0.8828125,
|
|
"logps/chosen": -14.3125,
|
|
"logps/rejected": -14.3125,
|
|
"loss": 1.1153709888458252,
|
|
"rewards/accuracies": 0.512499988079071,
|
|
"rewards/chosen": -136.7957305908203,
|
|
"rewards/margins": 1.56671142578125,
|
|
"rewards/rejected": -138.36244201660156,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.5493562231759657,
|
|
"grad_norm": 188.6296821878058,
|
|
"ht_mnpo/logit": 0.040885161608457565,
|
|
"ht_mnpo/residual": 0.03338516503572464,
|
|
"ht_mnpo/residual_abs": 0.35882776975631714,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5899116932879534e-07,
|
|
"logits/chosen": -0.83203125,
|
|
"logits/rejected": -0.87109375,
|
|
"logps/chosen": -14.1875,
|
|
"logps/rejected": -14.1875,
|
|
"loss": 0.46190714836120605,
|
|
"rewards/accuracies": 0.4937500059604645,
|
|
"rewards/chosen": -136.6307373046875,
|
|
"rewards/margins": 0.40885162353515625,
|
|
"rewards/rejected": -137.03958129882812,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.6180257510729614,
|
|
"grad_norm": 159.63381058216007,
|
|
"ht_mnpo/logit": 0.2624649107456207,
|
|
"ht_mnpo/residual": 0.2549648880958557,
|
|
"ht_mnpo/residual_abs": 0.4282897412776947,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.993912420112756e-07,
|
|
"logits/chosen": -0.83984375,
|
|
"logits/rejected": -0.84375,
|
|
"logps/chosen": -14.0625,
|
|
"logps/rejected": -14.125,
|
|
"loss": 0.8312055468559265,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -134.35748291015625,
|
|
"rewards/margins": 2.6246490478515625,
|
|
"rewards/rejected": -136.9821319580078,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.6866952789699571,
|
|
"grad_norm": 217.2628805931693,
|
|
"ht_mnpo/logit": 0.059761811047792435,
|
|
"ht_mnpo/residual": 0.05226181820034981,
|
|
"ht_mnpo/residual_abs": 0.303397536277771,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.426879939713322e-07,
|
|
"logits/chosen": -0.86328125,
|
|
"logits/rejected": -0.875,
|
|
"logps/chosen": -13.875,
|
|
"logps/rejected": -13.875,
|
|
"loss": 0.22050431370735168,
|
|
"rewards/accuracies": 0.518750011920929,
|
|
"rewards/chosen": -134.1347198486328,
|
|
"rewards/margins": 0.5976181030273438,
|
|
"rewards/rejected": -134.73233032226562,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.7553648068669528,
|
|
"grad_norm": 307.6598272099915,
|
|
"ht_mnpo/logit": 0.08397064357995987,
|
|
"ht_mnpo/residual": 0.07647065818309784,
|
|
"ht_mnpo/residual_abs": 0.3651581406593323,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.212693310317479e-08,
|
|
"logits/chosen": -0.83203125,
|
|
"logits/rejected": -0.84765625,
|
|
"logps/chosen": -13.9375,
|
|
"logps/rejected": -13.9375,
|
|
"loss": 0.46880072355270386,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": -133.74819946289062,
|
|
"rewards/margins": 0.8397064208984375,
|
|
"rewards/rejected": -134.58790588378906,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.8240343347639485,
|
|
"grad_norm": 101.02657333091942,
|
|
"ht_mnpo/logit": 0.02447204664349556,
|
|
"ht_mnpo/residual": 0.016972053796052933,
|
|
"ht_mnpo/residual_abs": 0.24492737650871277,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.060200865767605e-08,
|
|
"logits/chosen": -0.80859375,
|
|
"logits/rejected": -0.8359375,
|
|
"logps/chosen": -13.8125,
|
|
"logps/rejected": -13.875,
|
|
"loss": 0.1213923841714859,
|
|
"rewards/accuracies": 0.5625,
|
|
"rewards/chosen": -133.80438232421875,
|
|
"rewards/margins": 0.244720458984375,
|
|
"rewards/rejected": -134.04910278320312,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.8927038626609443,
|
|
"grad_norm": 62.457306149658216,
|
|
"ht_mnpo/logit": 0.18063659965991974,
|
|
"ht_mnpo/residual": 0.1731366068124771,
|
|
"ht_mnpo/residual_abs": 0.3555470108985901,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0489971081290193e-08,
|
|
"logits/chosen": -0.78125,
|
|
"logits/rejected": -0.80859375,
|
|
"logps/chosen": -13.875,
|
|
"logps/rejected": -13.9375,
|
|
"loss": 0.5861908197402954,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": -133.495361328125,
|
|
"rewards/margins": 1.806365966796875,
|
|
"rewards/rejected": -135.30172729492188,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.9613733905579399,
|
|
"grad_norm": 59.652930086354274,
|
|
"ht_mnpo/logit": 0.009463501162827015,
|
|
"ht_mnpo/residual": 0.001963504124432802,
|
|
"ht_mnpo/residual_abs": 0.3243860602378845,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5143346177878565e-09,
|
|
"logits/chosen": -0.79296875,
|
|
"logits/rejected": -0.83203125,
|
|
"logps/chosen": -13.875,
|
|
"logps/rejected": -13.9375,
|
|
"loss": 0.5267720818519592,
|
|
"rewards/accuracies": 0.518750011920929,
|
|
"rewards/chosen": -134.2991943359375,
|
|
"rewards/margins": 0.094635009765625,
|
|
"rewards/rejected": -134.39382934570312,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"eval_ht_mnpo/logit": 0.0946219190955162,
|
|
"eval_ht_mnpo/residual": 0.08712193369865417,
|
|
"eval_ht_mnpo/residual_abs": 0.23038248717784882,
|
|
"eval_ht_mnpo/target": 0.007499998901039362,
|
|
"eval_ht_mnpo/target_abs": 0.007499998901039362,
|
|
"eval_logits/chosen": -0.78515625,
|
|
"eval_logits/rejected": -0.76953125,
|
|
"eval_logps/chosen": -13.8125,
|
|
"eval_logps/rejected": -13.875,
|
|
"eval_loss": 0.09535243362188339,
|
|
"eval_rewards/accuracies": 0.6578947305679321,
|
|
"eval_rewards/chosen": -134.2300262451172,
|
|
"eval_rewards/margins": 0.9462206959724426,
|
|
"eval_rewards/rejected": -135.17623901367188,
|
|
"eval_runtime": 70.1838,
|
|
"eval_samples_per_second": 8.606,
|
|
"eval_steps_per_second": 1.083,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 146,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.5071899907229698,
|
|
"train_runtime": 3280.3544,
|
|
"train_samples_per_second": 5.682,
|
|
"train_steps_per_second": 0.045
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 146,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 1000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|