30c78df99f
Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s44 Source: Original Platform
3824 lines
143 KiB
JSON
3824 lines
143 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8600609209819029,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004778116227677238,
|
|
"grad_norm": 0.9296875,
|
|
"ht_mnpo/logit": 0.05088333040475845,
|
|
"ht_mnpo/residual": 0.04338332638144493,
|
|
"ht_mnpo/residual_abs": 0.19410482048988342,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -2.322283983230591,
|
|
"logits/rejected": -2.095109224319458,
|
|
"logps/chosen": -0.3028138279914856,
|
|
"logps/rejected": -0.31887882947921753,
|
|
"loss": 0.7703814506530762,
|
|
"loss/core": 0.7703814506530762,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8105131387710571,
|
|
"rewards/margins": 0.5088331699371338,
|
|
"rewards/rejected": 1.3016799688339233,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009556232455354476,
|
|
"grad_norm": 104.0,
|
|
"ht_mnpo/logit": 0.030571451410651207,
|
|
"ht_mnpo/residual": 0.02307145670056343,
|
|
"ht_mnpo/residual_abs": 0.12138247489929199,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -2.109570026397705,
|
|
"logits/rejected": -1.8389272689819336,
|
|
"logps/chosen": -0.3074098527431488,
|
|
"logps/rejected": -0.32182925939559937,
|
|
"loss": 0.11068801581859589,
|
|
"loss/core": 0.11068801581859589,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 2.1581244468688965,
|
|
"rewards/margins": 0.30571448802948,
|
|
"rewards/rejected": 1.8524096012115479,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014334348683031715,
|
|
"grad_norm": 63.25,
|
|
"ht_mnpo/logit": 0.03353700414299965,
|
|
"ht_mnpo/residual": 0.026036998257040977,
|
|
"ht_mnpo/residual_abs": 0.22524623572826385,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -2.1958093643188477,
|
|
"logits/rejected": -1.9058806896209717,
|
|
"logps/chosen": -0.27820906043052673,
|
|
"logps/rejected": -0.2950650751590729,
|
|
"loss": 0.6725027561187744,
|
|
"loss/core": 0.6725027561187744,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.2042908668518066,
|
|
"rewards/margins": 0.33536994457244873,
|
|
"rewards/rejected": 1.8689210414886475,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.019112464910708952,
|
|
"grad_norm": 2.203125,
|
|
"ht_mnpo/logit": 0.060489214956760406,
|
|
"ht_mnpo/residual": 0.05298921465873718,
|
|
"ht_mnpo/residual_abs": 0.06931939721107483,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -1.9993699789047241,
|
|
"logits/rejected": -1.799395203590393,
|
|
"logps/chosen": -0.3109528720378876,
|
|
"logps/rejected": -0.31548988819122314,
|
|
"loss": 0.02609763666987419,
|
|
"loss/core": 0.02609763666987419,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.7414203882217407,
|
|
"rewards/margins": 0.6048921346664429,
|
|
"rewards/rejected": 1.1365283727645874,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.023890581138386192,
|
|
"grad_norm": 114.5,
|
|
"ht_mnpo/logit": 0.09344500303268433,
|
|
"ht_mnpo/residual": 0.0859449952840805,
|
|
"ht_mnpo/residual_abs": 0.14693133533000946,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -2.228485345840454,
|
|
"logits/rejected": -1.9559946060180664,
|
|
"logps/chosen": -0.2875242233276367,
|
|
"logps/rejected": -0.2956809997558594,
|
|
"loss": 0.2483440339565277,
|
|
"loss/core": 0.2483440339565277,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.1028735637664795,
|
|
"rewards/margins": 0.9344498515129089,
|
|
"rewards/rejected": 1.1684236526489258,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.02866869736606343,
|
|
"grad_norm": 5.4375,
|
|
"ht_mnpo/logit": 0.0791626125574112,
|
|
"ht_mnpo/residual": 0.07166260480880737,
|
|
"ht_mnpo/residual_abs": 0.16665640473365784,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -2.1653201580047607,
|
|
"logits/rejected": -1.931023359298706,
|
|
"logps/chosen": -0.26770323514938354,
|
|
"logps/rejected": -0.28871315717697144,
|
|
"loss": 0.3910994529724121,
|
|
"loss/core": 0.3910994529724121,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.4544098377227783,
|
|
"rewards/margins": 0.7916260957717896,
|
|
"rewards/rejected": 1.6627838611602783,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.03344681359374067,
|
|
"grad_norm": 56.25,
|
|
"ht_mnpo/logit": 0.09365877509117126,
|
|
"ht_mnpo/residual": 0.08615876734256744,
|
|
"ht_mnpo/residual_abs": 0.14225038886070251,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -2.2201991081237793,
|
|
"logits/rejected": -1.9280380010604858,
|
|
"logps/chosen": -0.28348293900489807,
|
|
"logps/rejected": -0.28683170676231384,
|
|
"loss": 0.14075805246829987,
|
|
"loss/core": 0.14075805246829987,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.5023605823516846,
|
|
"rewards/margins": 0.9365876317024231,
|
|
"rewards/rejected": 1.5657732486724854,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.038224929821417904,
|
|
"grad_norm": 3.6875,
|
|
"ht_mnpo/logit": 0.023442532867193222,
|
|
"ht_mnpo/residual": 0.015942532569169998,
|
|
"ht_mnpo/residual_abs": 0.15496210753917694,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -2.354321002960205,
|
|
"logits/rejected": -1.9975166320800781,
|
|
"logps/chosen": -0.2890976667404175,
|
|
"logps/rejected": -0.30953389406204224,
|
|
"loss": 0.2759542167186737,
|
|
"loss/core": 0.2759542167186737,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.8200693130493164,
|
|
"rewards/margins": 0.23442523181438446,
|
|
"rewards/rejected": 1.585644006729126,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04300304604909514,
|
|
"grad_norm": 1.9140625,
|
|
"ht_mnpo/logit": 0.1023062914609909,
|
|
"ht_mnpo/residual": 0.09480629861354828,
|
|
"ht_mnpo/residual_abs": 0.10691990703344345,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -2.300888776779175,
|
|
"logits/rejected": -2.1800649166107178,
|
|
"logps/chosen": -0.294630229473114,
|
|
"logps/rejected": -0.2884281277656555,
|
|
"loss": 0.06364284455776215,
|
|
"loss/core": 0.06364284455776215,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0192909240722656,
|
|
"rewards/margins": 1.0230629444122314,
|
|
"rewards/rejected": 0.9962279200553894,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.047781162276772385,
|
|
"grad_norm": 0.93359375,
|
|
"ht_mnpo/logit": 0.05063255876302719,
|
|
"ht_mnpo/residual": 0.04313255473971367,
|
|
"ht_mnpo/residual_abs": 0.056837163865566254,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -2.179457902908325,
|
|
"logits/rejected": -1.9664961099624634,
|
|
"logps/chosen": -0.2966025769710541,
|
|
"logps/rejected": -0.30326494574546814,
|
|
"loss": 0.02287248522043228,
|
|
"loss/core": 0.02287248522043228,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.278033971786499,
|
|
"rewards/margins": 0.5063256025314331,
|
|
"rewards/rejected": 0.771708607673645,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05255927850444962,
|
|
"grad_norm": 0.90234375,
|
|
"ht_mnpo/logit": 0.10425949096679688,
|
|
"ht_mnpo/residual": 0.09675948321819305,
|
|
"ht_mnpo/residual_abs": 0.1356755793094635,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -2.292445182800293,
|
|
"logits/rejected": -2.0353176593780518,
|
|
"logps/chosen": -0.29430681467056274,
|
|
"logps/rejected": -0.28913360834121704,
|
|
"loss": 0.3527126908302307,
|
|
"loss/core": 0.3527126908302307,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0845179557800293,
|
|
"rewards/margins": 1.0425949096679688,
|
|
"rewards/rejected": 1.04192316532135,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.05733739473212686,
|
|
"grad_norm": 13.3125,
|
|
"ht_mnpo/logit": 0.17591974139213562,
|
|
"ht_mnpo/residual": 0.168419748544693,
|
|
"ht_mnpo/residual_abs": 0.17273147404193878,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -1.9740009307861328,
|
|
"logits/rejected": -1.7925126552581787,
|
|
"logps/chosen": -0.27490484714508057,
|
|
"logps/rejected": -0.29008954763412476,
|
|
"loss": 0.24429097771644592,
|
|
"loss/core": 0.24429097771644592,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.951267719268799,
|
|
"rewards/margins": 1.759197473526001,
|
|
"rewards/rejected": 1.1920703649520874,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.062115510959804096,
|
|
"grad_norm": 87.0,
|
|
"ht_mnpo/logit": -0.027438348159193993,
|
|
"ht_mnpo/residual": -0.03493834659457207,
|
|
"ht_mnpo/residual_abs": 0.11112980544567108,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -2.2002429962158203,
|
|
"logits/rejected": -2.0315022468566895,
|
|
"logps/chosen": -0.28836536407470703,
|
|
"logps/rejected": -0.29485026001930237,
|
|
"loss": 0.34929710626602173,
|
|
"loss/core": 0.34929710626602173,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.0422712564468384,
|
|
"rewards/margins": -0.2743833363056183,
|
|
"rewards/rejected": 1.3166545629501343,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06689362718748133,
|
|
"grad_norm": 1.8125,
|
|
"ht_mnpo/logit": -0.009448112919926643,
|
|
"ht_mnpo/residual": -0.01694810949265957,
|
|
"ht_mnpo/residual_abs": 0.10872050374746323,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -2.333392858505249,
|
|
"logits/rejected": -2.1000077724456787,
|
|
"logps/chosen": -0.2741190791130066,
|
|
"logps/rejected": -0.2725319266319275,
|
|
"loss": 0.1955917775630951,
|
|
"loss/core": 0.1955917775630951,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 1.531646490097046,
|
|
"rewards/margins": -0.09448114782571793,
|
|
"rewards/rejected": 1.6261276006698608,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07167174341515857,
|
|
"grad_norm": 79.5,
|
|
"ht_mnpo/logit": 0.09921145439147949,
|
|
"ht_mnpo/residual": 0.09171145409345627,
|
|
"ht_mnpo/residual_abs": 0.1735668033361435,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -2.177030324935913,
|
|
"logits/rejected": -1.9694257974624634,
|
|
"logps/chosen": -0.30770665407180786,
|
|
"logps/rejected": -0.30710548162460327,
|
|
"loss": 0.26593199372291565,
|
|
"loss/core": 0.26593199372291565,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.6823678016662598,
|
|
"rewards/margins": 0.9921146631240845,
|
|
"rewards/rejected": 1.6902530193328857,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07644985964283581,
|
|
"grad_norm": 15.5625,
|
|
"ht_mnpo/logit": 0.028042754158377647,
|
|
"ht_mnpo/residual": 0.02054273523390293,
|
|
"ht_mnpo/residual_abs": 0.27549320459365845,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -2.109937906265259,
|
|
"logits/rejected": -1.8973028659820557,
|
|
"logps/chosen": -0.3108934760093689,
|
|
"logps/rejected": -0.3061710000038147,
|
|
"loss": 1.2482848167419434,
|
|
"loss/core": 1.2482848167419434,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.4044840335845947,
|
|
"rewards/margins": 0.28042739629745483,
|
|
"rewards/rejected": 2.124056577682495,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08122797587051304,
|
|
"grad_norm": 2.625,
|
|
"ht_mnpo/logit": 0.04282355681061745,
|
|
"ht_mnpo/residual": 0.03532355651259422,
|
|
"ht_mnpo/residual_abs": 0.05199320241808891,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -2.1569108963012695,
|
|
"logits/rejected": -1.9936288595199585,
|
|
"logps/chosen": -0.29405292868614197,
|
|
"logps/rejected": -0.2927798628807068,
|
|
"loss": 0.015568750910460949,
|
|
"loss/core": 0.015568750910460949,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.2469490766525269,
|
|
"rewards/margins": 0.42823559045791626,
|
|
"rewards/rejected": 0.8187135457992554,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08600609209819028,
|
|
"grad_norm": 21.25,
|
|
"ht_mnpo/logit": 0.09608729183673859,
|
|
"ht_mnpo/residual": 0.08858727663755417,
|
|
"ht_mnpo/residual_abs": 0.12235699594020844,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -2.227970600128174,
|
|
"logits/rejected": -2.119290351867676,
|
|
"logps/chosen": -0.26927095651626587,
|
|
"logps/rejected": -0.2766638994216919,
|
|
"loss": 0.09564952552318573,
|
|
"loss/core": 0.09564952552318573,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.044128179550171,
|
|
"rewards/margins": 0.9608729481697083,
|
|
"rewards/rejected": 1.0832551717758179,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09078420832586753,
|
|
"grad_norm": 8.6875,
|
|
"ht_mnpo/logit": -0.01110445149242878,
|
|
"ht_mnpo/residual": -0.018604444339871407,
|
|
"ht_mnpo/residual_abs": 0.14139024913311005,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -2.525456190109253,
|
|
"logits/rejected": -2.102797746658325,
|
|
"logps/chosen": -0.27081310749053955,
|
|
"logps/rejected": -0.271461546421051,
|
|
"loss": 0.31182369589805603,
|
|
"loss/core": 0.31182369589805603,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.4296600818634033,
|
|
"rewards/margins": -0.11104442924261093,
|
|
"rewards/rejected": 1.540704607963562,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09556232455354477,
|
|
"grad_norm": 222.0,
|
|
"ht_mnpo/logit": -0.07714849710464478,
|
|
"ht_mnpo/residual": -0.0846485048532486,
|
|
"ht_mnpo/residual_abs": 0.18699464201927185,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -2.0692691802978516,
|
|
"logits/rejected": -1.7881990671157837,
|
|
"logps/chosen": -0.27515479922294617,
|
|
"logps/rejected": -0.30582091212272644,
|
|
"loss": 0.3758378028869629,
|
|
"loss/core": 0.3758378028869629,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.941310167312622,
|
|
"rewards/margins": -0.7714849710464478,
|
|
"rewards/rejected": 2.7127950191497803,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.100340440781222,
|
|
"grad_norm": 1.6640625,
|
|
"ht_mnpo/logit": 0.08836665749549866,
|
|
"ht_mnpo/residual": 0.08086666464805603,
|
|
"ht_mnpo/residual_abs": 0.1014769896864891,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -2.1733365058898926,
|
|
"logits/rejected": -1.8746709823608398,
|
|
"logps/chosen": -0.29187583923339844,
|
|
"logps/rejected": -0.2903345227241516,
|
|
"loss": 0.12463197857141495,
|
|
"loss/core": 0.12463197857141495,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 2.0642318725585938,
|
|
"rewards/margins": 0.8836666941642761,
|
|
"rewards/rejected": 1.1805652379989624,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10511855700889924,
|
|
"grad_norm": 19.125,
|
|
"ht_mnpo/logit": 0.08112769573926926,
|
|
"ht_mnpo/residual": 0.07362769544124603,
|
|
"ht_mnpo/residual_abs": 0.09633813798427582,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -2.219769239425659,
|
|
"logits/rejected": -1.847038984298706,
|
|
"logps/chosen": -0.27057093381881714,
|
|
"logps/rejected": -0.27792128920555115,
|
|
"loss": 0.04170053079724312,
|
|
"loss/core": 0.04170053079724312,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.387577772140503,
|
|
"rewards/margins": 0.8112770318984985,
|
|
"rewards/rejected": 1.576300859451294,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10989667323657648,
|
|
"grad_norm": 24.875,
|
|
"ht_mnpo/logit": 0.07629410922527313,
|
|
"ht_mnpo/residual": 0.06879410147666931,
|
|
"ht_mnpo/residual_abs": 0.17269201576709747,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -2.1905808448791504,
|
|
"logits/rejected": -1.9591457843780518,
|
|
"logps/chosen": -0.2870820164680481,
|
|
"logps/rejected": -0.29649895429611206,
|
|
"loss": 0.2637905478477478,
|
|
"loss/core": 0.2637905478477478,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.930060863494873,
|
|
"rewards/margins": 0.7629408836364746,
|
|
"rewards/rejected": 2.1671202182769775,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11467478946425372,
|
|
"grad_norm": 193.0,
|
|
"ht_mnpo/logit": -0.001218314515426755,
|
|
"ht_mnpo/residual": -0.00871831364929676,
|
|
"ht_mnpo/residual_abs": 0.10168764740228653,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -2.316711664199829,
|
|
"logits/rejected": -2.1473333835601807,
|
|
"logps/chosen": -0.28898805379867554,
|
|
"logps/rejected": -0.2972700595855713,
|
|
"loss": 0.08895155042409897,
|
|
"loss/core": 0.08895155042409897,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.3815490007400513,
|
|
"rewards/margins": -0.012183129787445068,
|
|
"rewards/rejected": 1.3937320709228516,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11945290569193096,
|
|
"grad_norm": 8.375,
|
|
"ht_mnpo/logit": 0.03216388076543808,
|
|
"ht_mnpo/residual": 0.024663884192705154,
|
|
"ht_mnpo/residual_abs": 0.04822679981589317,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -2.516610622406006,
|
|
"logits/rejected": -2.1666038036346436,
|
|
"logps/chosen": -0.2692713737487793,
|
|
"logps/rejected": -0.27771317958831787,
|
|
"loss": 0.009285898879170418,
|
|
"loss/core": 0.009285898879170418,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.463373064994812,
|
|
"rewards/margins": 0.32163888216018677,
|
|
"rewards/rejected": 1.1417343616485596,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.12423102191960819,
|
|
"grad_norm": 13.0,
|
|
"ht_mnpo/logit": 0.028603965416550636,
|
|
"ht_mnpo/residual": 0.021103959530591965,
|
|
"ht_mnpo/residual_abs": 0.10350266844034195,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -2.189800977706909,
|
|
"logits/rejected": -1.9241540431976318,
|
|
"logps/chosen": -0.28411176800727844,
|
|
"logps/rejected": -0.2913844585418701,
|
|
"loss": 0.07367049157619476,
|
|
"loss/core": 0.07367049157619476,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.4351568222045898,
|
|
"rewards/margins": 0.2860395610332489,
|
|
"rewards/rejected": 1.149117350578308,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12900913814728543,
|
|
"grad_norm": 102.0,
|
|
"ht_mnpo/logit": 0.07047243416309357,
|
|
"ht_mnpo/residual": 0.06297242641448975,
|
|
"ht_mnpo/residual_abs": 0.17785733938217163,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -2.143766403198242,
|
|
"logits/rejected": -1.9228655099868774,
|
|
"logps/chosen": -0.26399558782577515,
|
|
"logps/rejected": -0.27883967757225037,
|
|
"loss": 0.3106807768344879,
|
|
"loss/core": 0.3106807768344879,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.427741765975952,
|
|
"rewards/margins": 0.7047242522239685,
|
|
"rewards/rejected": 1.723017692565918,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13378725437496267,
|
|
"grad_norm": 17.875,
|
|
"ht_mnpo/logit": 0.12879535555839539,
|
|
"ht_mnpo/residual": 0.12129535526037216,
|
|
"ht_mnpo/residual_abs": 0.12546807527542114,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -2.2192699909210205,
|
|
"logits/rejected": -2.05328369140625,
|
|
"logps/chosen": -0.28640010952949524,
|
|
"logps/rejected": -0.2853923439979553,
|
|
"loss": 0.17789167165756226,
|
|
"loss/core": 0.17789167165756226,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.192383289337158,
|
|
"rewards/margins": 1.2879533767700195,
|
|
"rewards/rejected": 0.9044297337532043,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.1385653706026399,
|
|
"grad_norm": 13.9375,
|
|
"ht_mnpo/logit": 0.09813486784696579,
|
|
"ht_mnpo/residual": 0.09063487499952316,
|
|
"ht_mnpo/residual_abs": 0.16477659344673157,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -2.2470571994781494,
|
|
"logits/rejected": -1.975572943687439,
|
|
"logps/chosen": -0.26441556215286255,
|
|
"logps/rejected": -0.25484567880630493,
|
|
"loss": 0.24567930400371552,
|
|
"loss/core": 0.24567930400371552,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.544703960418701,
|
|
"rewards/margins": 0.9813488125801086,
|
|
"rewards/rejected": 1.5633552074432373,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14334348683031714,
|
|
"grad_norm": 15.25,
|
|
"ht_mnpo/logit": 0.13078755140304565,
|
|
"ht_mnpo/residual": 0.12328755855560303,
|
|
"ht_mnpo/residual_abs": 0.14581966400146484,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -2.325977325439453,
|
|
"logits/rejected": -2.084429979324341,
|
|
"logps/chosen": -0.29626038670539856,
|
|
"logps/rejected": -0.3051699995994568,
|
|
"loss": 0.28483083844184875,
|
|
"loss/core": 0.28483083844184875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.152726888656616,
|
|
"rewards/margins": 1.307875394821167,
|
|
"rewards/rejected": 0.8448513746261597,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.14812160305799438,
|
|
"grad_norm": 6.1875,
|
|
"ht_mnpo/logit": 0.0646345466375351,
|
|
"ht_mnpo/residual": 0.05713455006480217,
|
|
"ht_mnpo/residual_abs": 0.13885614275932312,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -2.069967269897461,
|
|
"logits/rejected": -1.802931547164917,
|
|
"logps/chosen": -0.26899904012680054,
|
|
"logps/rejected": -0.27681413292884827,
|
|
"loss": 0.12919534742832184,
|
|
"loss/core": 0.12919534742832184,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.119464874267578,
|
|
"rewards/margins": 0.6463454961776733,
|
|
"rewards/rejected": 1.4731194972991943,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15289971928567161,
|
|
"grad_norm": 282.0,
|
|
"ht_mnpo/logit": 0.07195184379816055,
|
|
"ht_mnpo/residual": 0.06445185840129852,
|
|
"ht_mnpo/residual_abs": 0.16485223174095154,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -2.126732110977173,
|
|
"logits/rejected": -1.9010658264160156,
|
|
"logps/chosen": -0.3052712678909302,
|
|
"logps/rejected": -0.3145846426486969,
|
|
"loss": 0.3342359960079193,
|
|
"loss/core": 0.3342359960079193,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.308896541595459,
|
|
"rewards/margins": 0.7195185422897339,
|
|
"rewards/rejected": 1.589377760887146,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15767783551334885,
|
|
"grad_norm": 16.25,
|
|
"ht_mnpo/logit": 0.13453207910060883,
|
|
"ht_mnpo/residual": 0.1270320862531662,
|
|
"ht_mnpo/residual_abs": 0.17115262150764465,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -2.2729172706604004,
|
|
"logits/rejected": -1.9023488759994507,
|
|
"logps/chosen": -0.26925191283226013,
|
|
"logps/rejected": -0.28397589921951294,
|
|
"loss": 0.7371556162834167,
|
|
"loss/core": 0.7371556162834167,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.71228289604187,
|
|
"rewards/margins": 1.3453209400177002,
|
|
"rewards/rejected": 1.3669623136520386,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.1624559517410261,
|
|
"grad_norm": 26.25,
|
|
"ht_mnpo/logit": 0.11945419013500214,
|
|
"ht_mnpo/residual": 0.11195418983697891,
|
|
"ht_mnpo/residual_abs": 0.14006376266479492,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -2.131185531616211,
|
|
"logits/rejected": -1.9534437656402588,
|
|
"logps/chosen": -0.28382784128189087,
|
|
"logps/rejected": -0.2792891263961792,
|
|
"loss": 0.2930060029029846,
|
|
"loss/core": 0.2930060029029846,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2859272956848145,
|
|
"rewards/margins": 1.1945419311523438,
|
|
"rewards/rejected": 1.0913852453231812,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16723406796870333,
|
|
"grad_norm": 5.625,
|
|
"ht_mnpo/logit": 0.039273008704185486,
|
|
"ht_mnpo/residual": 0.03177300840616226,
|
|
"ht_mnpo/residual_abs": 0.06933975219726562,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -2.2991464138031006,
|
|
"logits/rejected": -2.073848247528076,
|
|
"logps/chosen": -0.2741314768791199,
|
|
"logps/rejected": -0.276893675327301,
|
|
"loss": 0.032467905431985855,
|
|
"loss/core": 0.032467905431985855,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6739985942840576,
|
|
"rewards/margins": 0.39273008704185486,
|
|
"rewards/rejected": 1.2812687158584595,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17201218419638056,
|
|
"grad_norm": 17.75,
|
|
"ht_mnpo/logit": 0.09057626128196716,
|
|
"ht_mnpo/residual": 0.08307625353336334,
|
|
"ht_mnpo/residual_abs": 0.18893863260746002,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -2.2697224617004395,
|
|
"logits/rejected": -2.026869058609009,
|
|
"logps/chosen": -0.2926976680755615,
|
|
"logps/rejected": -0.3049258589744568,
|
|
"loss": 0.3669869601726532,
|
|
"loss/core": 0.3669869601726532,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.28019380569458,
|
|
"rewards/margins": 0.9057625532150269,
|
|
"rewards/rejected": 1.3744308948516846,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17679030042405783,
|
|
"grad_norm": 0.76171875,
|
|
"ht_mnpo/logit": 0.036733776330947876,
|
|
"ht_mnpo/residual": 0.02923377975821495,
|
|
"ht_mnpo/residual_abs": 0.051221203058958054,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -2.279662847518921,
|
|
"logits/rejected": -2.0322604179382324,
|
|
"logps/chosen": -0.3091420829296112,
|
|
"logps/rejected": -0.325103759765625,
|
|
"loss": 0.01263344008475542,
|
|
"loss/core": 0.01263344008475542,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.3614156246185303,
|
|
"rewards/margins": 0.36733779311180115,
|
|
"rewards/rejected": 0.9940778613090515,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18156841665173507,
|
|
"grad_norm": 38.25,
|
|
"ht_mnpo/logit": 0.15630559623241425,
|
|
"ht_mnpo/residual": 0.14880560338497162,
|
|
"ht_mnpo/residual_abs": 0.1863393634557724,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -2.1544783115386963,
|
|
"logits/rejected": -1.9566150903701782,
|
|
"logps/chosen": -0.28683212399482727,
|
|
"logps/rejected": -0.2882298231124878,
|
|
"loss": 0.36742496490478516,
|
|
"loss/core": 0.36742496490478516,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6428446769714355,
|
|
"rewards/margins": 1.5630559921264648,
|
|
"rewards/rejected": 1.0797886848449707,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.1863465328794123,
|
|
"grad_norm": 16.75,
|
|
"ht_mnpo/logit": 0.06805960834026337,
|
|
"ht_mnpo/residual": 0.06055961176753044,
|
|
"ht_mnpo/residual_abs": 0.09312574565410614,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -2.1282095909118652,
|
|
"logits/rejected": -1.9009084701538086,
|
|
"logps/chosen": -0.30094921588897705,
|
|
"logps/rejected": -0.28223350644111633,
|
|
"loss": 0.059374578297138214,
|
|
"loss/core": 0.059374578297138214,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.1673290729522705,
|
|
"rewards/margins": 0.6805960536003113,
|
|
"rewards/rejected": 1.4867329597473145,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19112464910708954,
|
|
"grad_norm": 144.0,
|
|
"ht_mnpo/logit": 0.10441961139440536,
|
|
"ht_mnpo/residual": 0.09691962599754333,
|
|
"ht_mnpo/residual_abs": 0.16735684871673584,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -2.2485835552215576,
|
|
"logits/rejected": -2.009777545928955,
|
|
"logps/chosen": -0.2734311521053314,
|
|
"logps/rejected": -0.2917312681674957,
|
|
"loss": 0.35473141074180603,
|
|
"loss/core": 0.35473141074180603,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.6118438243865967,
|
|
"rewards/margins": 1.0441962480545044,
|
|
"rewards/rejected": 1.5676480531692505,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19590276533476678,
|
|
"grad_norm": 14.375,
|
|
"ht_mnpo/logit": 0.11748448759317398,
|
|
"ht_mnpo/residual": 0.10998449474573135,
|
|
"ht_mnpo/residual_abs": 0.1774880737066269,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -2.0733883380889893,
|
|
"logits/rejected": -1.8939872980117798,
|
|
"logps/chosen": -0.32192596793174744,
|
|
"logps/rejected": -0.316753089427948,
|
|
"loss": 0.2780437469482422,
|
|
"loss/core": 0.2780437469482422,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.506334066390991,
|
|
"rewards/margins": 1.1748448610305786,
|
|
"rewards/rejected": 1.3314893245697021,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.200680881562444,
|
|
"grad_norm": 4.09375,
|
|
"ht_mnpo/logit": -0.01023789867758751,
|
|
"ht_mnpo/residual": -0.017737898975610733,
|
|
"ht_mnpo/residual_abs": 0.08284340798854828,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -2.1783037185668945,
|
|
"logits/rejected": -1.9191534519195557,
|
|
"logps/chosen": -0.3014274537563324,
|
|
"logps/rejected": -0.30610206723213196,
|
|
"loss": 0.0633908361196518,
|
|
"loss/core": 0.0633908361196518,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.9192719459533691,
|
|
"rewards/margins": -0.10237900167703629,
|
|
"rewards/rejected": 1.021651029586792,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20545899779012125,
|
|
"grad_norm": 20.25,
|
|
"ht_mnpo/logit": 0.04202476888895035,
|
|
"ht_mnpo/residual": 0.03452477231621742,
|
|
"ht_mnpo/residual_abs": 0.07536228746175766,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -2.296433210372925,
|
|
"logits/rejected": -2.0917699337005615,
|
|
"logps/chosen": -0.28844669461250305,
|
|
"logps/rejected": -0.30069172382354736,
|
|
"loss": 0.03301898017525673,
|
|
"loss/core": 0.03301898017525673,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5373624563217163,
|
|
"rewards/margins": 0.4202477037906647,
|
|
"rewards/rejected": 1.117114782333374,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.2102371140177985,
|
|
"grad_norm": 241.0,
|
|
"ht_mnpo/logit": 0.13244636356830597,
|
|
"ht_mnpo/residual": 0.12494637072086334,
|
|
"ht_mnpo/residual_abs": 0.161330446600914,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -2.25962495803833,
|
|
"logits/rejected": -2.10876727104187,
|
|
"logps/chosen": -0.3020437955856323,
|
|
"logps/rejected": -0.30034393072128296,
|
|
"loss": 0.20618359744548798,
|
|
"loss/core": 0.20618359744548798,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1069769859313965,
|
|
"rewards/margins": 1.3244636058807373,
|
|
"rewards/rejected": 1.7825132608413696,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.21501523024547572,
|
|
"grad_norm": 0.1259765625,
|
|
"ht_mnpo/logit": 0.03315456956624985,
|
|
"ht_mnpo/residual": 0.025654572993516922,
|
|
"ht_mnpo/residual_abs": 0.04697638005018234,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -2.288386583328247,
|
|
"logits/rejected": -2.069204807281494,
|
|
"logps/chosen": -0.308716356754303,
|
|
"logps/rejected": -0.3130949139595032,
|
|
"loss": 0.011724801734089851,
|
|
"loss/core": 0.011724801734089851,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.28463613986969,
|
|
"rewards/margins": 0.33154574036598206,
|
|
"rewards/rejected": 0.9530903697013855,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21979334647315296,
|
|
"grad_norm": 2.8125,
|
|
"ht_mnpo/logit": 0.0781010240316391,
|
|
"ht_mnpo/residual": 0.07060102373361588,
|
|
"ht_mnpo/residual_abs": 0.14166823029518127,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -2.4182097911834717,
|
|
"logits/rejected": -2.011842966079712,
|
|
"logps/chosen": -0.2857847809791565,
|
|
"logps/rejected": -0.2783315181732178,
|
|
"loss": 0.1992371380329132,
|
|
"loss/core": 0.1992371380329132,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.4587364196777344,
|
|
"rewards/margins": 0.7810102105140686,
|
|
"rewards/rejected": 1.6777263879776,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245714627008302,
|
|
"grad_norm": 3.3125,
|
|
"ht_mnpo/logit": 0.13024064898490906,
|
|
"ht_mnpo/residual": 0.12274064868688583,
|
|
"ht_mnpo/residual_abs": 0.15162988007068634,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -2.106680393218994,
|
|
"logits/rejected": -1.7685272693634033,
|
|
"logps/chosen": -0.3049893379211426,
|
|
"logps/rejected": -0.3265388607978821,
|
|
"loss": 0.2962871193885803,
|
|
"loss/core": 0.2962871193885803,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.6692185401916504,
|
|
"rewards/margins": 1.3024064302444458,
|
|
"rewards/rejected": 1.366811990737915,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22934957892850744,
|
|
"grad_norm": 1.015625,
|
|
"ht_mnpo/logit": -0.012782636098563671,
|
|
"ht_mnpo/residual": -0.02028263546526432,
|
|
"ht_mnpo/residual_abs": 0.10213587433099747,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -2.342169761657715,
|
|
"logits/rejected": -1.9928566217422485,
|
|
"logps/chosen": -0.2670109272003174,
|
|
"logps/rejected": -0.27326709032058716,
|
|
"loss": 0.09348958730697632,
|
|
"loss/core": 0.09348958730697632,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 1.772402048110962,
|
|
"rewards/margins": -0.12782634794712067,
|
|
"rewards/rejected": 1.900228500366211,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23412769515618467,
|
|
"grad_norm": 0.66015625,
|
|
"ht_mnpo/logit": 0.038539618253707886,
|
|
"ht_mnpo/residual": 0.03103962168097496,
|
|
"ht_mnpo/residual_abs": 0.07602906227111816,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -2.4512276649475098,
|
|
"logits/rejected": -2.143380880355835,
|
|
"logps/chosen": -0.2683179974555969,
|
|
"logps/rejected": -0.2903439998626709,
|
|
"loss": 0.0462082214653492,
|
|
"loss/core": 0.0462082214653492,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.954590082168579,
|
|
"rewards/margins": 0.38539624214172363,
|
|
"rewards/rejected": 1.5691938400268555,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.2389058113838619,
|
|
"grad_norm": 428.0,
|
|
"ht_mnpo/logit": 0.01622030697762966,
|
|
"ht_mnpo/residual": 0.008720312267541885,
|
|
"ht_mnpo/residual_abs": 0.15026597678661346,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -2.188828706741333,
|
|
"logits/rejected": -1.8585360050201416,
|
|
"logps/chosen": -0.3078772723674774,
|
|
"logps/rejected": -0.311143159866333,
|
|
"loss": 0.2350800484418869,
|
|
"loss/core": 0.2350800484418869,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 1.6038856506347656,
|
|
"rewards/margins": 0.1622031033039093,
|
|
"rewards/rejected": 1.4416824579238892,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24368392761153915,
|
|
"grad_norm": 2.234375,
|
|
"ht_mnpo/logit": 0.04869508370757103,
|
|
"ht_mnpo/residual": 0.041195083409547806,
|
|
"ht_mnpo/residual_abs": 0.08923383802175522,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -2.215907096862793,
|
|
"logits/rejected": -1.984976053237915,
|
|
"logps/chosen": -0.2803708612918854,
|
|
"logps/rejected": -0.30261167883872986,
|
|
"loss": 0.08057855069637299,
|
|
"loss/core": 0.08057855069637299,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.605520248413086,
|
|
"rewards/margins": 0.4869508147239685,
|
|
"rewards/rejected": 1.1185696125030518,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.24846204383921638,
|
|
"grad_norm": 6.5625,
|
|
"ht_mnpo/logit": 0.06030106544494629,
|
|
"ht_mnpo/residual": 0.05280106142163277,
|
|
"ht_mnpo/residual_abs": 0.29066184163093567,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -2.153928279876709,
|
|
"logits/rejected": -1.8087905645370483,
|
|
"logps/chosen": -0.2784316837787628,
|
|
"logps/rejected": -0.2740718424320221,
|
|
"loss": 0.6678136587142944,
|
|
"loss/core": 0.6678136587142944,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 3.0530622005462646,
|
|
"rewards/margins": 0.6030107736587524,
|
|
"rewards/rejected": 2.450051784515381,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25324016006689365,
|
|
"grad_norm": 61.25,
|
|
"ht_mnpo/logit": 0.0862557515501976,
|
|
"ht_mnpo/residual": 0.07875575870275497,
|
|
"ht_mnpo/residual_abs": 0.16950246691703796,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.1912569999694824,
|
|
"logits/rejected": -1.9450289011001587,
|
|
"logps/chosen": -0.2741378843784332,
|
|
"logps/rejected": -0.31863006949424744,
|
|
"loss": 0.3198677599430084,
|
|
"loss/core": 0.3198677599430084,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 2.4567675590515137,
|
|
"rewards/margins": 0.8625577092170715,
|
|
"rewards/rejected": 1.5942096710205078,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.25801827629457086,
|
|
"grad_norm": 79.5,
|
|
"ht_mnpo/logit": 0.04499917849898338,
|
|
"ht_mnpo/residual": 0.03749917820096016,
|
|
"ht_mnpo/residual_abs": 0.08561719208955765,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -2.34749698638916,
|
|
"logits/rejected": -2.015653371810913,
|
|
"logps/chosen": -0.26775068044662476,
|
|
"logps/rejected": -0.2911599576473236,
|
|
"loss": 0.04689795523881912,
|
|
"loss/core": 0.04689795523881912,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.4798815250396729,
|
|
"rewards/margins": 0.44999170303344727,
|
|
"rewards/rejected": 1.0298898220062256,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627963925222481,
|
|
"grad_norm": 62.0,
|
|
"ht_mnpo/logit": 0.07096774876117706,
|
|
"ht_mnpo/residual": 0.06346776336431503,
|
|
"ht_mnpo/residual_abs": 0.1662285178899765,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -2.1291439533233643,
|
|
"logits/rejected": -1.8797943592071533,
|
|
"logps/chosen": -0.2925297021865845,
|
|
"logps/rejected": -0.33546027541160583,
|
|
"loss": 0.3264152407646179,
|
|
"loss/core": 0.3264152407646179,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4164235591888428,
|
|
"rewards/margins": 0.7096775770187378,
|
|
"rewards/rejected": 1.7067458629608154,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26757450874992533,
|
|
"grad_norm": 3.3125,
|
|
"ht_mnpo/logit": 0.05690809339284897,
|
|
"ht_mnpo/residual": 0.04940810054540634,
|
|
"ht_mnpo/residual_abs": 0.07465264946222305,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -2.1697328090667725,
|
|
"logits/rejected": -1.9812034368515015,
|
|
"logps/chosen": -0.2690741717815399,
|
|
"logps/rejected": -0.27334779500961304,
|
|
"loss": 0.031530968844890594,
|
|
"loss/core": 0.031530968844890594,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.834280014038086,
|
|
"rewards/margins": 0.5690810084342957,
|
|
"rewards/rejected": 1.2651989459991455,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723526249776026,
|
|
"grad_norm": 15.375,
|
|
"ht_mnpo/logit": 0.05523994565010071,
|
|
"ht_mnpo/residual": 0.047739945352077484,
|
|
"ht_mnpo/residual_abs": 0.05985153466463089,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -2.1334269046783447,
|
|
"logits/rejected": -2.0200657844543457,
|
|
"logps/chosen": -0.3056999146938324,
|
|
"logps/rejected": -0.3090917766094208,
|
|
"loss": 0.02557971142232418,
|
|
"loss/core": 0.02557971142232418,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.5456047058105469,
|
|
"rewards/margins": 0.5523993372917175,
|
|
"rewards/rejected": 0.9932052493095398,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.2771307412052798,
|
|
"grad_norm": 5.875,
|
|
"ht_mnpo/logit": 0.03501705452799797,
|
|
"ht_mnpo/residual": 0.027517059817910194,
|
|
"ht_mnpo/residual_abs": 0.06787494570016861,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -2.2637059688568115,
|
|
"logits/rejected": -2.0740151405334473,
|
|
"logps/chosen": -0.3052566349506378,
|
|
"logps/rejected": -0.30728772282600403,
|
|
"loss": 0.02679051086306572,
|
|
"loss/core": 0.02679051086306572,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.183258056640625,
|
|
"rewards/margins": 0.3501706123352051,
|
|
"rewards/rejected": 0.8330875635147095,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28190885743295707,
|
|
"grad_norm": 28.5,
|
|
"ht_mnpo/logit": 0.14480124413967133,
|
|
"ht_mnpo/residual": 0.1373012363910675,
|
|
"ht_mnpo/residual_abs": 0.14611732959747314,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -2.285439968109131,
|
|
"logits/rejected": -2.081019878387451,
|
|
"logps/chosen": -0.3001931607723236,
|
|
"logps/rejected": -0.30784377455711365,
|
|
"loss": 0.6604355573654175,
|
|
"loss/core": 0.6604355573654175,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.0780365467071533,
|
|
"rewards/margins": 1.4480124711990356,
|
|
"rewards/rejected": 0.6300240755081177,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.2866869736606343,
|
|
"grad_norm": 4.90625,
|
|
"ht_mnpo/logit": 0.06456714868545532,
|
|
"ht_mnpo/residual": 0.0570671446621418,
|
|
"ht_mnpo/residual_abs": 0.14857563376426697,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -2.2999801635742188,
|
|
"logits/rejected": -1.9922819137573242,
|
|
"logps/chosen": -0.28616541624069214,
|
|
"logps/rejected": -0.2884747087955475,
|
|
"loss": 0.24228546023368835,
|
|
"loss/core": 0.24228546023368835,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.1733169555664062,
|
|
"rewards/margins": 0.6456714868545532,
|
|
"rewards/rejected": 1.5276457071304321,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.29146508988831155,
|
|
"grad_norm": 3.8125,
|
|
"ht_mnpo/logit": 0.04495188593864441,
|
|
"ht_mnpo/residual": 0.037451885640621185,
|
|
"ht_mnpo/residual_abs": 0.20770028233528137,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -2.157289981842041,
|
|
"logits/rejected": -1.8468250036239624,
|
|
"logps/chosen": -0.28001922369003296,
|
|
"logps/rejected": -0.29186612367630005,
|
|
"loss": 0.4534494876861572,
|
|
"loss/core": 0.4534494876861572,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 2.569370746612549,
|
|
"rewards/margins": 0.4495188593864441,
|
|
"rewards/rejected": 2.119851589202881,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.29624320611598876,
|
|
"grad_norm": 7.28125,
|
|
"ht_mnpo/logit": 0.024377625435590744,
|
|
"ht_mnpo/residual": 0.01687762141227722,
|
|
"ht_mnpo/residual_abs": 0.08526398241519928,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -2.399658441543579,
|
|
"logits/rejected": -2.0769169330596924,
|
|
"logps/chosen": -0.2818771004676819,
|
|
"logps/rejected": -0.3100544810295105,
|
|
"loss": 0.04117077589035034,
|
|
"loss/core": 0.04117077589035034,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.4926480054855347,
|
|
"rewards/margins": 0.24377623200416565,
|
|
"rewards/rejected": 1.2488718032836914,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.301021322343666,
|
|
"grad_norm": 1.4609375,
|
|
"ht_mnpo/logit": 0.1227768212556839,
|
|
"ht_mnpo/residual": 0.11527681350708008,
|
|
"ht_mnpo/residual_abs": 0.14000196754932404,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -2.276912212371826,
|
|
"logits/rejected": -2.0756640434265137,
|
|
"logps/chosen": -0.29674750566482544,
|
|
"logps/rejected": -0.30335813760757446,
|
|
"loss": 0.5935509204864502,
|
|
"loss/core": 0.5935509204864502,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0897817611694336,
|
|
"rewards/margins": 1.2277681827545166,
|
|
"rewards/rejected": 0.8620136380195618,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.30579943857134323,
|
|
"grad_norm": 240.0,
|
|
"ht_mnpo/logit": 0.22828857600688934,
|
|
"ht_mnpo/residual": 0.22078856825828552,
|
|
"ht_mnpo/residual_abs": 0.23243160545825958,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -2.360725164413452,
|
|
"logits/rejected": -2.132331371307373,
|
|
"logps/chosen": -0.28701794147491455,
|
|
"logps/rejected": -0.28683117032051086,
|
|
"loss": 1.1871272325515747,
|
|
"loss/core": 1.1871272325515747,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 3.358243465423584,
|
|
"rewards/margins": 2.2828855514526367,
|
|
"rewards/rejected": 1.0753579139709473,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.3105775547990205,
|
|
"grad_norm": 1.4921875,
|
|
"ht_mnpo/logit": 0.06212912127375603,
|
|
"ht_mnpo/residual": 0.0546291284263134,
|
|
"ht_mnpo/residual_abs": 0.20826132595539093,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -2.275365114212036,
|
|
"logits/rejected": -2.0728507041931152,
|
|
"logps/chosen": -0.26758795976638794,
|
|
"logps/rejected": -0.27879461646080017,
|
|
"loss": 0.41284671425819397,
|
|
"loss/core": 0.41284671425819397,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.737895965576172,
|
|
"rewards/margins": 0.6212912797927856,
|
|
"rewards/rejected": 2.116604804992676,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.3153556710266977,
|
|
"grad_norm": 0.81640625,
|
|
"ht_mnpo/logit": 0.06797120720148087,
|
|
"ht_mnpo/residual": 0.06047120690345764,
|
|
"ht_mnpo/residual_abs": 0.07851966470479965,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -2.2605273723602295,
|
|
"logits/rejected": -2.134922504425049,
|
|
"logps/chosen": -0.2739706039428711,
|
|
"logps/rejected": -0.2764506936073303,
|
|
"loss": 0.05020144581794739,
|
|
"loss/core": 0.05020144581794739,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6947124004364014,
|
|
"rewards/margins": 0.6797120571136475,
|
|
"rewards/rejected": 1.0150004625320435,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32013378725437497,
|
|
"grad_norm": 0.75390625,
|
|
"ht_mnpo/logit": 0.19798977673053741,
|
|
"ht_mnpo/residual": 0.1904897689819336,
|
|
"ht_mnpo/residual_abs": 0.21845956146717072,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -2.047623872756958,
|
|
"logits/rejected": -1.9703845977783203,
|
|
"logps/chosen": -0.29154783487319946,
|
|
"logps/rejected": -0.3145486116409302,
|
|
"loss": 0.6114920973777771,
|
|
"loss/core": 0.6114920973777771,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0861105918884277,
|
|
"rewards/margins": 1.9798977375030518,
|
|
"rewards/rejected": 1.106212854385376,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3249119034820522,
|
|
"grad_norm": 41.5,
|
|
"ht_mnpo/logit": 0.11068092286586761,
|
|
"ht_mnpo/residual": 0.10318093001842499,
|
|
"ht_mnpo/residual_abs": 0.25756824016571045,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -1.9539859294891357,
|
|
"logits/rejected": -1.8528114557266235,
|
|
"logps/chosen": -0.2756306827068329,
|
|
"logps/rejected": -0.30442994832992554,
|
|
"loss": 0.9918975830078125,
|
|
"loss/core": 0.9918975830078125,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.8355953693389893,
|
|
"rewards/margins": 1.106809377670288,
|
|
"rewards/rejected": 1.7287861108779907,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.32969001970972944,
|
|
"grad_norm": 0.703125,
|
|
"ht_mnpo/logit": 0.015581438317894936,
|
|
"ht_mnpo/residual": 0.008081444539129734,
|
|
"ht_mnpo/residual_abs": 0.10090570151805878,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -2.3269078731536865,
|
|
"logits/rejected": -2.0692667961120605,
|
|
"logps/chosen": -0.2906230390071869,
|
|
"logps/rejected": -0.29532790184020996,
|
|
"loss": 0.09663812071084976,
|
|
"loss/core": 0.09663812071084976,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.2236311435699463,
|
|
"rewards/margins": 0.15581440925598145,
|
|
"rewards/rejected": 1.0678166151046753,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33446813593740665,
|
|
"grad_norm": 1.4296875,
|
|
"ht_mnpo/logit": 0.07701550424098969,
|
|
"ht_mnpo/residual": 0.06951549649238586,
|
|
"ht_mnpo/residual_abs": 0.08727418631315231,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -2.214233875274658,
|
|
"logits/rejected": -1.9848973751068115,
|
|
"logps/chosen": -0.3011823892593384,
|
|
"logps/rejected": -0.294931560754776,
|
|
"loss": 0.11329641193151474,
|
|
"loss/core": 0.11329641193151474,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9033756256103516,
|
|
"rewards/margins": 0.7701549530029297,
|
|
"rewards/rejected": 1.133220911026001,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3392462521650839,
|
|
"grad_norm": 416.0,
|
|
"ht_mnpo/logit": 0.0463683120906353,
|
|
"ht_mnpo/residual": 0.03886830434203148,
|
|
"ht_mnpo/residual_abs": 0.14574840664863586,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -2.264720916748047,
|
|
"logits/rejected": -1.9842700958251953,
|
|
"logps/chosen": -0.2861303985118866,
|
|
"logps/rejected": -0.2938997149467468,
|
|
"loss": 0.17697656154632568,
|
|
"loss/core": 0.17697656154632568,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.538461685180664,
|
|
"rewards/margins": 0.46368303894996643,
|
|
"rewards/rejected": 2.0747783184051514,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.3440243683927611,
|
|
"grad_norm": 494.0,
|
|
"ht_mnpo/logit": 0.12127704918384552,
|
|
"ht_mnpo/residual": 0.1137770414352417,
|
|
"ht_mnpo/residual_abs": 0.1548972874879837,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -2.3002190589904785,
|
|
"logits/rejected": -2.164665460586548,
|
|
"logps/chosen": -0.2934378981590271,
|
|
"logps/rejected": -0.2929591238498688,
|
|
"loss": 0.482862651348114,
|
|
"loss/core": 0.482862651348114,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.9623796939849854,
|
|
"rewards/margins": 1.2127704620361328,
|
|
"rewards/rejected": 0.7496089339256287,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.3488024846204384,
|
|
"grad_norm": 424.0,
|
|
"ht_mnpo/logit": 0.010891343466937542,
|
|
"ht_mnpo/residual": 0.00339134712703526,
|
|
"ht_mnpo/residual_abs": 0.08606524765491486,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -2.2042396068573,
|
|
"logits/rejected": -1.8955295085906982,
|
|
"logps/chosen": -0.31718710064888,
|
|
"logps/rejected": -0.3305971920490265,
|
|
"loss": 0.12844952940940857,
|
|
"loss/core": 0.12844952940940857,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.4607499837875366,
|
|
"rewards/margins": 0.10891339927911758,
|
|
"rewards/rejected": 1.3518364429473877,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.35358060084811566,
|
|
"grad_norm": 251.0,
|
|
"ht_mnpo/logit": 0.11479277908802032,
|
|
"ht_mnpo/residual": 0.10729275643825531,
|
|
"ht_mnpo/residual_abs": 0.11697915941476822,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.4074511528015137,
|
|
"logits/rejected": -2.101377487182617,
|
|
"logps/chosen": -0.2906573712825775,
|
|
"logps/rejected": -0.29892730712890625,
|
|
"loss": 0.11968455463647842,
|
|
"loss/core": 0.11968455463647842,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.02966046333313,
|
|
"rewards/margins": 1.1479276418685913,
|
|
"rewards/rejected": 0.8817326426506042,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35835871707579287,
|
|
"grad_norm": 68.0,
|
|
"ht_mnpo/logit": 0.09350790083408356,
|
|
"ht_mnpo/residual": 0.08600790798664093,
|
|
"ht_mnpo/residual_abs": 0.1067660003900528,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -2.127977132797241,
|
|
"logits/rejected": -1.9483768939971924,
|
|
"logps/chosen": -0.26925674080848694,
|
|
"logps/rejected": -0.2880834937095642,
|
|
"loss": 0.106331467628479,
|
|
"loss/core": 0.106331467628479,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.7844947576522827,
|
|
"rewards/margins": 0.9350789785385132,
|
|
"rewards/rejected": 0.8494156002998352,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36313683330347013,
|
|
"grad_norm": 1.265625,
|
|
"ht_mnpo/logit": 0.004885909613221884,
|
|
"ht_mnpo/residual": -0.002614091383293271,
|
|
"ht_mnpo/residual_abs": 0.10483495146036148,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -2.168792724609375,
|
|
"logits/rejected": -1.950922966003418,
|
|
"logps/chosen": -0.28580015897750854,
|
|
"logps/rejected": -0.30009040236473083,
|
|
"loss": 0.0811811238527298,
|
|
"loss/core": 0.0811811238527298,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6719993352890015,
|
|
"rewards/margins": 0.04885910823941231,
|
|
"rewards/rejected": 1.6231400966644287,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36791494953114734,
|
|
"grad_norm": 22.625,
|
|
"ht_mnpo/logit": 0.1782342940568924,
|
|
"ht_mnpo/residual": 0.17073427140712738,
|
|
"ht_mnpo/residual_abs": 0.20563189685344696,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -2.0228257179260254,
|
|
"logits/rejected": -1.7602002620697021,
|
|
"logps/chosen": -0.28422048687934875,
|
|
"logps/rejected": -0.282476007938385,
|
|
"loss": 0.31415554881095886,
|
|
"loss/core": 0.31415554881095886,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.310380458831787,
|
|
"rewards/margins": 1.7823426723480225,
|
|
"rewards/rejected": 1.5280377864837646,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726930657588246,
|
|
"grad_norm": 19.875,
|
|
"ht_mnpo/logit": 0.18474507331848145,
|
|
"ht_mnpo/residual": 0.17724506556987762,
|
|
"ht_mnpo/residual_abs": 0.24547035992145538,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -2.205859422683716,
|
|
"logits/rejected": -1.906855821609497,
|
|
"logps/chosen": -0.2814488708972931,
|
|
"logps/rejected": -0.28038260340690613,
|
|
"loss": 0.5938295125961304,
|
|
"loss/core": 0.5938295125961304,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.566596269607544,
|
|
"rewards/margins": 1.847450852394104,
|
|
"rewards/rejected": 1.7191451787948608,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.3774711819865018,
|
|
"grad_norm": 528.0,
|
|
"ht_mnpo/logit": 0.11254217475652695,
|
|
"ht_mnpo/residual": 0.10504218190908432,
|
|
"ht_mnpo/residual_abs": 0.1626928150653839,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -2.29341983795166,
|
|
"logits/rejected": -1.993295431137085,
|
|
"logps/chosen": -0.27144044637680054,
|
|
"logps/rejected": -0.29052314162254333,
|
|
"loss": 0.3823533058166504,
|
|
"loss/core": 0.3823533058166504,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.345606565475464,
|
|
"rewards/margins": 1.1254217624664307,
|
|
"rewards/rejected": 1.2201849222183228,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.3822492982141791,
|
|
"grad_norm": 1.53125,
|
|
"ht_mnpo/logit": -0.04141603037714958,
|
|
"ht_mnpo/residual": -0.0489160381257534,
|
|
"ht_mnpo/residual_abs": 0.11461643129587173,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -2.3432741165161133,
|
|
"logits/rejected": -1.979839563369751,
|
|
"logps/chosen": -0.26395756006240845,
|
|
"logps/rejected": -0.2680288255214691,
|
|
"loss": 0.25311708450317383,
|
|
"loss/core": 0.25311708450317383,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.3107450008392334,
|
|
"rewards/margins": -0.41416025161743164,
|
|
"rewards/rejected": 1.724905252456665,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.3870274144418563,
|
|
"grad_norm": 796.0,
|
|
"ht_mnpo/logit": 0.0915079265832901,
|
|
"ht_mnpo/residual": 0.08400792628526688,
|
|
"ht_mnpo/residual_abs": 0.20653638243675232,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -2.0984344482421875,
|
|
"logits/rejected": -1.869283676147461,
|
|
"logps/chosen": -0.26586979627609253,
|
|
"logps/rejected": -0.2844234108924866,
|
|
"loss": 0.7313702702522278,
|
|
"loss/core": 0.7313702702522278,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7626240253448486,
|
|
"rewards/margins": 0.9150789976119995,
|
|
"rewards/rejected": 1.8475449085235596,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39180553066953355,
|
|
"grad_norm": 5.09375,
|
|
"ht_mnpo/logit": 0.07326017320156097,
|
|
"ht_mnpo/residual": 0.06576018035411835,
|
|
"ht_mnpo/residual_abs": 0.08752019703388214,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -2.194375514984131,
|
|
"logits/rejected": -2.0342750549316406,
|
|
"logps/chosen": -0.281860888004303,
|
|
"logps/rejected": -0.3041122555732727,
|
|
"loss": 0.044236548244953156,
|
|
"loss/core": 0.044236548244953156,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6896095275878906,
|
|
"rewards/margins": 0.7326017618179321,
|
|
"rewards/rejected": 0.9570077061653137,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.39658364689721076,
|
|
"grad_norm": 640.0,
|
|
"ht_mnpo/logit": -0.07659461349248886,
|
|
"ht_mnpo/residual": -0.08409460633993149,
|
|
"ht_mnpo/residual_abs": 0.14910180866718292,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -2.4320578575134277,
|
|
"logits/rejected": -2.1560490131378174,
|
|
"logps/chosen": -0.2776671051979065,
|
|
"logps/rejected": -0.293962299823761,
|
|
"loss": 0.5594010949134827,
|
|
"loss/core": 0.5594010949134827,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.1026519536972046,
|
|
"rewards/margins": -0.7659462690353394,
|
|
"rewards/rejected": 1.868598222732544,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.401361763124888,
|
|
"grad_norm": 185.0,
|
|
"ht_mnpo/logit": 0.076129250228405,
|
|
"ht_mnpo/residual": 0.06862926483154297,
|
|
"ht_mnpo/residual_abs": 0.1663965880870819,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -2.326869249343872,
|
|
"logits/rejected": -2.0747084617614746,
|
|
"logps/chosen": -0.2761536240577698,
|
|
"logps/rejected": -0.28266650438308716,
|
|
"loss": 0.34604477882385254,
|
|
"loss/core": 0.34604477882385254,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.188345432281494,
|
|
"rewards/margins": 0.761292576789856,
|
|
"rewards/rejected": 1.4270528554916382,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.40613987935256524,
|
|
"grad_norm": 0.42578125,
|
|
"ht_mnpo/logit": 0.07638172805309296,
|
|
"ht_mnpo/residual": 0.06888173520565033,
|
|
"ht_mnpo/residual_abs": 0.11476260423660278,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -2.2939515113830566,
|
|
"logits/rejected": -2.034564971923828,
|
|
"logps/chosen": -0.27834567427635193,
|
|
"logps/rejected": -0.3104603886604309,
|
|
"loss": 0.17839036881923676,
|
|
"loss/core": 0.17839036881923676,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.17610239982605,
|
|
"rewards/margins": 0.7638173699378967,
|
|
"rewards/rejected": 1.4122850894927979,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.4109179955802425,
|
|
"grad_norm": 3.15625,
|
|
"ht_mnpo/logit": -0.03292038291692734,
|
|
"ht_mnpo/residual": -0.04042038321495056,
|
|
"ht_mnpo/residual_abs": 0.18147443234920502,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -2.2801480293273926,
|
|
"logits/rejected": -2.009410858154297,
|
|
"logps/chosen": -0.2517848610877991,
|
|
"logps/rejected": -0.28678059577941895,
|
|
"loss": 0.35211431980133057,
|
|
"loss/core": 0.35211431980133057,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.0473055839538574,
|
|
"rewards/margins": -0.3292037844657898,
|
|
"rewards/rejected": 2.376509428024292,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.4156961118079197,
|
|
"grad_norm": 38.5,
|
|
"ht_mnpo/logit": 0.06122002750635147,
|
|
"ht_mnpo/residual": 0.05372002720832825,
|
|
"ht_mnpo/residual_abs": 0.08932183682918549,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.134705066680908,
|
|
"logits/rejected": -1.9749943017959595,
|
|
"logps/chosen": -0.3081589341163635,
|
|
"logps/rejected": -0.31096822023391724,
|
|
"loss": 0.053012412041425705,
|
|
"loss/core": 0.053012412041425705,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.6446977853775024,
|
|
"rewards/margins": 0.6122003197669983,
|
|
"rewards/rejected": 1.0324976444244385,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.420474228035597,
|
|
"grad_norm": 2.875,
|
|
"ht_mnpo/logit": 0.09818646311759949,
|
|
"ht_mnpo/residual": 0.09068647027015686,
|
|
"ht_mnpo/residual_abs": 0.12812697887420654,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -2.1747286319732666,
|
|
"logits/rejected": -1.9857454299926758,
|
|
"logps/chosen": -0.2803365886211395,
|
|
"logps/rejected": -0.2985735535621643,
|
|
"loss": 0.1355080008506775,
|
|
"loss/core": 0.1355080008506775,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.7439346313476562,
|
|
"rewards/margins": 0.9818647503852844,
|
|
"rewards/rejected": 1.7620700597763062,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.4252523442632742,
|
|
"grad_norm": 28.875,
|
|
"ht_mnpo/logit": 0.07558500021696091,
|
|
"ht_mnpo/residual": 0.06808499991893768,
|
|
"ht_mnpo/residual_abs": 0.1335534304380417,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -2.0056984424591064,
|
|
"logits/rejected": -1.829700231552124,
|
|
"logps/chosen": -0.2837182879447937,
|
|
"logps/rejected": -0.301053524017334,
|
|
"loss": 0.12916430830955505,
|
|
"loss/core": 0.12916430830955505,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.1971168518066406,
|
|
"rewards/margins": 0.7558499574661255,
|
|
"rewards/rejected": 1.4412667751312256,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.43003046049095145,
|
|
"grad_norm": 26.125,
|
|
"ht_mnpo/logit": 0.0845903530716896,
|
|
"ht_mnpo/residual": 0.07709035277366638,
|
|
"ht_mnpo/residual_abs": 0.26011234521865845,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -2.2318408489227295,
|
|
"logits/rejected": -1.978154182434082,
|
|
"logps/chosen": -0.26634055376052856,
|
|
"logps/rejected": -0.28922945261001587,
|
|
"loss": 0.7810267210006714,
|
|
"loss/core": 0.7810267210006714,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.7954800128936768,
|
|
"rewards/margins": 0.8459035754203796,
|
|
"rewards/rejected": 1.9495761394500732,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.43480857671862866,
|
|
"grad_norm": 2.734375,
|
|
"ht_mnpo/logit": 0.025683075189590454,
|
|
"ht_mnpo/residual": 0.01818307302892208,
|
|
"ht_mnpo/residual_abs": 0.08304707705974579,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -2.370572328567505,
|
|
"logits/rejected": -2.1037092208862305,
|
|
"logps/chosen": -0.26799899339675903,
|
|
"logps/rejected": -0.27521270513534546,
|
|
"loss": 0.059945620596408844,
|
|
"loss/core": 0.059945620596408844,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.4931560754776,
|
|
"rewards/margins": 0.25683075189590454,
|
|
"rewards/rejected": 1.2363255023956299,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395866929463059,
|
|
"grad_norm": 740.0,
|
|
"ht_mnpo/logit": 0.1160498708486557,
|
|
"ht_mnpo/residual": 0.10854987800121307,
|
|
"ht_mnpo/residual_abs": 0.13926836848258972,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -2.282945156097412,
|
|
"logits/rejected": -2.147752285003662,
|
|
"logps/chosen": -0.29756516218185425,
|
|
"logps/rejected": -0.28938478231430054,
|
|
"loss": 0.4518727660179138,
|
|
"loss/core": 0.4518727660179138,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.2263054847717285,
|
|
"rewards/margins": 1.1604987382888794,
|
|
"rewards/rejected": 1.0658067464828491,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44436480917398313,
|
|
"grad_norm": 5.875,
|
|
"ht_mnpo/logit": 0.09502001106739044,
|
|
"ht_mnpo/residual": 0.08752001821994781,
|
|
"ht_mnpo/residual_abs": 0.1076284870505333,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -2.368155002593994,
|
|
"logits/rejected": -2.1031692028045654,
|
|
"logps/chosen": -0.28982120752334595,
|
|
"logps/rejected": -0.3006080985069275,
|
|
"loss": 0.20630280673503876,
|
|
"loss/core": 0.20630280673503876,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.638486623764038,
|
|
"rewards/margins": 0.9502002596855164,
|
|
"rewards/rejected": 0.6882864236831665,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4491429254016604,
|
|
"grad_norm": 20.75,
|
|
"ht_mnpo/logit": 0.11079178750514984,
|
|
"ht_mnpo/residual": 0.10329178720712662,
|
|
"ht_mnpo/residual_abs": 0.11315907537937164,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -2.191417694091797,
|
|
"logits/rejected": -2.082339286804199,
|
|
"logps/chosen": -0.25483283400535583,
|
|
"logps/rejected": -0.2595880925655365,
|
|
"loss": 0.08468286693096161,
|
|
"loss/core": 0.08468286693096161,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.0527219772338867,
|
|
"rewards/margins": 1.1079180240631104,
|
|
"rewards/rejected": 0.9448040723800659,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.4539210416293376,
|
|
"grad_norm": 1.5546875,
|
|
"ht_mnpo/logit": 0.041524358093738556,
|
|
"ht_mnpo/residual": 0.03402435779571533,
|
|
"ht_mnpo/residual_abs": 0.1066092848777771,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.2947616577148438,
|
|
"logits/rejected": -2.062729597091675,
|
|
"logps/chosen": -0.2770024240016937,
|
|
"logps/rejected": -0.28233516216278076,
|
|
"loss": 0.07373562455177307,
|
|
"loss/core": 0.07373562455177307,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.652806043624878,
|
|
"rewards/margins": 0.41524356603622437,
|
|
"rewards/rejected": 1.237562656402588,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45869915785701487,
|
|
"grad_norm": 109.0,
|
|
"ht_mnpo/logit": 0.12709861993789673,
|
|
"ht_mnpo/residual": 0.11959858983755112,
|
|
"ht_mnpo/residual_abs": 0.2165347784757614,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -2.141500234603882,
|
|
"logits/rejected": -1.88735032081604,
|
|
"logps/chosen": -0.32017025351524353,
|
|
"logps/rejected": -0.33095675706863403,
|
|
"loss": 0.7335715889930725,
|
|
"loss/core": 0.7335715889930725,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 3.164018154144287,
|
|
"rewards/margins": 1.2709861993789673,
|
|
"rewards/rejected": 1.8930323123931885,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.46347727408469214,
|
|
"grad_norm": 2.84375,
|
|
"ht_mnpo/logit": 0.03613976389169693,
|
|
"ht_mnpo/residual": 0.028639763593673706,
|
|
"ht_mnpo/residual_abs": 0.10108862817287445,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -2.169957399368286,
|
|
"logits/rejected": -1.9736477136611938,
|
|
"logps/chosen": -0.2766631245613098,
|
|
"logps/rejected": -0.2809513807296753,
|
|
"loss": 0.07384424656629562,
|
|
"loss/core": 0.07384424656629562,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.7005043029785156,
|
|
"rewards/margins": 0.3613975942134857,
|
|
"rewards/rejected": 1.339106559753418,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.46825539031236935,
|
|
"grad_norm": 8.875,
|
|
"ht_mnpo/logit": 0.053298842161893845,
|
|
"ht_mnpo/residual": 0.045798830687999725,
|
|
"ht_mnpo/residual_abs": 0.2681800127029419,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -2.0894553661346436,
|
|
"logits/rejected": -1.8972949981689453,
|
|
"logps/chosen": -0.3042840361595154,
|
|
"logps/rejected": -0.33394938707351685,
|
|
"loss": 0.8056422472000122,
|
|
"loss/core": 0.8056422472000122,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.712467908859253,
|
|
"rewards/margins": 0.5329883694648743,
|
|
"rewards/rejected": 2.1794795989990234,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.4730335065400466,
|
|
"grad_norm": 20.375,
|
|
"ht_mnpo/logit": 0.06616373360157013,
|
|
"ht_mnpo/residual": 0.058663733303546906,
|
|
"ht_mnpo/residual_abs": 0.07755361497402191,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -2.2147159576416016,
|
|
"logits/rejected": -2.0647025108337402,
|
|
"logps/chosen": -0.26900672912597656,
|
|
"logps/rejected": -0.27678027749061584,
|
|
"loss": 0.034580301493406296,
|
|
"loss/core": 0.034580301493406296,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6060407161712646,
|
|
"rewards/margins": 0.6616373062133789,
|
|
"rewards/rejected": 0.9444035291671753,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.4778116227677238,
|
|
"grad_norm": 1.171875,
|
|
"ht_mnpo/logit": 0.15142087638378143,
|
|
"ht_mnpo/residual": 0.14392085373401642,
|
|
"ht_mnpo/residual_abs": 0.2897629141807556,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -1.9548299312591553,
|
|
"logits/rejected": -1.732587456703186,
|
|
"logps/chosen": -0.2988852262496948,
|
|
"logps/rejected": -0.3125118017196655,
|
|
"loss": 0.9474697113037109,
|
|
"loss/core": 0.9474697113037109,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 3.979719638824463,
|
|
"rewards/margins": 1.5142085552215576,
|
|
"rewards/rejected": 2.465510845184326,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825897389954011,
|
|
"grad_norm": 3.03125,
|
|
"ht_mnpo/logit": 0.06197315454483032,
|
|
"ht_mnpo/residual": 0.0544731430709362,
|
|
"ht_mnpo/residual_abs": 0.146002858877182,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -2.1246144771575928,
|
|
"logits/rejected": -1.9374281167984009,
|
|
"logps/chosen": -0.24218598008155823,
|
|
"logps/rejected": -0.2681754529476166,
|
|
"loss": 0.19025397300720215,
|
|
"loss/core": 0.19025397300720215,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2516086101531982,
|
|
"rewards/margins": 0.6197315454483032,
|
|
"rewards/rejected": 1.6318775415420532,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4873678552230783,
|
|
"grad_norm": 63.5,
|
|
"ht_mnpo/logit": 0.07161416858434677,
|
|
"ht_mnpo/residual": 0.06411416828632355,
|
|
"ht_mnpo/residual_abs": 0.11509843915700912,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -2.34808611869812,
|
|
"logits/rejected": -2.110934019088745,
|
|
"logps/chosen": -0.2828849256038666,
|
|
"logps/rejected": -0.29181432723999023,
|
|
"loss": 0.08646970242261887,
|
|
"loss/core": 0.08646970242261887,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.3318309783935547,
|
|
"rewards/margins": 0.7161417603492737,
|
|
"rewards/rejected": 1.6156890392303467,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49214597145075556,
|
|
"grad_norm": 59.0,
|
|
"ht_mnpo/logit": 0.07742169499397278,
|
|
"ht_mnpo/residual": 0.06992168724536896,
|
|
"ht_mnpo/residual_abs": 0.07737835496664047,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -2.3272345066070557,
|
|
"logits/rejected": -2.0595836639404297,
|
|
"logps/chosen": -0.2829161584377289,
|
|
"logps/rejected": -0.2974115014076233,
|
|
"loss": 0.04383247345685959,
|
|
"loss/core": 0.04383247345685959,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7030824422836304,
|
|
"rewards/margins": 0.7742169499397278,
|
|
"rewards/rejected": 0.9288653135299683,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.49692408767843277,
|
|
"grad_norm": 0.2099609375,
|
|
"ht_mnpo/logit": 0.0885908305644989,
|
|
"ht_mnpo/residual": 0.08109083026647568,
|
|
"ht_mnpo/residual_abs": 0.12812694907188416,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -2.1603314876556396,
|
|
"logits/rejected": -2.0189948081970215,
|
|
"logps/chosen": -0.28825613856315613,
|
|
"logps/rejected": -0.30118125677108765,
|
|
"loss": 0.17555531859397888,
|
|
"loss/core": 0.17555531859397888,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.3579986095428467,
|
|
"rewards/margins": 0.885908305644989,
|
|
"rewards/rejected": 1.472090244293213,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.50170220390611,
|
|
"grad_norm": 12.3125,
|
|
"ht_mnpo/logit": 0.047115158289670944,
|
|
"ht_mnpo/residual": 0.039615172892808914,
|
|
"ht_mnpo/residual_abs": 0.1817067414522171,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -2.351896047592163,
|
|
"logits/rejected": -2.1078438758850098,
|
|
"logps/chosen": -0.2944217622280121,
|
|
"logps/rejected": -0.28825289011001587,
|
|
"loss": 0.6127477884292603,
|
|
"loss/core": 0.6127477884292603,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.208118438720703,
|
|
"rewards/margins": 0.47115176916122437,
|
|
"rewards/rejected": 1.7369667291641235,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5064803201337873,
|
|
"grad_norm": 1000.0,
|
|
"ht_mnpo/logit": 0.06004665046930313,
|
|
"ht_mnpo/residual": 0.05254665017127991,
|
|
"ht_mnpo/residual_abs": 0.23322728276252747,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -2.1865041255950928,
|
|
"logits/rejected": -1.8540855646133423,
|
|
"logps/chosen": -0.2854662537574768,
|
|
"logps/rejected": -0.32376745343208313,
|
|
"loss": 0.5732264518737793,
|
|
"loss/core": 0.5732264518737793,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 3.1418910026550293,
|
|
"rewards/margins": 0.6004664301872253,
|
|
"rewards/rejected": 2.5414247512817383,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5112584363614645,
|
|
"grad_norm": 27.0,
|
|
"ht_mnpo/logit": 0.13712215423583984,
|
|
"ht_mnpo/residual": 0.1296221762895584,
|
|
"ht_mnpo/residual_abs": 0.17209194600582123,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -2.173382043838501,
|
|
"logits/rejected": -1.8262176513671875,
|
|
"logps/chosen": -0.2807655930519104,
|
|
"logps/rejected": -0.2747199237346649,
|
|
"loss": 0.29922977089881897,
|
|
"loss/core": 0.29922977089881897,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.9412617683410645,
|
|
"rewards/margins": 1.3712217807769775,
|
|
"rewards/rejected": 1.570040225982666,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5160365525891417,
|
|
"grad_norm": 0.578125,
|
|
"ht_mnpo/logit": 0.061078108847141266,
|
|
"ht_mnpo/residual": 0.05357811599969864,
|
|
"ht_mnpo/residual_abs": 0.14000064134597778,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -2.1986470222473145,
|
|
"logits/rejected": -2.0166780948638916,
|
|
"logps/chosen": -0.2934110760688782,
|
|
"logps/rejected": -0.31152665615081787,
|
|
"loss": 0.2973403334617615,
|
|
"loss/core": 0.2973403334617615,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.6136223077774048,
|
|
"rewards/margins": 0.6107810735702515,
|
|
"rewards/rejected": 1.0028412342071533,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.5208146688168189,
|
|
"grad_norm": 30.875,
|
|
"ht_mnpo/logit": 0.09199979156255722,
|
|
"ht_mnpo/residual": 0.084499791264534,
|
|
"ht_mnpo/residual_abs": 0.1785065233707428,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.1413793563842773,
|
|
"logits/rejected": -1.9582176208496094,
|
|
"logps/chosen": -0.26738229393959045,
|
|
"logps/rejected": -0.2807932198047638,
|
|
"loss": 0.19445274770259857,
|
|
"loss/core": 0.19445274770259857,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.202378511428833,
|
|
"rewards/margins": 0.919998049736023,
|
|
"rewards/rejected": 2.2823805809020996,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5255927850444962,
|
|
"grad_norm": 624.0,
|
|
"ht_mnpo/logit": 0.048402734100818634,
|
|
"ht_mnpo/residual": 0.04090272635221481,
|
|
"ht_mnpo/residual_abs": 0.20708465576171875,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -2.335815191268921,
|
|
"logits/rejected": -2.078155994415283,
|
|
"logps/chosen": -0.33746832609176636,
|
|
"logps/rejected": -0.307259738445282,
|
|
"loss": 0.6829358339309692,
|
|
"loss/core": 0.6829358339309692,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.586956024169922,
|
|
"rewards/margins": 0.48402705788612366,
|
|
"rewards/rejected": 2.102928876876831,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5303709012721735,
|
|
"grad_norm": 504.0,
|
|
"ht_mnpo/logit": 0.17546024918556213,
|
|
"ht_mnpo/residual": 0.1679602712392807,
|
|
"ht_mnpo/residual_abs": 0.21153728663921356,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -2.0990540981292725,
|
|
"logits/rejected": -1.8069511651992798,
|
|
"logps/chosen": -0.29029348492622375,
|
|
"logps/rejected": -0.284585177898407,
|
|
"loss": 0.5383775234222412,
|
|
"loss/core": 0.5383775234222412,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 2.9643406867980957,
|
|
"rewards/margins": 1.7546026706695557,
|
|
"rewards/rejected": 1.2097381353378296,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5351490174998507,
|
|
"grad_norm": 12.3125,
|
|
"ht_mnpo/logit": 0.1519746482372284,
|
|
"ht_mnpo/residual": 0.14447465538978577,
|
|
"ht_mnpo/residual_abs": 0.17106272280216217,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -2.2186014652252197,
|
|
"logits/rejected": -2.0292179584503174,
|
|
"logps/chosen": -0.2765801250934601,
|
|
"logps/rejected": -0.26710256934165955,
|
|
"loss": 0.511273205280304,
|
|
"loss/core": 0.511273205280304,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.9151830673217773,
|
|
"rewards/margins": 1.5197465419769287,
|
|
"rewards/rejected": 1.3954365253448486,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5399271337275279,
|
|
"grad_norm": 2.375,
|
|
"ht_mnpo/logit": -0.013432973995804787,
|
|
"ht_mnpo/residual": -0.020932968705892563,
|
|
"ht_mnpo/residual_abs": 0.15191642940044403,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -2.257132053375244,
|
|
"logits/rejected": -2.013878345489502,
|
|
"logps/chosen": -0.2700672447681427,
|
|
"logps/rejected": -0.2897999882698059,
|
|
"loss": 0.5020833015441895,
|
|
"loss/core": 0.5020833015441895,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.8380361795425415,
|
|
"rewards/margins": -0.1343296468257904,
|
|
"rewards/rejected": 1.9723659753799438,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5447052499552052,
|
|
"grad_norm": 32.25,
|
|
"ht_mnpo/logit": 0.10750623792409897,
|
|
"ht_mnpo/residual": 0.10000623762607574,
|
|
"ht_mnpo/residual_abs": 0.11906830966472626,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -2.1714699268341064,
|
|
"logits/rejected": -1.9297101497650146,
|
|
"logps/chosen": -0.29392221570014954,
|
|
"logps/rejected": -0.2952617406845093,
|
|
"loss": 0.24284780025482178,
|
|
"loss/core": 0.24284780025482178,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.1015772819519043,
|
|
"rewards/margins": 1.0750623941421509,
|
|
"rewards/rejected": 1.0265148878097534,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5494833661828824,
|
|
"grad_norm": 350.0,
|
|
"ht_mnpo/logit": 0.1779872626066208,
|
|
"ht_mnpo/residual": 0.17048725485801697,
|
|
"ht_mnpo/residual_abs": 0.19057273864746094,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.037717342376709,
|
|
"logits/rejected": -1.7374416589736938,
|
|
"logps/chosen": -0.2867358326911926,
|
|
"logps/rejected": -0.28076958656311035,
|
|
"loss": 0.4524991512298584,
|
|
"loss/core": 0.4524991512298584,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 2.9693145751953125,
|
|
"rewards/margins": 1.7798726558685303,
|
|
"rewards/rejected": 1.1894419193267822,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5542614824105596,
|
|
"grad_norm": 52.5,
|
|
"ht_mnpo/logit": 0.07053961604833603,
|
|
"ht_mnpo/residual": 0.0630396157503128,
|
|
"ht_mnpo/residual_abs": 0.16084904968738556,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -2.151390790939331,
|
|
"logits/rejected": -1.9337005615234375,
|
|
"logps/chosen": -0.28611868619918823,
|
|
"logps/rejected": -0.2922998368740082,
|
|
"loss": 0.2780520021915436,
|
|
"loss/core": 0.2780520021915436,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.186372756958008,
|
|
"rewards/margins": 0.7053961753845215,
|
|
"rewards/rejected": 1.4809764623641968,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5590395986382368,
|
|
"grad_norm": 159.0,
|
|
"ht_mnpo/logit": 0.073150135576725,
|
|
"ht_mnpo/residual": 0.06565012782812119,
|
|
"ht_mnpo/residual_abs": 0.17466747760772705,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -2.1101319789886475,
|
|
"logits/rejected": -1.8840051889419556,
|
|
"logps/chosen": -0.2872307598590851,
|
|
"logps/rejected": -0.3082466125488281,
|
|
"loss": 0.2757607400417328,
|
|
"loss/core": 0.2757607400417328,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.8962421417236328,
|
|
"rewards/margins": 0.7315012812614441,
|
|
"rewards/rejected": 1.1647409200668335,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5638177148659141,
|
|
"grad_norm": 1.609375,
|
|
"ht_mnpo/logit": 0.057926855981349945,
|
|
"ht_mnpo/residual": 0.05042685940861702,
|
|
"ht_mnpo/residual_abs": 0.12516820430755615,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -2.291469097137451,
|
|
"logits/rejected": -2.0484461784362793,
|
|
"logps/chosen": -0.3092489242553711,
|
|
"logps/rejected": -0.3214951455593109,
|
|
"loss": 0.18674132227897644,
|
|
"loss/core": 0.18674132227897644,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.030933380126953,
|
|
"rewards/margins": 0.5792685747146606,
|
|
"rewards/rejected": 1.451664686203003,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5685958310935914,
|
|
"grad_norm": 3.640625,
|
|
"ht_mnpo/logit": 0.034106455743312836,
|
|
"ht_mnpo/residual": 0.02660645917057991,
|
|
"ht_mnpo/residual_abs": 0.06431782990694046,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -2.3490476608276367,
|
|
"logits/rejected": -2.112255096435547,
|
|
"logps/chosen": -0.2819194793701172,
|
|
"logps/rejected": -0.2832038104534149,
|
|
"loss": 0.019458934664726257,
|
|
"loss/core": 0.019458934664726257,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5581998825073242,
|
|
"rewards/margins": 0.34106460213661194,
|
|
"rewards/rejected": 1.2171355485916138,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5733739473212686,
|
|
"grad_norm": 1.4296875,
|
|
"ht_mnpo/logit": 0.1805572807788849,
|
|
"ht_mnpo/residual": 0.17305730283260345,
|
|
"ht_mnpo/residual_abs": 0.18950334191322327,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -2.1632628440856934,
|
|
"logits/rejected": -1.9910504817962646,
|
|
"logps/chosen": -0.2667522430419922,
|
|
"logps/rejected": -0.29135292768478394,
|
|
"loss": 0.4613543152809143,
|
|
"loss/core": 0.4613543152809143,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.934541702270508,
|
|
"rewards/margins": 1.8055731058120728,
|
|
"rewards/rejected": 1.1289689540863037,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5781520635489459,
|
|
"grad_norm": 1.6015625,
|
|
"ht_mnpo/logit": 0.048975661396980286,
|
|
"ht_mnpo/residual": 0.04147566109895706,
|
|
"ht_mnpo/residual_abs": 0.1043766587972641,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -2.1373214721679688,
|
|
"logits/rejected": -1.9456546306610107,
|
|
"logps/chosen": -0.2933582365512848,
|
|
"logps/rejected": -0.29404282569885254,
|
|
"loss": 0.10271594673395157,
|
|
"loss/core": 0.10271594673395157,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.032723903656006,
|
|
"rewards/margins": 0.4897565245628357,
|
|
"rewards/rejected": 1.5429675579071045,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5829301797766231,
|
|
"grad_norm": 8.3125,
|
|
"ht_mnpo/logit": 0.06926552206277847,
|
|
"ht_mnpo/residual": 0.06176552176475525,
|
|
"ht_mnpo/residual_abs": 0.12705007195472717,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -2.3621904850006104,
|
|
"logits/rejected": -2.105658531188965,
|
|
"logps/chosen": -0.29055026173591614,
|
|
"logps/rejected": -0.2932811677455902,
|
|
"loss": 0.21939155459403992,
|
|
"loss/core": 0.21939155459403992,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.4611763954162598,
|
|
"rewards/margins": 0.6926552057266235,
|
|
"rewards/rejected": 1.7685215473175049,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.5877082960043003,
|
|
"grad_norm": 6.90625,
|
|
"ht_mnpo/logit": 0.05083639174699783,
|
|
"ht_mnpo/residual": 0.04333639144897461,
|
|
"ht_mnpo/residual_abs": 0.09056956321001053,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -2.2035889625549316,
|
|
"logits/rejected": -2.006361484527588,
|
|
"logps/chosen": -0.27977409958839417,
|
|
"logps/rejected": -0.29095375537872314,
|
|
"loss": 0.05176713317632675,
|
|
"loss/core": 0.05176713317632675,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.685443639755249,
|
|
"rewards/margins": 0.5083640217781067,
|
|
"rewards/rejected": 1.1770797967910767,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5924864122319775,
|
|
"grad_norm": 2.171875,
|
|
"ht_mnpo/logit": 0.1474437713623047,
|
|
"ht_mnpo/residual": 0.13994379341602325,
|
|
"ht_mnpo/residual_abs": 0.16675330698490143,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -2.2853574752807617,
|
|
"logits/rejected": -1.9415066242218018,
|
|
"logps/chosen": -0.291348934173584,
|
|
"logps/rejected": -0.30028507113456726,
|
|
"loss": 0.7507929801940918,
|
|
"loss/core": 0.7507929801940918,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.765855312347412,
|
|
"rewards/margins": 1.4744378328323364,
|
|
"rewards/rejected": 1.2914174795150757,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5972645284596548,
|
|
"grad_norm": 206.0,
|
|
"ht_mnpo/logit": 0.14434558153152466,
|
|
"ht_mnpo/residual": 0.13684558868408203,
|
|
"ht_mnpo/residual_abs": 0.2338484227657318,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -2.187645673751831,
|
|
"logits/rejected": -1.9885934591293335,
|
|
"logps/chosen": -0.27156367897987366,
|
|
"logps/rejected": -0.28177887201309204,
|
|
"loss": 0.889374852180481,
|
|
"loss/core": 0.889374852180481,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.960251569747925,
|
|
"rewards/margins": 1.4434558153152466,
|
|
"rewards/rejected": 1.5167958736419678,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.602042644687332,
|
|
"grad_norm": 242.0,
|
|
"ht_mnpo/logit": 0.11875858157873154,
|
|
"ht_mnpo/residual": 0.11125858873128891,
|
|
"ht_mnpo/residual_abs": 0.1652863770723343,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -2.1578569412231445,
|
|
"logits/rejected": -1.9522138833999634,
|
|
"logps/chosen": -0.3227333724498749,
|
|
"logps/rejected": -0.32840460538864136,
|
|
"loss": 0.2998679280281067,
|
|
"loss/core": 0.2998679280281067,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.60225510597229,
|
|
"rewards/margins": 1.187585711479187,
|
|
"rewards/rejected": 1.414669156074524,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6068207609150092,
|
|
"grad_norm": 4.34375,
|
|
"ht_mnpo/logit": 0.09825298935174942,
|
|
"ht_mnpo/residual": 0.0907529965043068,
|
|
"ht_mnpo/residual_abs": 0.15308210253715515,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -2.2780983448028564,
|
|
"logits/rejected": -2.0277860164642334,
|
|
"logps/chosen": -0.31773704290390015,
|
|
"logps/rejected": -0.3203713297843933,
|
|
"loss": 0.22551850974559784,
|
|
"loss/core": 0.22551850974559784,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.1558048725128174,
|
|
"rewards/margins": 0.9825299382209778,
|
|
"rewards/rejected": 1.1732749938964844,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6115988771426865,
|
|
"grad_norm": 0.90625,
|
|
"ht_mnpo/logit": 0.030277207493782043,
|
|
"ht_mnpo/residual": 0.02277720905840397,
|
|
"ht_mnpo/residual_abs": 0.129721999168396,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -2.434414863586426,
|
|
"logits/rejected": -2.18612003326416,
|
|
"logps/chosen": -0.2906445264816284,
|
|
"logps/rejected": -0.2876962721347809,
|
|
"loss": 0.2192559689283371,
|
|
"loss/core": 0.2192559689283371,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5939425230026245,
|
|
"rewards/margins": 0.30277204513549805,
|
|
"rewards/rejected": 1.2911707162857056,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6163769933703638,
|
|
"grad_norm": 106.0,
|
|
"ht_mnpo/logit": 0.03566845878958702,
|
|
"ht_mnpo/residual": 0.028168460354208946,
|
|
"ht_mnpo/residual_abs": 0.08532391488552094,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -2.1786632537841797,
|
|
"logits/rejected": -1.9741607904434204,
|
|
"logps/chosen": -0.27927350997924805,
|
|
"logps/rejected": -0.2722075581550598,
|
|
"loss": 0.03750378638505936,
|
|
"loss/core": 0.03750378638505936,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.107971668243408,
|
|
"rewards/margins": 0.3566845953464508,
|
|
"rewards/rejected": 1.7512871026992798,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.621155109598041,
|
|
"grad_norm": 3.203125,
|
|
"ht_mnpo/logit": 0.05108143016695976,
|
|
"ht_mnpo/residual": 0.04358143359422684,
|
|
"ht_mnpo/residual_abs": 0.056876249611377716,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -2.1610920429229736,
|
|
"logits/rejected": -1.9564762115478516,
|
|
"logps/chosen": -0.28243714570999146,
|
|
"logps/rejected": -0.28669795393943787,
|
|
"loss": 0.015014773234724998,
|
|
"loss/core": 0.015014773234724998,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.650695562362671,
|
|
"rewards/margins": 0.510814368724823,
|
|
"rewards/rejected": 1.1398811340332031,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6259332258257182,
|
|
"grad_norm": 19.625,
|
|
"ht_mnpo/logit": 0.15237808227539062,
|
|
"ht_mnpo/residual": 0.1448780596256256,
|
|
"ht_mnpo/residual_abs": 0.1591993272304535,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -2.39475679397583,
|
|
"logits/rejected": -2.1787381172180176,
|
|
"logps/chosen": -0.28849363327026367,
|
|
"logps/rejected": -0.29972606897354126,
|
|
"loss": 0.4378318190574646,
|
|
"loss/core": 0.4378318190574646,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.363157272338867,
|
|
"rewards/margins": 1.5237809419631958,
|
|
"rewards/rejected": 0.8393763303756714,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6307113420533954,
|
|
"grad_norm": 256.0,
|
|
"ht_mnpo/logit": 0.019719239324331284,
|
|
"ht_mnpo/residual": 0.012219244614243507,
|
|
"ht_mnpo/residual_abs": 0.1551227569580078,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -2.1171586513519287,
|
|
"logits/rejected": -1.8910051584243774,
|
|
"logps/chosen": -0.2649215757846832,
|
|
"logps/rejected": -0.2766799330711365,
|
|
"loss": 0.2119613140821457,
|
|
"loss/core": 0.2119613140821457,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.211346387863159,
|
|
"rewards/margins": 0.19719253480434418,
|
|
"rewards/rejected": 2.0141539573669434,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6354894582810727,
|
|
"grad_norm": 74.5,
|
|
"ht_mnpo/logit": 0.10560693591833115,
|
|
"ht_mnpo/residual": 0.09810693562030792,
|
|
"ht_mnpo/residual_abs": 0.13916924595832825,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -2.324584484100342,
|
|
"logits/rejected": -2.0863864421844482,
|
|
"logps/chosen": -0.25897565484046936,
|
|
"logps/rejected": -0.27116915583610535,
|
|
"loss": 0.12608017027378082,
|
|
"loss/core": 0.12608017027378082,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.370238780975342,
|
|
"rewards/margins": 1.056069254875183,
|
|
"rewards/rejected": 1.3141696453094482,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.6402675745087499,
|
|
"grad_norm": 213.0,
|
|
"ht_mnpo/logit": 0.055773068219423294,
|
|
"ht_mnpo/residual": 0.04827307164669037,
|
|
"ht_mnpo/residual_abs": 0.09928829967975616,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -2.439232349395752,
|
|
"logits/rejected": -2.102266788482666,
|
|
"logps/chosen": -0.2713320255279541,
|
|
"logps/rejected": -0.3047031760215759,
|
|
"loss": 0.08617246150970459,
|
|
"loss/core": 0.08617246150970459,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.5870898962020874,
|
|
"rewards/margins": 0.5577307343482971,
|
|
"rewards/rejected": 1.0293591022491455,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6450456907364271,
|
|
"grad_norm": 13.1875,
|
|
"ht_mnpo/logit": -0.06488653272390366,
|
|
"ht_mnpo/residual": -0.07238653302192688,
|
|
"ht_mnpo/residual_abs": 0.12053285539150238,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.3484644889831543,
|
|
"logits/rejected": -2.097277879714966,
|
|
"logps/chosen": -0.2894912660121918,
|
|
"logps/rejected": -0.29819828271865845,
|
|
"loss": 0.35253381729125977,
|
|
"loss/core": 0.35253381729125977,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 0.9312378764152527,
|
|
"rewards/margins": -0.6488652229309082,
|
|
"rewards/rejected": 1.5801031589508057,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6498238069641044,
|
|
"grad_norm": 3.390625,
|
|
"ht_mnpo/logit": 0.08097716420888901,
|
|
"ht_mnpo/residual": 0.07347716391086578,
|
|
"ht_mnpo/residual_abs": 0.09472920745611191,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -2.3550667762756348,
|
|
"logits/rejected": -2.101799964904785,
|
|
"logps/chosen": -0.27208298444747925,
|
|
"logps/rejected": -0.28226494789123535,
|
|
"loss": 0.09052277356386185,
|
|
"loss/core": 0.09052277356386185,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.032668113708496,
|
|
"rewards/margins": 0.809771716594696,
|
|
"rewards/rejected": 1.2228964567184448,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.6546019231917817,
|
|
"grad_norm": 5.09375,
|
|
"ht_mnpo/logit": -0.013841989450156689,
|
|
"ht_mnpo/residual": -0.02134198695421219,
|
|
"ht_mnpo/residual_abs": 0.12373582273721695,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -2.093864679336548,
|
|
"logits/rejected": -1.9131803512573242,
|
|
"logps/chosen": -0.27449527382850647,
|
|
"logps/rejected": -0.2798653542995453,
|
|
"loss": 0.2753180265426636,
|
|
"loss/core": 0.2753180265426636,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.514013648033142,
|
|
"rewards/margins": -0.13841985166072845,
|
|
"rewards/rejected": 1.6524333953857422,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6593800394194589,
|
|
"grad_norm": 81.5,
|
|
"ht_mnpo/logit": 0.10633112490177155,
|
|
"ht_mnpo/residual": 0.09883112460374832,
|
|
"ht_mnpo/residual_abs": 0.16929301619529724,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -2.175103187561035,
|
|
"logits/rejected": -2.002682685852051,
|
|
"logps/chosen": -0.28046196699142456,
|
|
"logps/rejected": -0.302472323179245,
|
|
"loss": 0.4346230924129486,
|
|
"loss/core": 0.4346230924129486,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.207476854324341,
|
|
"rewards/margins": 1.063311219215393,
|
|
"rewards/rejected": 2.144165515899658,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6641581556471361,
|
|
"grad_norm": 3.96875,
|
|
"ht_mnpo/logit": 0.037042342126369476,
|
|
"ht_mnpo/residual": 0.02954234555363655,
|
|
"ht_mnpo/residual_abs": 0.07325103133916855,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -2.2522799968719482,
|
|
"logits/rejected": -2.0153305530548096,
|
|
"logps/chosen": -0.2950429916381836,
|
|
"logps/rejected": -0.29979586601257324,
|
|
"loss": 0.030671823769807816,
|
|
"loss/core": 0.030671823769807816,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.3066444396972656,
|
|
"rewards/margins": 0.37042343616485596,
|
|
"rewards/rejected": 0.9362211227416992,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6689362718748133,
|
|
"grad_norm": 176.0,
|
|
"ht_mnpo/logit": -0.02812144160270691,
|
|
"ht_mnpo/residual": -0.035621438175439835,
|
|
"ht_mnpo/residual_abs": 0.12590409815311432,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -2.437011241912842,
|
|
"logits/rejected": -2.2066168785095215,
|
|
"logps/chosen": -0.2685781419277191,
|
|
"logps/rejected": -0.2836915850639343,
|
|
"loss": 0.22989264130592346,
|
|
"loss/core": 0.22989264130592346,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5847865343093872,
|
|
"rewards/margins": -0.28121432662010193,
|
|
"rewards/rejected": 1.8660008907318115,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6737143881024906,
|
|
"grad_norm": 8.0625,
|
|
"ht_mnpo/logit": 0.09928493201732635,
|
|
"ht_mnpo/residual": 0.09178493171930313,
|
|
"ht_mnpo/residual_abs": 0.13132280111312866,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -2.0853044986724854,
|
|
"logits/rejected": -1.8013473749160767,
|
|
"logps/chosen": -0.2867644727230072,
|
|
"logps/rejected": -0.31725770235061646,
|
|
"loss": 0.159874826669693,
|
|
"loss/core": 0.159874826669693,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.3235867023468018,
|
|
"rewards/margins": 0.9928493499755859,
|
|
"rewards/rejected": 1.3307373523712158,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6784925043301678,
|
|
"grad_norm": 1.71875,
|
|
"ht_mnpo/logit": 0.19838109612464905,
|
|
"ht_mnpo/residual": 0.19088108837604523,
|
|
"ht_mnpo/residual_abs": 0.19749945402145386,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -2.0413479804992676,
|
|
"logits/rejected": -1.9397804737091064,
|
|
"logps/chosen": -0.2884281873703003,
|
|
"logps/rejected": -0.29842838644981384,
|
|
"loss": 0.7291596531867981,
|
|
"loss/core": 0.7291596531867981,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.16792631149292,
|
|
"rewards/margins": 1.9838106632232666,
|
|
"rewards/rejected": 1.1841155290603638,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.683270620557845,
|
|
"grad_norm": 11.75,
|
|
"ht_mnpo/logit": 0.0204590056091547,
|
|
"ht_mnpo/residual": 0.012959008105099201,
|
|
"ht_mnpo/residual_abs": 0.06075044348835945,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -2.1333656311035156,
|
|
"logits/rejected": -1.8975515365600586,
|
|
"logps/chosen": -0.2886390686035156,
|
|
"logps/rejected": -0.294972687959671,
|
|
"loss": 0.032932449132204056,
|
|
"loss/core": 0.032932449132204056,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.033470630645752,
|
|
"rewards/margins": 0.2045900523662567,
|
|
"rewards/rejected": 0.8288804888725281,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6880487367855223,
|
|
"grad_norm": 7.0625,
|
|
"ht_mnpo/logit": 0.07070360332727432,
|
|
"ht_mnpo/residual": 0.0632036104798317,
|
|
"ht_mnpo/residual_abs": 0.178388774394989,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -2.3582139015197754,
|
|
"logits/rejected": -2.154236316680908,
|
|
"logps/chosen": -0.27584514021873474,
|
|
"logps/rejected": -0.2994580864906311,
|
|
"loss": 0.5289813280105591,
|
|
"loss/core": 0.5289813280105591,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.8845659494400024,
|
|
"rewards/margins": 0.7070361375808716,
|
|
"rewards/rejected": 1.1775298118591309,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6928268530131996,
|
|
"grad_norm": 22.875,
|
|
"ht_mnpo/logit": 0.11204788833856583,
|
|
"ht_mnpo/residual": 0.1045479029417038,
|
|
"ht_mnpo/residual_abs": 0.23556935787200928,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -2.127692699432373,
|
|
"logits/rejected": -1.873258352279663,
|
|
"logps/chosen": -0.2844100296497345,
|
|
"logps/rejected": -0.34677594900131226,
|
|
"loss": 0.522869884967804,
|
|
"loss/core": 0.522869884967804,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 3.1595566272735596,
|
|
"rewards/margins": 1.1204789876937866,
|
|
"rewards/rejected": 2.0390777587890625,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6976049692408768,
|
|
"grad_norm": 34.25,
|
|
"ht_mnpo/logit": 0.0576045997440815,
|
|
"ht_mnpo/residual": 0.05010459944605827,
|
|
"ht_mnpo/residual_abs": 0.14411763846874237,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -2.3395884037017822,
|
|
"logits/rejected": -2.088940143585205,
|
|
"logps/chosen": -0.2896740436553955,
|
|
"logps/rejected": -0.2858702540397644,
|
|
"loss": 0.16655156016349792,
|
|
"loss/core": 0.16655156016349792,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.1040432453155518,
|
|
"rewards/margins": 0.5760459899902344,
|
|
"rewards/rejected": 1.5279972553253174,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.702383085468554,
|
|
"grad_norm": 3.40625,
|
|
"ht_mnpo/logit": 0.04949961230158806,
|
|
"ht_mnpo/residual": 0.04199961572885513,
|
|
"ht_mnpo/residual_abs": 0.04821108654141426,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -2.2227725982666016,
|
|
"logits/rejected": -2.019742250442505,
|
|
"logps/chosen": -0.28382936120033264,
|
|
"logps/rejected": -0.2969701290130615,
|
|
"loss": 0.010463174432516098,
|
|
"loss/core": 0.010463174432516098,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.1914409399032593,
|
|
"rewards/margins": 0.4949961304664612,
|
|
"rewards/rejected": 0.6964449286460876,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.7071612016962313,
|
|
"grad_norm": 1.71875,
|
|
"ht_mnpo/logit": 0.20136702060699463,
|
|
"ht_mnpo/residual": 0.193867027759552,
|
|
"ht_mnpo/residual_abs": 0.28303730487823486,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -2.123994827270508,
|
|
"logits/rejected": -1.8912299871444702,
|
|
"logps/chosen": -0.31486859917640686,
|
|
"logps/rejected": -0.25920960307121277,
|
|
"loss": 1.0651627779006958,
|
|
"loss/core": 1.0651627779006958,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.329946517944336,
|
|
"rewards/margins": 2.013669967651367,
|
|
"rewards/rejected": 1.3162766695022583,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7119393179239085,
|
|
"grad_norm": 57.0,
|
|
"ht_mnpo/logit": 0.07492740452289581,
|
|
"ht_mnpo/residual": 0.06742740422487259,
|
|
"ht_mnpo/residual_abs": 0.09558016806840897,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -2.3298563957214355,
|
|
"logits/rejected": -2.0255560874938965,
|
|
"logps/chosen": -0.29623550176620483,
|
|
"logps/rejected": -0.29635828733444214,
|
|
"loss": 0.0679873377084732,
|
|
"loss/core": 0.0679873377084732,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.242774248123169,
|
|
"rewards/margins": 0.7492740750312805,
|
|
"rewards/rejected": 1.493499994277954,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7167174341515857,
|
|
"grad_norm": 4.71875,
|
|
"ht_mnpo/logit": 0.02682856284081936,
|
|
"ht_mnpo/residual": 0.019328564405441284,
|
|
"ht_mnpo/residual_abs": 0.10245944559574127,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -2.2698183059692383,
|
|
"logits/rejected": -2.0956759452819824,
|
|
"logps/chosen": -0.2996281385421753,
|
|
"logps/rejected": -0.3088133931159973,
|
|
"loss": 0.11470890045166016,
|
|
"loss/core": 0.11470890045166016,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.4476314783096313,
|
|
"rewards/margins": 0.2682856619358063,
|
|
"rewards/rejected": 1.179345726966858,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.7214955503792629,
|
|
"grad_norm": 368.0,
|
|
"ht_mnpo/logit": 0.11839409917593002,
|
|
"ht_mnpo/residual": 0.1108940988779068,
|
|
"ht_mnpo/residual_abs": 0.19081872701644897,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -2.1703250408172607,
|
|
"logits/rejected": -1.846046805381775,
|
|
"logps/chosen": -0.2736147940158844,
|
|
"logps/rejected": -0.27697068452835083,
|
|
"loss": 0.3880789875984192,
|
|
"loss/core": 0.3880789875984192,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 3.0956952571868896,
|
|
"rewards/margins": 1.1839410066604614,
|
|
"rewards/rejected": 1.9117543697357178,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7262736666069403,
|
|
"grad_norm": 4.9375,
|
|
"ht_mnpo/logit": 0.1197742447257042,
|
|
"ht_mnpo/residual": 0.11227424442768097,
|
|
"ht_mnpo/residual_abs": 0.14611002802848816,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -2.176957607269287,
|
|
"logits/rejected": -1.8870060443878174,
|
|
"logps/chosen": -0.30091139674186707,
|
|
"logps/rejected": -0.3017504811286926,
|
|
"loss": 0.22610747814178467,
|
|
"loss/core": 0.22610747814178467,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.655430793762207,
|
|
"rewards/margins": 1.1977424621582031,
|
|
"rewards/rejected": 1.457688570022583,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7310517828346175,
|
|
"grad_norm": 27.75,
|
|
"ht_mnpo/logit": 0.07762764394283295,
|
|
"ht_mnpo/residual": 0.07012763619422913,
|
|
"ht_mnpo/residual_abs": 0.12880326807498932,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -1.9855482578277588,
|
|
"logits/rejected": -1.878286361694336,
|
|
"logps/chosen": -0.27580446004867554,
|
|
"logps/rejected": -0.27556952834129333,
|
|
"loss": 0.1310841143131256,
|
|
"loss/core": 0.1310841143131256,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.5765607357025146,
|
|
"rewards/margins": 0.7762764692306519,
|
|
"rewards/rejected": 1.8002843856811523,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7358298990622947,
|
|
"grad_norm": 6.0,
|
|
"ht_mnpo/logit": 0.10170654207468033,
|
|
"ht_mnpo/residual": 0.0942065417766571,
|
|
"ht_mnpo/residual_abs": 0.16815176606178284,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -2.3226284980773926,
|
|
"logits/rejected": -2.1349689960479736,
|
|
"logps/chosen": -0.29955098032951355,
|
|
"logps/rejected": -0.2934722900390625,
|
|
"loss": 0.691552460193634,
|
|
"loss/core": 0.691552460193634,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.19858980178833,
|
|
"rewards/margins": 1.0170655250549316,
|
|
"rewards/rejected": 1.1815242767333984,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7406080152899719,
|
|
"grad_norm": 7.53125,
|
|
"ht_mnpo/logit": 0.04109996557235718,
|
|
"ht_mnpo/residual": 0.033599965274333954,
|
|
"ht_mnpo/residual_abs": 0.19126072525978088,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -2.1104021072387695,
|
|
"logits/rejected": -1.9156906604766846,
|
|
"logps/chosen": -0.2990146577358246,
|
|
"logps/rejected": -0.3021544814109802,
|
|
"loss": 0.3742893636226654,
|
|
"loss/core": 0.3742893636226654,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.993298053741455,
|
|
"rewards/margins": 0.410999596118927,
|
|
"rewards/rejected": 2.5822982788085938,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7453861315176492,
|
|
"grad_norm": 17.5,
|
|
"ht_mnpo/logit": 0.12976470589637756,
|
|
"ht_mnpo/residual": 0.12226470559835434,
|
|
"ht_mnpo/residual_abs": 0.14971300959587097,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -2.1731667518615723,
|
|
"logits/rejected": -2.022315263748169,
|
|
"logps/chosen": -0.2888116240501404,
|
|
"logps/rejected": -0.30564987659454346,
|
|
"loss": 0.3799237906932831,
|
|
"loss/core": 0.3799237906932831,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4324753284454346,
|
|
"rewards/margins": 1.2976469993591309,
|
|
"rewards/rejected": 1.1348284482955933,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7501642477453264,
|
|
"grad_norm": 896.0,
|
|
"ht_mnpo/logit": 0.2294764518737793,
|
|
"ht_mnpo/residual": 0.22197644412517548,
|
|
"ht_mnpo/residual_abs": 0.27920347452163696,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.785697340965271,
|
|
"logits/rejected": -1.6078464984893799,
|
|
"logps/chosen": -0.30665987730026245,
|
|
"logps/rejected": -0.31216126680374146,
|
|
"loss": 0.8990079164505005,
|
|
"loss/core": 0.8990079164505005,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.655916690826416,
|
|
"rewards/margins": 2.294764757156372,
|
|
"rewards/rejected": 1.361151933670044,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7549423639730036,
|
|
"grad_norm": 4.96875,
|
|
"ht_mnpo/logit": 0.0998556837439537,
|
|
"ht_mnpo/residual": 0.09235568344593048,
|
|
"ht_mnpo/residual_abs": 0.1307271420955658,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -2.0636184215545654,
|
|
"logits/rejected": -1.8661634922027588,
|
|
"logps/chosen": -0.2779093384742737,
|
|
"logps/rejected": -0.27925077080726624,
|
|
"loss": 0.07385687530040741,
|
|
"loss/core": 0.07385687530040741,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.1228384971618652,
|
|
"rewards/margins": 0.998556911945343,
|
|
"rewards/rejected": 2.124281644821167,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7597204802006808,
|
|
"grad_norm": 1608.0,
|
|
"ht_mnpo/logit": 0.07121960818767548,
|
|
"ht_mnpo/residual": 0.06371960043907166,
|
|
"ht_mnpo/residual_abs": 0.16604584455490112,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -2.2949540615081787,
|
|
"logits/rejected": -2.0313305854797363,
|
|
"logps/chosen": -0.3159402906894684,
|
|
"logps/rejected": -0.31764060258865356,
|
|
"loss": 0.4635314345359802,
|
|
"loss/core": 0.4635314345359802,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.100029230117798,
|
|
"rewards/margins": 0.7121961116790771,
|
|
"rewards/rejected": 1.3878331184387207,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7644985964283582,
|
|
"grad_norm": 292.0,
|
|
"ht_mnpo/logit": 0.012357105500996113,
|
|
"ht_mnpo/residual": 0.004857116844505072,
|
|
"ht_mnpo/residual_abs": 0.20216397941112518,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.1980741024017334,
|
|
"logits/rejected": -1.9934120178222656,
|
|
"logps/chosen": -0.24281847476959229,
|
|
"logps/rejected": -0.2602716088294983,
|
|
"loss": 0.5272241234779358,
|
|
"loss/core": 0.5272241234779358,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3460819721221924,
|
|
"rewards/margins": 0.12357117235660553,
|
|
"rewards/rejected": 2.222511053085327,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7692767126560354,
|
|
"grad_norm": 508.0,
|
|
"ht_mnpo/logit": -0.049846429377794266,
|
|
"ht_mnpo/residual": -0.05734643340110779,
|
|
"ht_mnpo/residual_abs": 0.2035159170627594,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -2.2257566452026367,
|
|
"logits/rejected": -1.9966598749160767,
|
|
"logps/chosen": -0.29499825835227966,
|
|
"logps/rejected": -0.32620564103126526,
|
|
"loss": 0.7258022427558899,
|
|
"loss/core": 0.7258022427558899,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.3188430070877075,
|
|
"rewards/margins": -0.4984644055366516,
|
|
"rewards/rejected": 1.8173072338104248,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.7740548288837126,
|
|
"grad_norm": 1.203125,
|
|
"ht_mnpo/logit": -0.05915957689285278,
|
|
"ht_mnpo/residual": -0.06665956974029541,
|
|
"ht_mnpo/residual_abs": 0.12409082800149918,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -2.266317844390869,
|
|
"logits/rejected": -1.98003351688385,
|
|
"logps/chosen": -0.29652053117752075,
|
|
"logps/rejected": -0.2886161804199219,
|
|
"loss": 0.35598623752593994,
|
|
"loss/core": 0.35598623752593994,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.1817328929901123,
|
|
"rewards/margins": -0.5915957689285278,
|
|
"rewards/rejected": 1.7733285427093506,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7788329451113898,
|
|
"grad_norm": 4.53125,
|
|
"ht_mnpo/logit": -0.044502951204776764,
|
|
"ht_mnpo/residual": -0.052002958953380585,
|
|
"ht_mnpo/residual_abs": 0.22781427204608917,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -2.026960849761963,
|
|
"logits/rejected": -1.768654227256775,
|
|
"logps/chosen": -0.2787315249443054,
|
|
"logps/rejected": -0.32032424211502075,
|
|
"loss": 0.4553064703941345,
|
|
"loss/core": 0.4553064703941345,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9568564891815186,
|
|
"rewards/margins": -0.44502943754196167,
|
|
"rewards/rejected": 2.401885747909546,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7836110613390671,
|
|
"grad_norm": 178.0,
|
|
"ht_mnpo/logit": 0.08812891691923141,
|
|
"ht_mnpo/residual": 0.0806289091706276,
|
|
"ht_mnpo/residual_abs": 0.19937752187252045,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -2.264019012451172,
|
|
"logits/rejected": -2.017019271850586,
|
|
"logps/chosen": -0.2904714047908783,
|
|
"logps/rejected": -0.2898346483707428,
|
|
"loss": 0.6707652807235718,
|
|
"loss/core": 0.6707652807235718,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.9655208587646484,
|
|
"rewards/margins": 0.8812893033027649,
|
|
"rewards/rejected": 2.0842318534851074,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.7883891775667443,
|
|
"grad_norm": 102.5,
|
|
"ht_mnpo/logit": 0.14062213897705078,
|
|
"ht_mnpo/residual": 0.13312214612960815,
|
|
"ht_mnpo/residual_abs": 0.18947193026542664,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -2.1509218215942383,
|
|
"logits/rejected": -1.9344791173934937,
|
|
"logps/chosen": -0.30122148990631104,
|
|
"logps/rejected": -0.30858007073402405,
|
|
"loss": 0.46916574239730835,
|
|
"loss/core": 0.46916574239730835,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.392636299133301,
|
|
"rewards/margins": 1.4062215089797974,
|
|
"rewards/rejected": 0.986414909362793,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7931672937944215,
|
|
"grad_norm": 368.0,
|
|
"ht_mnpo/logit": 0.21851785480976105,
|
|
"ht_mnpo/residual": 0.21101781725883484,
|
|
"ht_mnpo/residual_abs": 0.30629879236221313,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -2.0330405235290527,
|
|
"logits/rejected": -1.8500254154205322,
|
|
"logps/chosen": -0.2705790102481842,
|
|
"logps/rejected": -0.3051673471927643,
|
|
"loss": 0.8804137110710144,
|
|
"loss/core": 0.8804137110710144,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.997424602508545,
|
|
"rewards/margins": 2.185178279876709,
|
|
"rewards/rejected": 1.8122460842132568,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7979454100220987,
|
|
"grad_norm": 16.75,
|
|
"ht_mnpo/logit": 0.06802691519260406,
|
|
"ht_mnpo/residual": 0.06052691861987114,
|
|
"ht_mnpo/residual_abs": 0.10366304218769073,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -2.204408884048462,
|
|
"logits/rejected": -2.0475637912750244,
|
|
"logps/chosen": -0.2984268367290497,
|
|
"logps/rejected": -0.3082461953163147,
|
|
"loss": 0.08803679049015045,
|
|
"loss/core": 0.08803679049015045,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8154687881469727,
|
|
"rewards/margins": 0.6802691221237183,
|
|
"rewards/rejected": 1.1351996660232544,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.802723526249776,
|
|
"grad_norm": 94.5,
|
|
"ht_mnpo/logit": 0.1807180941104889,
|
|
"ht_mnpo/residual": 0.17321810126304626,
|
|
"ht_mnpo/residual_abs": 0.2219761312007904,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -2.1479125022888184,
|
|
"logits/rejected": -1.9582202434539795,
|
|
"logps/chosen": -0.3181340992450714,
|
|
"logps/rejected": -0.2776985466480255,
|
|
"loss": 0.5196969509124756,
|
|
"loss/core": 0.5196969509124756,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 3.3365378379821777,
|
|
"rewards/margins": 1.8071807622909546,
|
|
"rewards/rejected": 1.5293573141098022,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8075016424774533,
|
|
"grad_norm": 3.1875,
|
|
"ht_mnpo/logit": 0.021912937983870506,
|
|
"ht_mnpo/residual": 0.014412937685847282,
|
|
"ht_mnpo/residual_abs": 0.0572718009352684,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -2.237180709838867,
|
|
"logits/rejected": -1.9485756158828735,
|
|
"logps/chosen": -0.30848002433776855,
|
|
"logps/rejected": -0.3073003888130188,
|
|
"loss": 0.02010386623442173,
|
|
"loss/core": 0.02010386623442173,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.0542051792144775,
|
|
"rewards/margins": 0.21912936866283417,
|
|
"rewards/rejected": 0.8350757360458374,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8122797587051305,
|
|
"grad_norm": 524.0,
|
|
"ht_mnpo/logit": 0.1235678419470787,
|
|
"ht_mnpo/residual": 0.11606784909963608,
|
|
"ht_mnpo/residual_abs": 0.15284273028373718,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -2.293239116668701,
|
|
"logits/rejected": -2.054229974746704,
|
|
"logps/chosen": -0.3143402934074402,
|
|
"logps/rejected": -0.31967416405677795,
|
|
"loss": 0.381173700094223,
|
|
"loss/core": 0.381173700094223,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.127030849456787,
|
|
"rewards/margins": 1.2356785535812378,
|
|
"rewards/rejected": 0.8913521766662598,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8170578749328078,
|
|
"grad_norm": 10.9375,
|
|
"ht_mnpo/logit": 0.14451901614665985,
|
|
"ht_mnpo/residual": 0.13701900839805603,
|
|
"ht_mnpo/residual_abs": 0.28897708654403687,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -2.1433966159820557,
|
|
"logits/rejected": -1.8137801885604858,
|
|
"logps/chosen": -0.2558881938457489,
|
|
"logps/rejected": -0.2788417935371399,
|
|
"loss": 1.2627418041229248,
|
|
"loss/core": 1.2627418041229248,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 3.3266005516052246,
|
|
"rewards/margins": 1.445190191268921,
|
|
"rewards/rejected": 1.8814102411270142,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.821835991160485,
|
|
"grad_norm": 47.0,
|
|
"ht_mnpo/logit": 0.06459364295005798,
|
|
"ht_mnpo/residual": 0.05709363892674446,
|
|
"ht_mnpo/residual_abs": 0.1335936039686203,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -2.194952964782715,
|
|
"logits/rejected": -2.061500310897827,
|
|
"logps/chosen": -0.3744109570980072,
|
|
"logps/rejected": -0.30319473147392273,
|
|
"loss": 0.15060871839523315,
|
|
"loss/core": 0.15060871839523315,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.501992702484131,
|
|
"rewards/margins": 0.6459364295005798,
|
|
"rewards/rejected": 1.8560559749603271,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.8266141073881622,
|
|
"grad_norm": 1.6015625,
|
|
"ht_mnpo/logit": 0.14823171496391296,
|
|
"ht_mnpo/residual": 0.14073172211647034,
|
|
"ht_mnpo/residual_abs": 0.17731209099292755,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -2.154472827911377,
|
|
"logits/rejected": -1.924818992614746,
|
|
"logps/chosen": -0.3006120026111603,
|
|
"logps/rejected": -0.309847354888916,
|
|
"loss": 0.4654766023159027,
|
|
"loss/core": 0.4654766023159027,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.900441884994507,
|
|
"rewards/margins": 1.4823172092437744,
|
|
"rewards/rejected": 1.418124794960022,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8313922236158394,
|
|
"grad_norm": 1.65625,
|
|
"ht_mnpo/logit": 0.08181913942098618,
|
|
"ht_mnpo/residual": 0.07431914657354355,
|
|
"ht_mnpo/residual_abs": 0.10339722782373428,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -2.1827683448791504,
|
|
"logits/rejected": -1.9179191589355469,
|
|
"logps/chosen": -0.28617894649505615,
|
|
"logps/rejected": -0.2885843515396118,
|
|
"loss": 0.17244181036949158,
|
|
"loss/core": 0.17244181036949158,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": 2.336758852005005,
|
|
"rewards/margins": 0.8181915283203125,
|
|
"rewards/rejected": 1.5185672044754028,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8361703398435167,
|
|
"grad_norm": 11.25,
|
|
"ht_mnpo/logit": 0.05557389184832573,
|
|
"ht_mnpo/residual": 0.04807388782501221,
|
|
"ht_mnpo/residual_abs": 0.18917661905288696,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -2.171966075897217,
|
|
"logits/rejected": -1.9326622486114502,
|
|
"logps/chosen": -0.2600437104701996,
|
|
"logps/rejected": -0.2582940459251404,
|
|
"loss": 0.24842104315757751,
|
|
"loss/core": 0.24842104315757751,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.4055025577545166,
|
|
"rewards/margins": 0.5557388067245483,
|
|
"rewards/rejected": 1.8497635126113892,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840948456071194,
|
|
"grad_norm": 15.9375,
|
|
"ht_mnpo/logit": 0.0831659659743309,
|
|
"ht_mnpo/residual": 0.07566596567630768,
|
|
"ht_mnpo/residual_abs": 0.11803103983402252,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -2.134854555130005,
|
|
"logits/rejected": -1.8504482507705688,
|
|
"logps/chosen": -0.25479409098625183,
|
|
"logps/rejected": -0.27924638986587524,
|
|
"loss": 0.12039796262979507,
|
|
"loss/core": 0.12039796262979507,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.021028757095337,
|
|
"rewards/margins": 0.8316596150398254,
|
|
"rewards/rejected": 1.1893693208694458,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8457265722988712,
|
|
"grad_norm": 19.625,
|
|
"ht_mnpo/logit": 0.0474698506295681,
|
|
"ht_mnpo/residual": 0.039969850331544876,
|
|
"ht_mnpo/residual_abs": 0.05242999643087387,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -2.2313616275787354,
|
|
"logits/rejected": -2.0359046459198,
|
|
"logps/chosen": -0.328023761510849,
|
|
"logps/rejected": -0.3314855694770813,
|
|
"loss": 0.025688689202070236,
|
|
"loss/core": 0.025688689202070236,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.1934434175491333,
|
|
"rewards/margins": 0.474698543548584,
|
|
"rewards/rejected": 0.7187449336051941,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8505046885265484,
|
|
"grad_norm": 207.0,
|
|
"ht_mnpo/logit": 0.13611260056495667,
|
|
"ht_mnpo/residual": 0.12861260771751404,
|
|
"ht_mnpo/residual_abs": 0.22040756046772003,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -2.1162843704223633,
|
|
"logits/rejected": -1.941434621810913,
|
|
"logps/chosen": -0.31317272782325745,
|
|
"logps/rejected": -0.3353462815284729,
|
|
"loss": 0.7812239527702332,
|
|
"loss/core": 0.7812239527702332,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.680392265319824,
|
|
"rewards/margins": 1.361126184463501,
|
|
"rewards/rejected": 1.3192662000656128,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.8552828047542257,
|
|
"grad_norm": 588.0,
|
|
"ht_mnpo/logit": -0.056675732135772705,
|
|
"ht_mnpo/residual": -0.06417573988437653,
|
|
"ht_mnpo/residual_abs": 0.21688473224639893,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -2.205193042755127,
|
|
"logits/rejected": -1.8673622608184814,
|
|
"logps/chosen": -0.29924076795578003,
|
|
"logps/rejected": -0.3369528651237488,
|
|
"loss": 0.5442713499069214,
|
|
"loss/core": 0.5442713499069214,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.8029203414916992,
|
|
"rewards/margins": -0.566757321357727,
|
|
"rewards/rejected": 2.369677782058716,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8600609209819029,
|
|
"grad_norm": 1.1796875,
|
|
"ht_mnpo/logit": -0.059881217777729034,
|
|
"ht_mnpo/residual": -0.06738121807575226,
|
|
"ht_mnpo/residual_abs": 0.10789163410663605,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -2.390237331390381,
|
|
"logits/rejected": -2.1097664833068848,
|
|
"logps/chosen": -0.30715036392211914,
|
|
"logps/rejected": -0.3250555992126465,
|
|
"loss": 0.425441175699234,
|
|
"loss/core": 0.425441175699234,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 0.8894049525260925,
|
|
"rewards/margins": -0.5988121628761292,
|
|
"rewards/rejected": 1.4882172346115112,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8600609209819029,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.3132047750718064,
|
|
"train_runtime": 7069.0811,
|
|
"train_samples_per_second": 2.037,
|
|
"train_steps_per_second": 0.127
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|